chore(release): 6.12.2 [skip ci]

## [6.12.2](https://git.datacontroller.io/dc/dc/compare/v6.12.1...v6.12.2) (2025-01-27)

### Bug Fixes

* unnecessary zeros when adding new row (data schema default values) ([a1a9051](a1a90519c5))

CHANGELOG.md

@@ -1,3 +1,44 @@
+## [6.12.2](https://git.datacontroller.io/dc/dc/compare/v6.12.1...v6.12.2) (2025-01-27)
+
+
+### Bug Fixes
+
+* unnecessary zeros when adding new row (data schema default values) ([a1a9051](https://git.datacontroller.io/dc/dc/commit/a1a90519c535ca25e00822b4d3358c991ac9662e))
+
+## [6.12.1](https://git.datacontroller.io/dc/dc/compare/v6.12.0...v6.12.1) (2024-12-31)
+
+
+### Bug Fixes
+
+* no upcase of pk fields in MPE_TABLES in delete scenario ([3de095f](https://git.datacontroller.io/dc/dc/commit/3de095fe7797cde60f0e232c188305fe423c27eb)), closes [#134](https://git.datacontroller.io/dc/dc/issues/134)
+* reduce length of tmp table names.  Closes [#130](https://git.datacontroller.io/dc/dc/issues/130) ([f9c2491](https://git.datacontroller.io/dc/dc/commit/f9c2491ab6e7b528b7ffc011fd9e45c963c5f6bf))
+
+# [6.12.0](https://git.datacontroller.io/dc/dc/compare/v6.11.1...v6.12.0) (2024-09-02)
+
+
+### Bug Fixes
+
+* added appLoc to the system page ([dd2138a](https://git.datacontroller.io/dc/dc/commit/dd2138ac5e6067de310e83d16fccc9b9764ba3ff))
+* bumping core for passthrough fix, [#124](https://git.datacontroller.io/dc/dc/issues/124) ([caa9854](https://git.datacontroller.io/dc/dc/commit/caa9854ff0431ccbb6ff1d6d3509dc877362cceb))
+* excel with password flow, introducing web worker for XLSX.read ([a3ce367](https://git.datacontroller.io/dc/dc/commit/a3ce36795007a4e3b6ac3499ffd119dc3758f387))
+* implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height ([4218da9](https://git.datacontroller.io/dc/dc/commit/4218da91cd193aa45346ad7e34ccc00ca89df4fb))
+* **multi load:** xlsx read file ahead of time, while user choose datasets ([6547461](https://git.datacontroller.io/dc/dc/commit/65474616379e1dacc1329b3bdc5eb14f34428bb1))
+* refactored adapter request wrapper function to return job log as well ([67436f4](https://git.datacontroller.io/dc/dc/commit/67436f4ff9bb4d77d5f897f47a3e3d472981f275))
+* using temporary names for temporary tables ([ce50365](https://git.datacontroller.io/dc/dc/commit/ce503653cd9fc36f72fb172bd14816e07c792e14)), closes [#124](https://git.datacontroller.io/dc/dc/issues/124)
+
+
+### Features
+
+* searching data in excel files using new algorithm (massive performance improvement) ([bbb725c](https://git.datacontroller.io/dc/dc/commit/bbb725c64cc23ed701b189623992408c42fdde8f))
+
+## [6.11.1](https://git.datacontroller.io/dc/dc/compare/v6.11.0...v6.11.1) (2024-07-02)
+
+
+### Bug Fixes
+
+* adding SYSSITE, part of [#116](https://git.datacontroller.io/dc/dc/issues/116) ([a156c01](https://git.datacontroller.io/dc/dc/commit/a156c0111b3de5e3744e38d377d6e9aa09915803))
+* ensuring review_reason_txt in output.  Closes [#117](https://git.datacontroller.io/dc/dc/issues/117) ([e5d93fd](https://git.datacontroller.io/dc/dc/commit/e5d93fd7d6d86bc47ff56664bd812b4d9d0749a5))
+
 # [6.11.0](https://git.datacontroller.io/dc/dc/compare/v6.10.1...v6.11.0) (2024-06-27)
 
 

client/angular.json

@@ -69,7 +69,8 @@
             ],
             "scripts": [
               "node_modules/marked/marked.min.js"
-            ]
+            ],
+            "webWorkerTsConfig": "tsconfig.worker.json"
           },
           "configurations": {
             "production": {
@@ -148,7 +149,8 @@
               "src/styles.scss"
             ],
             "scripts": [],
-            "karmaConfig": "karma.conf.js"
+            "karmaConfig": "karma.conf.js",
+            "webWorkerTsConfig": "tsconfig.worker.json"
           }
         },
         "lint": {

client/cypress/e2e/excel-multi-load.cy.ts

@@ -192,7 +192,10 @@ const checkIfTreeHasTables = (tables: string[], clickOnNode?: string, callback?:
       }
     }
 
-    if (nodeToClick) cy.get(nodeToClick).trigger('click')
+    if (nodeToClick) {
+      cy.wait(1000)
+      cy.get(nodeToClick).trigger('click')
+    }
 
     if (callback) callback(nodesCorrect)
   })
@@ -201,6 +204,7 @@ const checkIfTreeHasTables = (tables: string[], clickOnNode?: string, callback?:
 const submitTables = () => {
   cy.get('#submit-all').trigger('click')
   cy.get('#submit-tables').trigger('click')
+  cy.wait(1000)
 }
 
 const hasSuccessSubmits = (expectedNoOfSubmits: number, callback: (valid: boolean) => void) => {

client/cypress/e2e/excel.cy.ts

@@ -109,13 +109,8 @@ context('excel tests: ', function () {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
 
     attachExcelFile('duplicate_column_excel.xlsx', () => {
-      cy.get('.abortMsg', { timeout: longerCommandTimeout })
+      submitExcel()
-        .should('exist')
+      rejectExcel(done)
-        .then((elements: any) => {
-          if (elements[0]) {
-            if (elements[0].innerText.toLowerCase().includes('missing')) done()
-          }
-        })
     })
   })
 

client/cypress/tsconfig.json

@@ -6,5 +6,8 @@
       "lib": ["es2019", "dom"],
       "types": ["cypress", "cypress-real-events"]
     },
-    "include": ["**/*.ts"]
+    "include": [
+      "**/*.ts",
+      "../cypress.config.ts"
+    ]
   }

client/package-lock.json

@@ -52,6 +52,7 @@
         "tslib": "^2.3.0",
         "vm": "^0.1.0",
         "webpack": "^5.91.0",
+        "xlsx": "^0.18.5",
         "zone.js": "~0.14.4"
       },
       "devDependencies": {
@@ -7122,6 +7123,14 @@
         "node": ">=8.9.0"
       }
     },
+    "node_modules/adler-32": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
+      "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/agent-base": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
@@ -8259,6 +8268,18 @@
       "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
       "dev": true
     },
+    "node_modules/cfb": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
+      "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
+      "dependencies": {
+        "adler-32": "~1.3.0",
+        "crc-32": "~1.2.0"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -8579,6 +8600,14 @@
       "integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==",
       "dev": true
     },
+    "node_modules/codepage": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
+      "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/color-convert": {
       "version": "1.9.3",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -8966,6 +8995,17 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/crc-32": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
+      "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
+      "bin": {
+        "crc32": "bin/crc32.njs"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/create-ecdh": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
@@ -11578,6 +11618,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/frac": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
+      "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/fraction.js": {
       "version": "4.3.7",
       "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz",
@@ -19340,6 +19388,17 @@
       "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
       "dev": true
     },
+    "node_modules/ssf": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
+      "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
+      "dependencies": {
+        "frac": "~1.1.2"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/sshpk": {
       "version": "1.18.0",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@@ -21323,6 +21382,22 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/wmf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
+      "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/word": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
+      "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/word-wrap": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@@ -21469,6 +21544,26 @@
         }
       }
     },
+    "node_modules/xlsx": {
+      "version": "0.18.5",
+      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
+      "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
+      "dependencies": {
+        "adler-32": "~1.3.0",
+        "cfb": "~1.2.1",
+        "codepage": "~1.15.0",
+        "crc-32": "~1.2.1",
+        "ssf": "~0.11.2",
+        "wmf": "~1.0.1",
+        "word": "~0.3.0"
+      },
+      "bin": {
+        "xlsx": "bin/xlsx.njs"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/xmldoc": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz",

client/package.json

@@ -80,6 +80,7 @@
     "tslib": "^2.3.0",
     "vm": "^0.1.0",
     "webpack": "^5.91.0",
+    "xlsx": "^0.18.5",
     "zone.js": "~0.14.4"
   },
   "devDependencies": {

client/src/app/app.component.ts

@@ -18,6 +18,7 @@ import {
   ClarityIcons,
   exclamationTriangleIcon,
   moonIcon,
+  processOnVmIcon,
   sunIcon,
   tableIcon,
   trashIcon
@@ -28,7 +29,8 @@ ClarityIcons.addIcons(
   sunIcon,
   exclamationTriangleIcon,
   tableIcon,
-  trashIcon
+  trashIcon,
+  processOnVmIcon
 )
 
 @Component({

client/src/app/deploy/sections/manual/manual.component.ts

@@ -1,6 +1,7 @@
 import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core'
 import SASjs, { SASjsConfig } from '@sasjs/adapter'
 import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 import { DeployService } from 'src/app/services/deploy.service'
 import { EventService } from 'src/app/services/event.service'
 import { LoggerService } from 'src/app/services/logger.service'
@@ -303,10 +304,10 @@ export class ManualComponent implements OnInit {
 
     this.sasService
       .request('public/startupservice', null)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        this.loggerService.log(res)
+        this.loggerService.log(res.adapterResponse)
 
-        if (res.saslibs) {
+        if (res.adapterResponse.saslibs) {
           this.validationState = 'success'
         } else {
           this.validationState = 'error'

client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts

@@ -3,6 +3,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'
 import SASjs, { SASjsConfig } from '@sasjs/adapter'
 import { ServerType } from '@sasjs/utils/types/serverType'
 import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 import { SASGroup } from 'src/app/models/sas/public-getgroups.model'
 import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model'
 import { SasService } from 'src/app/services/sas.service'
@@ -68,11 +69,11 @@ export class SasjsConfiguratorComponent implements OnInit {
     this.loading = true
 
     this.sasService.request('usernav/usergroupsbymember', null).then(
-      (res: any) => {
+      (res: RequestWrapperResponse) => {
-        this.METAPERSON = res.MF_GETUSER
+        this.METAPERSON = res.adapterResponse.MF_GETUSER
-        this.SYSUSERID = res.SYSUSERID
+        this.SYSUSERID = res.adapterResponse.SYSUSERID
-        this.SYSHOSTNAME = res.SYSHOSTNAME
+        this.SYSHOSTNAME = res.adapterResponse.SYSHOSTNAME
-        this.SYSVLONG = res.SYSVLONG
+        this.SYSVLONG = res.adapterResponse.SYSVLONG
 
         /*
           We would like to present a default DCPATH (deployment path) to the
@@ -88,12 +89,14 @@ export class SasjsConfiguratorComponent implements OnInit {
         */
         this.dcDirectory =
           this.tmpDirectories[
-            ['L', 'H', 'A', 'S'].includes(res.SYSSCPL.substring(0, 1))
+            ['L', 'H', 'A', 'S'].includes(
+              res.adapterResponse.SYSSCPL.substring(0, 1)
+            )
               ? 'linux'
               : 'windows'
           ]
 
-        this.dcAdminGroupList = res.groups
+        this.dcAdminGroupList = res.adapterResponse.groups
         this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME
 
         this.loading = false

client/src/app/editor/editor.component.ts

@@ -57,10 +57,10 @@ import { LicenceService } from '../services/licence.service'
 import * as numbro from 'numbro'
 import * as languages from 'numbro/dist/languages.min'
 import { FileUploadEncoding } from '../models/FileUploadEncoding'
-import {
+import { SpreadsheetService } from '../services/spreadsheet.service'
-  ParseResult,
+import { UploadFileResponse } from '../models/UploadFile'
-  SpreadsheetService
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
-} from '../services/spreadsheet.service'
+import { ParseResult } from '../models/ParseResult.interface'
 
 @Component({
   selector: 'app-editor',
@@ -571,20 +571,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
     this.sasService
       .uploadFile(this.uploadUrl, filesToUpload, { table: this.libds })
       .then(
-        (res: any) => {
+        (res: UploadFileResponse) => {
-          if (typeof res.sasjsAbort === 'undefined') {
+          if (typeof res.adapterResponse.sasjsAbort === 'undefined') {
-            if (typeof res.sasparams === 'undefined') {
+            if (typeof res.adapterResponse.sasparams === 'undefined') {
               return
             } else {
               this.uploadLoading = false
-              let params = res.sasparams[0]
+              let params = res.adapterResponse.sasparams[0]
               this.successEnable = true
               this.tableId = params.DSID
               this.router.navigateByUrl('/stage/' + this.tableId)
             }
           } else {
             // handle succesfull response
-            const abortRes = res
+            const abortRes = res.adapterResponse
             const abortMsg = abortRes.sasjsAbort[0].MSG
             const macMsg = abortRes.sasjsAbort[0].MAC
 
@@ -609,7 +609,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
             this.fileUploadInputCompList.first.nativeElement.value = ''
           }
           this.uploader.queue = []
-          this.eventService.catchResponseError('file upload', err)
+          this.eventService.catchResponseError(
+            'file upload',
+            err.adapterResponse
+          )
         }
       )
   }
@@ -1561,9 +1564,11 @@ export class EditorComponent implements OnInit, AfterViewInit {
             'editors/stagedata',
             this.$dataFormats
           )
-          .then((res: any) => {
+          .then((res: RequestWrapperResponse) => {
-            if (typeof res.sasparams !== 'undefined') {
+            if (typeof res.adapterResponse.sasparams !== 'undefined') {
-              this.router.navigateByUrl('/stage/' + res.sasparams[0].DSID)
+              this.router.navigateByUrl(
+                '/stage/' + res.adapterResponse.sasparams[0].DSID
+              )
 
               return true
             }
@@ -1591,7 +1596,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
             this.disableSubmit = true
             this.submit = false
 
-            let errorText = typeof err === 'string' ? err : JSON.stringify(err)
+            let errorText =
+              typeof err.adapterRespnse === 'string'
+                ? err.adapterRespnse
+                : JSON.stringify(err.adapterRespnse)
 
             this.eventService.showAbortModal(
               'editors/stagedata',
@@ -1983,8 +1991,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
             suppressSuccessAbortModal: true,
             suppressErrorAbortModal: true
           })
-          .then((res: any) => {
+          .then((res: RequestWrapperResponse) => {
-            const colSource = res.dynamic_values.map(
+            const colSource = res.adapterResponse.dynamic_values.map(
               (el: any) => el[this.cellValidationFields.RAW_VALUE]
             )
 
@@ -1998,8 +2006,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
                   ...this.cellValidationSource[validationSourceIndex],
                   row: row,
                   col: column,
-                  values: res.dynamic_values,
+                  values: res.adapterResponse.dynamic_values,
-                  extended_values: res.dynamic_extended_values
+                  extended_values: res.adapterResponse.dynamic_extended_values
                 }
               }
 

client/src/app/group/group.component.ts

@@ -6,6 +6,7 @@ import { ServerType } from '@sasjs/utils/types/serverType'
 import { HelperService } from '../services/helper.service'
 import { SasService } from '../services/sas.service'
 import { globals } from '../_globals'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-group',
@@ -82,11 +83,13 @@ export class GroupComponent implements OnInit {
               globals.usernav.groupList = groups
             })
         } else {
-          this.sasService.request('public/getgroups', null).then((res: any) => {
+          this.sasService
-            this.loading = false
+            .request('public/getgroups', null)
-            this.groups = res.groups
+            .then((res: RequestWrapperResponse) => {
-            globals.usernav.groupList = res.groups
+              this.loading = false
-          })
+              this.groups = res.adapterResponse.groups
+              globals.usernav.groupList = res.adapterResponse.groups
+            })
         }
       } else {
         this.groups = globals.usernav.groupList
@@ -128,14 +131,15 @@ export class GroupComponent implements OnInit {
           let data = { iwant: [{ groupid: this.paramURI }] }
           this.sasService
             .request('usernav/usermembersbygroup', data)
-            .then((res: any) => {
+            .then((res: RequestWrapperResponse) => {
-              this.groupMembers = res.sasmembers
+              this.groupMembers = res.adapterResponse.sasmembers
-              this.groupMemberCount = res.sasmembers.length
+              this.groupMemberCount = res.adapterResponse.sasmembers.length
-              if (res.sasmembers[0] !== undefined) {
+              if (res.adapterResponse.sasmembers[0] !== undefined) {
                 this.loading = false
-                this.groupUri = res.sasmembers[0].URIMEM || this.paramURI
+                this.groupUri =
-                this.groupName = res.sasmembers[0].GROUPNAME
+                  res.adapterResponse.sasmembers[0].URIMEM || this.paramURI
-                this.groupDesc = res.sasmembers[0].GROUPDESC
+                this.groupName = res.adapterResponse.sasmembers[0].GROUPNAME
+                this.groupDesc = res.adapterResponse.sasmembers[0].GROUPDESC
 
                 if (!this.groupName) {
                   this.groupName = this.paramURI
@@ -202,13 +206,13 @@ export class GroupComponent implements OnInit {
 
       this.sasService
         .request('usernav/usermembersbygroup', data)
-        .then((res: any) => {
+        .then((res: RequestWrapperResponse) => {
           this.loading = false
           this.groupUri = group.GROUPURI
           this.groupName = group.GROUPNAME
           this.groupDesc = group.GROUPDESC
-          this.groupMembers = res.sasmembers
+          this.groupMembers = res.adapterResponse.sasmembers
-          this.groupMemberCount = res.sasmembers.length
+          this.groupMemberCount = res.adapterResponse.sasmembers.length
         })
     }
   }

client/src/app/licensing/licensing.component.ts

@@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'
 import { ActivatedRoute, Router } from '@angular/router'
 import { AppService, LicenceService, SasService } from '../services'
 import { LicenseKeyData } from '../models/LicenseKeyData'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 enum LicenseActions {
   key = 'key',
@@ -116,8 +117,12 @@ export class LicensingComponent implements OnInit {
 
     this.sasService
       .request('admin/registerkey', table)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
+        if (
+          res.adapterResponse.return &&
+          res.adapterResponse.return[0] &&
+          res.adapterResponse.return[0].MSG === 'SUCCESS'
+        ) {
           location.replace(location.href.split('#')[0])
         }
       })

client/src/app/lineage/lineage.component.ts

@@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
 import * as saveSvg from 'save-svg-as-png'
 import { LoggerService } from '../services/logger.service'
 import { LicenceService } from '../services/licence.service'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 const moment = require('moment')
 
 @Component({
@@ -115,8 +116,8 @@ export class LineageComponent {
 
     await this.sasService
       .request('lineage/getmetacols', libTable)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        this.columnsList = res.metacols
+        this.columnsList = res.adapterResponse.metacols
         if (this.columnsList && this.columnsList.length > 0) {
           // this.column = this.columnsList[0]['COLURI']
 
@@ -174,8 +175,8 @@ export class LineageComponent {
     let libTable = { SASControlTable: [{ liburi: $event }] }
     await this.sasService
       .request('lineage/getmetatables', libTable)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        this.tablesList = res.metatables
+        this.tablesList = res.adapterResponse.metatables
 
         if (this.tablesList && this.tablesList.length > 0) {
           library['tables'] = this.tablesList
@@ -295,8 +296,8 @@ export class LineageComponent {
       } else {
         await this.sasService
           .request('public/viewlibs', null)
-          .then((res: any) => {
+          .then((res: RequestWrapperResponse) => {
-            this.libraryList = res.saslibs
+            this.libraryList = res.adapterResponse.saslibs
             this.helperService.displayLibraries(this.libraryList)
 
             if (this.libraryList) {
@@ -402,8 +403,8 @@ export class LineageComponent {
     return new Promise<void>((resolve, reject) => {
       this.sasService
         .request('lineage/fetchtablelineage', libTable)
-        .then(async (res: any) => {
+        .then(async (res: RequestWrapperResponse) => {
-          if (res.flatdata.length > 0) {
+          if (res.adapterResponse.flatdata.length > 0) {
             if (this.licenceService.checkLineageLimit()) {
               this.eventService.showInfoModal(
                 'Notice',
@@ -421,20 +422,22 @@ export class LineageComponent {
           }
 
           this.lineageTableName =
-            res.info[0].LIBREF + '.' + res.info[0].TABLENAME
+            res.adapterResponse.info[0].LIBREF +
+            '.' +
+            res.adapterResponse.info[0].TABLENAME
 
-          let dotArray = res.finalfinal
+          let dotArray = res.adapterResponse.finalfinal
           let vizTmp: string = ''
 
           for (let i = 0; i < dotArray.length; i++) {
             vizTmp += unescape(dotArray[i].LINE) + '\n'
           }
 
-          this.flatdata = res.flatdata
+          this.flatdata = res.adapterResponse.flatdata
 
           if (this.libraryList) {
             let libraryToSelect = this.libraryList.find((library: any) =>
-              res.info[0].LIBURI.toUpperCase().includes(
+              res.adapterResponse.info[0].LIBURI.toUpperCase().includes(
                 library.LIBRARYID.toUpperCase()
               )
             )
@@ -450,7 +453,7 @@ export class LineageComponent {
               if (libraryToSelect['tables']) {
                 tableToSelect = libraryToSelect['tables'].find((table: any) =>
                   table.TABLEURI.toUpperCase().includes(
-                    res.info[0].TABLEID.toUpperCase()
+                    res.adapterResponse.info[0].TABLEID.toUpperCase()
                   )
                 )
 
@@ -495,10 +498,10 @@ export class LineageComponent {
             .replace(/\sds:/g, '\nds:')
             .replace(/\s\n/g, '\n')
 
-          this.idlookup = res.idlookup
+          this.idlookup = res.adapterResponse.idlookup
 
-          if (res.finalfinal.length > this.largeDotFileLimit) {
+          if (res.adapterResponse.finalfinal.length > this.largeDotFileLimit) {
-            this.largeDotFileLines = res.finalfinal.length
+            this.largeDotFileLines = res.adapterResponse.finalfinal.length
           } else {
             this.buildGraph()
           }
@@ -619,8 +622,8 @@ export class LineageComponent {
     return new Promise<void>((resolve, reject) => {
       this.sasService
         .request('lineage/fetchcollineage', libTable)
-        .then(async (res: any) => {
+        .then(async (res: RequestWrapperResponse) => {
-          if (res.flatdata.length > 0) {
+          if (res.adapterResponse.flatdata.length > 0) {
             if (this.licenceService.checkLineageLimit()) {
               this.eventService.showInfoModal(
                 'Notice',
@@ -631,18 +634,21 @@ export class LineageComponent {
             }
           }
 
-          if (typeof res === 'string') {
+          if (typeof res.adapterResponse === 'string') {
             this.vizInput = 'digraph G {SAS Error}'
             this.buildGraph()
             return
           }
 
-          this.lineageTableName = res.info[0].LIBREF + '.' + res.info[0].TABNAME
+          this.lineageTableName =
-          this.lineageColumnName = res.info[0].COLNAME
+            res.adapterResponse.info[0].LIBREF +
+            '.' +
+            res.adapterResponse.info[0].TABNAME
+          this.lineageColumnName = res.adapterResponse.info[0].COLNAME
 
-          this.idlookup = res.idlookup
+          this.idlookup = res.adapterResponse.idlookup
 
-          let dotArray = res.fromsas
+          let dotArray = res.adapterResponse.fromsas
           let vizTmp: string = ''
           for (let i = 0; i < dotArray.length; i++) {
             vizTmp += unescape(dotArray[i].STRING) + '\n'
@@ -653,11 +659,11 @@ export class LineageComponent {
             .replace(/\sds:/g, '\nds:')
             .replace(/\s\n/g, '\n')
 
-          this.flatdata = res.flatdata
+          this.flatdata = res.adapterResponse.flatdata
 
           if (this.libraryList) {
             let libraryToSelect = this.libraryList.find((library: any) =>
-              res.info[0]?.LIBURI?.toUpperCase()?.includes(
+              res.adapterResponse.info[0]?.LIBURI?.toUpperCase()?.includes(
                 library?.LIBRARYID?.toUpperCase()
               )
             )
@@ -672,7 +678,8 @@ export class LineageComponent {
 
               if (libraryToSelect['tables']) {
                 tableToSelect = libraryToSelect['tables'].find(
-                  (table: any) => table.TABLEURI === res.info[0].TABURI
+                  (table: any) =>
+                    table.TABLEURI === res.adapterResponse.info[0].TABURI
                 )
 
                 if (tableToSelect) {
@@ -714,8 +721,8 @@ export class LineageComponent {
             }
           }
 
-          if (res.fromsas.length > this.largeDotFileLimit) {
+          if (res.adapterResponse.fromsas.length > this.largeDotFileLimit) {
-            this.largeDotFileLines = res.fromsas.length
+            this.largeDotFileLines = res.adapterResponse.fromsas.length
           } else {
             this.buildGraph()
           }

client/src/app/metadata/metadata.component.ts

@@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
 import { globals } from '../_globals'
 
 import { Injectable } from '@angular/core'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 interface MetaData {
   NAME: any
@@ -109,44 +110,52 @@ export class MetadataComponent implements OnInit {
       this.metatypesLoading = false
       this.metaDataSearch = globals.metadata.metaDataSearch
     } else {
-      this.sasService.request('metanav/metatypes', null).then((res: any) => {
+      this.sasService
-        this.metaDataList = res.types
+        .request('metanav/metatypes', null)
-        globals.metadata.metaDataList = this.metaDataList
+        .then((res: RequestWrapperResponse) => {
-        this.loading = false
+          this.metaDataList = res.adapterResponse.types
-        this.metatypesLoading = false
+          globals.metadata.metaDataList = this.metaDataList
-      })
+          this.loading = false
+          this.metatypesLoading = false
+        })
 
-      this.sasService.request('metanav/metarepos', null).then((res: any) => {
+      this.sasService
-        let foundation = false
+        .request('metanav/metarepos', null)
-        this.repositories = []
+        .then((res: RequestWrapperResponse) => {
-        for (let index = 0; index < res.outrepos.length; index++) {
+          let foundation = false
-          this.repositories.push(res.outrepos[index].NAME)
+          this.repositories = []
-          if (res.outrepos[index].NAME === 'Foundation') {
+          for (
-            foundation = true
+            let index = 0;
+            index < res.adapterResponse.outrepos.length;
+            index++
+          ) {
+            this.repositories.push(res.adapterResponse.outrepos[index].NAME)
+            if (res.adapterResponse.outrepos[index].NAME === 'Foundation') {
+              foundation = true
+            }
           }
-        }
+          if (foundation) {
-        if (foundation) {
+            this.repository = 'Foundation'
-          this.repository = 'Foundation'
+          } else {
-        } else {
+            this.repository = res.adapterResponse.outrepos[0].NAME
-          this.repository = res.outrepos[0].NAME
+          }
-        }
+          globals.metadata.metaRepositories = this.repositories
-        globals.metadata.metaRepositories = this.repositories
+          globals.metadata.selectedRepository = this.repository
-        globals.metadata.selectedRepository = this.repository
+          if (this.objectRoute) {
-        if (this.objectRoute) {
+            this.eventService.closeSidebar()
-          this.eventService.closeSidebar()
+            this.showData = true
-          this.showData = true
+            let name = ''
-          let name = ''
+            let id = this.route.snapshot.params['objectID']
-          let id = this.route.snapshot.params['objectID']
+            // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
-          // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
+            this.metaObjectList = []
-          this.metaObjectList = []
+            this.metaObjectList.push({ ID: id, NAME: name })
-          this.metaObjectList.push({ ID: id, NAME: name })
+            this.metaObjectShowList = this.metaObjectList
-          this.metaObjectShowList = this.metaObjectList
+            this.metaObjectOnClick(
-          this.metaObjectOnClick(
+              this.metaObjectShowList[0].ID,
-            this.metaObjectShowList[0].ID,
+              this.metaObjectShowList[0]
-            this.metaObjectShowList[0]
+            )
-          )
+          }
-        }
+        })
-      })
     }
   }
 
@@ -183,56 +192,64 @@ export class MetadataComponent implements OnInit {
     const data: any = {
       SASControlTable: [{ metatype: $event, repo: this.repository }]
     }
-    this.sasService.request('metanav/metaobjects', data).then((res: any) => {
+    this.sasService
-      this.metaObjectList = res.objects
+      .request('metanav/metaobjects', data)
-      this.getMetaObjectAttributes(this.metaObjectSize)
+      .then((res: RequestWrapperResponse) => {
-      this.loading = false
+        this.metaObjectList = res.adapterResponse.objects
-      this.assoTypeSelected = $event
+        this.getMetaObjectAttributes(this.metaObjectSize)
-      this.eventService.closeSidebar()
+        this.loading = false
-      this.showData = true
+        this.assoTypeSelected = $event
-    })
+        this.eventService.closeSidebar()
+        this.showData = true
+      })
   }
 
   public async selectmetaObject($event: any, metaData?: any) {
     let data: any = {
       SASControlTable: [{ objecturi: $event }]
     }
-    this.sasService.request('metanav/metadetails', data).then((res: any) => {
+    this.sasService
-      this.metaObjectAssociations = res.associations
+      .request('metanav/metadetails', data)
-      this.root$ = of(this.getAssosiationsCount(res.associations))
+      .then((res: RequestWrapperResponse) => {
-      this.showAcc = true
+        this.metaObjectAssociations = res.adapterResponse.associations
-      this.showTable = true
+        this.root$ = of(
-      let metaObjectName = res.attributes.find(
+          this.getAssosiationsCount(res.adapterResponse.associations)
-        (x: any) => x.NAME === 'Name'
-      ).VALUE
-      this.assoObjectSelected = metaObjectName
-      metaData.NAME = metaObjectName
-      let url = this.router.url
-      if (this.objectRoute) {
-        // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
-        this.location.replaceState(
-          url.slice(0, url.lastIndexOf('object')) +
-            'object/' +
-            $event.slice(1 + $event.indexOf('\\'))
         )
-      } else {
+        this.showAcc = true
-        // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+        this.showTable = true
-        this.location.replaceState(
+        let metaObjectName = res.adapterResponse.attributes.find(
-          url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
+          (x: any) => x.NAME === 'Name'
-        )
+        ).VALUE
-      }
+        this.assoObjectSelected = metaObjectName
-      this.metaObjectAttributes = res.attributes
+        metaData.NAME = metaObjectName
-    })
+        let url = this.router.url
+        if (this.objectRoute) {
+          // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+          this.location.replaceState(
+            url.slice(0, url.lastIndexOf('object')) +
+              'object/' +
+              $event.slice(1 + $event.indexOf('\\'))
+          )
+        } else {
+          // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+          this.location.replaceState(
+            url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
+          )
+        }
+        this.metaObjectAttributes = res.adapterResponse.attributes
+      })
   }
 
   public async selectAssosiationsDetails($event: any, metaData?: any) {
     let data: any = {
       SASControlTable: [{ objecturi: $event }]
     }
-    this.sasService.request('metanav/metadetails', data).then((res: any) => {
+    this.sasService
-      this.metaObjectAttributes = res.attributes
+      .request('metanav/metadetails', data)
-      this.showTable = true
+      .then((res: RequestWrapperResponse) => {
-    })
+        this.metaObjectAttributes = res.adapterResponse.attributes
+        this.showTable = true
+      })
   }
 
   public getAssosiationsCount(assosiationList: Array<any>) {
@@ -244,7 +261,7 @@ export class MetadataComponent implements OnInit {
           details: []
         })
       }
-      let assocObj = assosiationsHash.get(assosiation.ASSOC)
+      let assocObj: any = assosiationsHash.get(assosiation.ASSOC)
       assocObj.count++
       assocObj.details.push({
         ASSOCURI: assosiation.ASSOCURI,
@@ -254,7 +271,7 @@ export class MetadataComponent implements OnInit {
       })
     }
     let assocGrouped: Array<any> = []
-    assosiationsHash.forEach(function (val, key) {
+    assosiationsHash.forEach(function (val: any, key) {
       assocGrouped.push({
         ASSOC: key,
         count: val.count,
@@ -294,9 +311,9 @@ export class MetadataComponent implements OnInit {
     }
     return this.sasService
       .request('metanav/metadetails', data)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
         this.showTable = true
-        this.metaObjectAttributes = res.attributes
+        this.metaObjectAttributes = res.adapterResponse.attributes
         this.assoObjectSelected = asso.NAME
         let url = this.router.url
         if (this.objectRoute) {
@@ -314,7 +331,7 @@ export class MetadataComponent implements OnInit {
               asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\'))
           )
         }
-        return this.getAssosiationsCount(res.associations)
+        return this.getAssosiationsCount(res.adapterResponse.associations)
       })
   }
 

client/src/app/models/ParseParams.interface.ts

@@ -0,0 +1,29 @@
+import { DcValidator } from '../shared/dc-validator/dc-validator'
+import { FileUploadEncoding } from './FileUploadEncoding'
+import { FileUploader } from './FileUploader.class'
+import { ExcelRule } from './TableData'
+import XLSX from 'xlsx'
+
+export interface ParseParams {
+  file: File
+  password?: string
+  dcValidator: DcValidator
+  /**
+   * If workbook is provided, parse function will not run a XLSX.read()
+   * it will use this property instead. So the client must do a file read beforehand
+   */
+  workbook?: XLSX.WorkBook
+  /**
+   * Parse function will manipulate and return the uploader array which can be provided with files already in the queue
+   * Otherwise new empty instance will be created.
+   */
+  uploader?: FileUploader
+  headerPks: string[]
+  headerArray: string[]
+  headerShow: string[]
+  timeHeaders: string[]
+  dateHeaders: string[]
+  dateTimeHeaders: string[]
+  xlRules: ExcelRule[]
+  encoding?: FileUploadEncoding
+}

client/src/app/models/ParseResult.interface.ts

@@ -0,0 +1,15 @@
+import { FileUploader } from './FileUploader.class'
+import FoundRangeInfo from './RangeInfo'
+
+export interface ParseResult {
+  /**
+   * In case of CSV file, won't be returned
+   */
+  data?: any[]
+  /**
+   * In case of CSV file, won't be returned
+   */
+  headerShow?: string[]
+  rangeSheetRes?: FoundRangeInfo
+  uploader: FileUploader
+}

client/src/app/models/RangeInfo.ts

@@ -0,0 +1,13 @@
+export default interface FoundRangeInfo {
+  found: boolean
+  sheetName: string
+  rangeStartAddress: string
+  rangeEndAddress: string
+  rangeAddress: string
+  missingHeaders: MissingHeaders[]
+}
+
+export interface MissingHeaders {
+  sheetName: string
+  missingHeaders: string[]
+}

client/src/app/models/SearchDataExcelResult.interface.ts

@@ -0,0 +1,13 @@
+import { MissingHeaders } from './RangeInfo'
+
+export interface SearchDataExcelResult {
+  missing?: MissingHeaders[]
+  found?: {
+    data: any
+    arrayData: any[]
+    sheetName: string
+    headers: string[]
+    startAddress?: string
+    endAddress?: string
+  }
+}

client/src/app/models/UploadFile.ts

@@ -0,0 +1,4 @@
+export interface UploadFileResponse {
+  adapterResponse: any
+  log?: string
+}

client/src/app/models/request-wrapper/RequestWrapperResponse.ts

@@ -0,0 +1,4 @@
+export interface RequestWrapperResponse<responseType = any> {
+  adapterResponse: responseType
+  log?: string
+}

client/src/app/multi-dataset/multi-dataset.component.html

@@ -22,8 +22,8 @@
     />
   </div>
 
-  <ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
+  <ng-container *ngIf="parsedDatasets.length && !submittedCsvDatasets.length">
-    <div class="text-center mb-10">
+    <div *ngIf="!excelsSubmitted" class="text-center mb-10">
       <button (click)="onDiscard()" class="btn btn-danger btn-sm mr-10">
         Discard
       </button>
@@ -41,30 +41,54 @@
       <clr-tree-node *ngFor="let dataset of parsedDatasets">
         <button
           (click)="onParsedDatasetClick(dataset)"
-          class="clr-treenode-link whitespace-nowrap"
+          class="clr-treenode-link whitespace-nowrap d-flex clr-align-items-center"
           [class.active]="dataset.active"
         >
-          <cds-icon
+          <ng-container *ngIf="dataset.submitResult">
-            *ngIf="!(dataset.datasource && dataset.parseResult)"
+            <cds-icon
-            status="danger"
+              *ngIf="dataset.submitResult.error"
-            shape="exclamation-circle"
+              status="danger"
-          ></cds-icon>
+              shape="exclamation-circle"
-          <cds-icon
+            ></cds-icon>
-            *ngIf="dataset.datasource && dataset.parseResult"
+            <cds-icon
-            shape="table"
+              *ngIf="dataset.submitResult.success"
-          ></cds-icon>
+              status="success"
-          {{ dataset.libds }}
+              shape="check-circle"
+            ></cds-icon>
+          </ng-container>
+
+          <ng-container *ngIf="!dataset.submitResult">
+            <ng-container *ngIf="dataset.datasource">
+              <cds-icon
+                *ngIf="!(dataset.datasource.length && dataset.parseResult)"
+                status="danger"
+                shape="exclamation-circle"
+              ></cds-icon>
+              <cds-icon
+                *ngIf="dataset.datasource.length && dataset.parseResult"
+                shape="table"
+              ></cds-icon>
+            </ng-container>
+
+            <ng-container *ngIf="!dataset.datasource">
+              <cds-icon *ngIf="!dataset.parsingTable" shape="table"></cds-icon>
+
+              <clr-spinner *ngIf="dataset.parsingTable" clrSmall></clr-spinner>
+            </ng-container>
+          </ng-container>
+
+          <span class="ml-5"> {{ dataset.libds }} </span>
         </button>
       </clr-tree-node>
     </clr-tree>
   </ng-container>
 
-  <ng-container *ngIf="submittedDatasets.length">
+  <ng-container *ngIf="submittedCsvDatasets.length">
     <p cds-text="caption" class="ml-10 mb-10 mt-10">Submitted tables:</p>
     <clr-tree>
-      <clr-tree-node *ngFor="let dataset of submittedDatasets">
+      <clr-tree-node *ngFor="let dataset of submittedCsvDatasets">
         <button
-          (click)="onSubmittedDatasetClick(dataset)"
+          (click)="onSubmittedCsvDatasetClick(dataset)"
           class="clr-treenode-link whitespace-nowrap"
           [class.active]="dataset.active"
         >
@@ -90,7 +114,7 @@
   </div> -->
 </app-sidebar>
 
-<div class="content-area">
+<div #contentArea class="content-area">
   <div class="card no-borders h-100 d-flex clr-flex-column">
     <div
       class="header-row clr-row justify-content-between clr-justify-content-center w-100 m-0"
@@ -116,7 +140,7 @@
       <ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
         <div class="d-flex clr-justify-content-center mt-15">
           <div class="dataset-input-wrapper">
-            <p cds-text="secondary regular" class="mb-20">
+            <p cds-text="secondary regular" class="mb-5">
               Selected file: <strong>{{ selectedFile.name }}</strong>
               <clr-tooltip>
                 <cds-icon
@@ -129,6 +153,9 @@
                 <clr-tooltip-content> Discard the file </clr-tooltip-content>
               </clr-tooltip>
             </p>
+            <p cds-text="secondary regular" class="mb-20">
+              File size: <strong>{{ selectedFile.sizeMB }} MB</strong>
+            </p>
             <p cds-text="secondary regular" class="mb-15">
               Paste or type the list of datasets to upload:
             </p>
@@ -151,7 +178,7 @@
 
             <div class="dataset-selection-actions text-right mt-10">
               <button
-                (click)="onUploadFile()"
+                (click)="onStartParsingFile()"
                 id="continue-btn"
                 class="btn btn-primary btn-sm"
                 [disabled]="!matchedDatasets.length"
@@ -164,23 +191,100 @@
         </div>
       </ng-container>
 
-      <ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
+      <ng-container
+        *ngIf="parsedDatasets.length && !submittedCsvDatasets.length"
+      >
         <div
           *ngIf="!activeParsedDataset"
           class="no-table-selected pointer-events-none"
         >
-          <clr-icon
+          <ng-container *ngIf="fileLoadingState !== FileLoadingState.parsed">
-            shape="warning-standard"
+            <clr-icon
-            size="40"
+              shape="process-on-vm"
-            class="is-info icon-dc-fill"
+              size="40"
-          ></clr-icon>
+              class="is-info icon-dc-fill"
-          <p class="text-center color-gray mt-10" cds-text="section">
+            ></clr-icon>
-            Please select a dataset on the left to review the data
+
-          </p>
+            <p class="text-center color-gray mt-10" cds-text="section">
+              {{ fileLoadingState }}...
+            </p>
+          </ng-container>
+
+          <ng-container *ngIf="fileLoadingState === FileLoadingState.parsed">
+            <clr-icon
+              shape="warning-standard"
+              size="40"
+              class="is-info icon-dc-fill"
+            ></clr-icon>
+            <p class="text-center color-gray mt-10" cds-text="section">
+              Please select a dataset on the left to review the data
+            </p>
+          </ng-container>
         </div>
 
         <ng-container *ngIf="activeParsedDataset">
-          <div class="d-flex clr-justify-content-between p-10">
+          <div
+            *ngIf="activeParsedDataset.submitResult"
+            class="d-flex clr-justify-content-between p-10 mt-15 submission-results"
+          >
+            <div>
+              <p cds-text="secondary regular" class="mb-10">
+                Submit Status:
+                <span
+                  *ngIf="activeParsedDataset.submitResult?.success"
+                  class="color-green"
+                  ><strong>SUCCESS</strong></span
+                >
+                <span
+                  *ngIf="activeParsedDataset.submitResult?.error"
+                  class="color-red"
+                  ><strong>ERROR</strong></span
+                >
+              </p>
+              <p
+                *ngIf="activeParsedDataset.submitResult?.error"
+                cds-text="secondary regular"
+              >
+                Error details:
+              </p>
+            </div>
+
+            <div>
+              <button
+                *ngIf="
+                  !submittingCsv && activeParsedDataset.submitResult?.error
+                "
+                (click)="reSubmitTable(activeParsedDataset)"
+                class="btn btn-primary mt-10"
+                [clrLoading]="submitLoading"
+              >
+                Resubmit
+              </button>
+              <button
+                (click)="
+                  downloadFile(
+                    activeParsedDataset.submitResult.log ||
+                      activeParsedDataset.submitResult.success ||
+                      activeParsedDataset.submitResult.error
+                  )
+                "
+                class="btn btn-primary-outline mt-10"
+              >
+                Download log
+              </button>
+            </div>
+          </div>
+
+          <div
+            *ngIf="activeParsedDataset.submitResult?.error"
+            class="error-field mt-15"
+          >
+            <div class="log-wrapper">
+              {{ activeParsedDataset.submitResult?.error | json }}
+            </div>
+          </div>
+
+          <div class="d-flex clr-justify-content-between p-10 mt-15">
             <div>
               <p cds-text="secondary regular" class="mb-10">
                 Found in range:
@@ -197,7 +301,17 @@
                 </ng-container>
 
                 <ng-container *ngIf="!activeParsedDataset.parseResult">
-                  <strong>No data found</strong>
+                  <strong *ngIf="!activeParsedDataset.parsingTable"
+                    >No data found</strong
+                  >
+
+                  <span
+                    *ngIf="activeParsedDataset.parsingTable"
+                    class="d-flex clr-align-items-center"
+                  >
+                    <strong>Searching for the data...</strong>
+                    <clr-spinner class="ml-5" clrSmall></clr-spinner>
+                  </span>
                 </ng-container>
               </p>
               <p cds-text="secondary regular">
@@ -262,9 +376,9 @@
         </ng-container>
       </ng-container>
 
-      <ng-container *ngIf="submittedDatasets.length">
+      <ng-container *ngIf="submittedCsvDatasets.length">
         <div
-          *ngIf="!activeSubmittedDataset"
+          *ngIf="!activeSubmittedCsvDataset"
           class="no-table-selected pointer-events-none"
         >
           <clr-icon
@@ -278,31 +392,17 @@
         </div>
       </ng-container>
 
-      <ng-container *ngIf="activeSubmittedDataset">
+      <ng-container *ngIf="activeSubmittedCsvDataset">
         <div class="d-flex clr-justify-content-between p-10">
           <div>
-            <p
-              *ngIf="activeSubmittedDataset.parseResult"
-              cds-text="secondary regular"
-              class="mb-10"
-            >
-              Found in range:
-              <strong
-                >"{{
-                  activeSubmittedDataset.parseResult.rangeSheetRes?.sheetName
-                }}"!{{
-                  activeSubmittedDataset.parseResult.rangeSheetRes?.rangeAddress
-                }}</strong
-              >
-            </p>
             <p cds-text="secondary regular" class="mb-10">
               Matched with dataset:
               <strong>
                 <clr-tooltip>
                   <a
                     clrTooltipTrigger
-                    [routerLink]="'/editor/' + activeSubmittedDataset.libds"
+                    [routerLink]="'/editor/' + activeSubmittedCsvDataset.libds"
-                    >{{ activeSubmittedDataset.libds }}</a
+                    >{{ activeSubmittedCsvDataset.libds }}</a
                   >
                   <clr-tooltip-content
                     [clrPosition]="'top-right'"
@@ -315,15 +415,17 @@
             </p>
             <p cds-text="secondary regular" class="mb-10">
               Status:
-              <span *ngIf="activeSubmittedDataset.success" class="color-green"
+              <span
+                *ngIf="activeSubmittedCsvDataset.success"
+                class="color-green"
                 ><strong>SUCCESS</strong></span
               >
-              <span *ngIf="activeSubmittedDataset.error" class="color-red"
+              <span *ngIf="activeSubmittedCsvDataset.error" class="color-red"
                 ><strong>ERROR</strong></span
               >
             </p>
             <p
-              *ngIf="activeSubmittedDataset.error"
+              *ngIf="activeSubmittedCsvDataset.error"
               cds-text="secondary regular"
             >
               Error details:
@@ -331,18 +433,11 @@
           </div>
 
           <div>
-            <button
-              *ngIf="!submittingCsv && activeSubmittedDataset.error"
-              (click)="reSubmitTable(activeSubmittedDataset)"
-              class="btn btn-primary mt-10"
-              [clrLoading]="submitLoading"
-            >
-              Resubmit
-            </button>
             <button
               (click)="
                 downloadFile(
-                  activeSubmittedDataset.success || activeSubmittedDataset.error
+                  activeSubmittedCsvDataset.success ||
+                    activeSubmittedCsvDataset.error
                 )
               "
               class="btn btn-primary-outline mt-10"
@@ -352,9 +447,9 @@
           </div>
         </div>
 
-        <div *ngIf="activeSubmittedDataset.error" class="error-field mt-15">
+        <div *ngIf="activeSubmittedCsvDataset.error" class="error-field mt-15">
           <div class="log-wrapper">
-            {{ activeSubmittedDataset.error | json }}
+            {{ activeSubmittedCsvDataset.error | json }}
           </div>
         </div>
       </ng-container>

client/src/app/multi-dataset/multi-dataset.component.scss

@@ -47,4 +47,8 @@
 
 .licence-limit-notice {
   color: var(--cds-alias-status-warning-dark);
+}
+
+.submission-results {
+  border-bottom: 1px solid #d3d3d3;
 }

client/src/app/multi-dataset/multi-dataset.component.ts

@@ -1,8 +1,10 @@
 import {
   ChangeDetectorRef,
   Component,
+  ElementRef,
   HostBinding,
-  OnInit
+  OnInit,
+  ViewChild
 } from '@angular/core'
 import {
   EventService,
@@ -11,17 +13,13 @@ import {
   SasService,
   SasStoreService
 } from '../services'
-import * as XLSX from '@sheet/crypto'
 import { globals } from '../_globals'
 import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
 import { DcValidator } from '../shared/dc-validator/dc-validator'
 import { ExcelRule } from '../models/TableData'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { Col } from '../shared/dc-validator/models/col.model'
-import {
+import { SpreadsheetService } from '../services/spreadsheet.service'
-  ParseResult,
-  SpreadsheetService
-} from '../services/spreadsheet.service'
 import Handsontable from 'handsontable'
 import { HotTableRegisterer } from '@handsontable/angular'
 import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
@@ -29,6 +27,20 @@ import { CellChange, ChangeSource } from 'handsontable/common'
 import { baseAfterGetColHeader } from '../shared/utils/hot.utils'
 import { ColumnSettings } from 'handsontable/settings'
 import { UploadFile } from '@sasjs/adapter'
+import { UploadFileResponse } from '../models/UploadFile'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { ParseResult } from '../models/ParseResult.interface'
+import XLSX from 'xlsx'
+
+enum FileLoadingState {
+  reading = 'Reading the file',
+  parsing = 'Searching for the data in the file',
+  parsed = 'Searching for the data finished',
+  /**
+   * Defualt value
+   */
+  notSelected = 'File not selected'
+}
 
 @Component({
   selector: 'app-multi-dataset',
@@ -37,10 +49,16 @@ import { UploadFile } from '@sasjs/adapter'
 })
 export class MultiDatasetComponent implements OnInit {
   @HostBinding('class.content-container') contentContainerClass = true
+  @ViewChild('contentArea', { static: true }) contentAreaRef!: ElementRef
 
   public licenceState = this.licenceService.licenceState
   public Infinity = Infinity
 
+  public workbookInterval: any
+  public fileLoadingState: FileLoadingState = FileLoadingState.notSelected
+
+  public FileLoadingState = FileLoadingState
+
   public hotTableLicenseKey: string | undefined = undefined
   public hotTableMaxRows =
     this.licenceState.value.viewer_rows_allowed || Infinity
@@ -48,15 +66,19 @@ export class MultiDatasetComponent implements OnInit {
   public csvFiles: UploadFile[] = []
   public csvSubmitting: boolean = false
 
-  public selectedFile: File | null = null
+  public autoDetectingColumns: boolean = false
+
+  public selectedFile: SelectedFile | null = null
   public parsedDatasets: ParsedDataset[] = []
-  public submittedDatasets: SubmittedDatasetResult[] = []
+  public submittedCsvDatasets: SubmittedCsvDatasetResult[] = []
 
   public datasetsLoading: boolean = false
   public uploadLoading: boolean = false
   public submitLoading: boolean = false
 
   public matchedDatasets: string[] = []
+  public sheetNames: string[] = []
+
   public userInputDatasets: string = ''
 
   public libsAndTables: {
@@ -157,7 +179,7 @@ export class MultiDatasetComponent implements OnInit {
     }
   }
 
-  onFileChange(event: any) {
+  async onFileChange(event: any) {
     const files = event?.target?.files || []
 
     if (files.length < 1) {
@@ -194,6 +216,20 @@ export class MultiDatasetComponent implements OnInit {
       // For EXCEL if multiple files, we only take one (the first one)
       this.selectedFile = event.target.files[0]
 
+      if (this.selectedFile) {
+        this.fileLoadingState = FileLoadingState.reading
+
+        this.selectedFile.sizeMB = this.spreadsheetService.bytesToMB(
+          this.selectedFile.size
+        )
+
+        // Read the excel file to be ready
+        this.spreadsheetService.xlsxReadFile(this.selectedFile!).then((wb) => {
+          this.fileLoadingState = FileLoadingState.parsing
+          this.selectedFile!.workbook = wb
+        })
+      }
+
       this.initUserInputHot()
       this.onAutoDetectColumns()
     } else if (matchedExtension === 'csv') {
@@ -246,7 +282,7 @@ export class MultiDatasetComponent implements OnInit {
     this.userInputDatasets = ''
   }
 
-  async onUploadFile() {
+  async onStartParsingFile() {
     this.uploadLoading = true
 
     const datasetFetchingPromises: Promise<
@@ -271,58 +307,65 @@ export class MultiDatasetComponent implements OnInit {
 
       const datasetObjects = this.buildDatasetsObjects(datasets)
 
-      for (let datasetObject of datasetObjects) {
+      datasetObjects.forEach((datasetObject) => {
-        this.spreadsheetService
+        this.parsedDatasets.push({
-          .parseExcelFile({
+          libds: datasetObject.libds,
-            file: this.selectedFile!,
+          includeInSubmission: true,
-            dcValidator: datasetObject.dcValidator!,
+          datasetInfo: datasetObject,
-            headerPks: datasetObject.headerPks,
+          parsingTable: true
-            headerArray: datasetObject.headerArray,
+        })
-            headerShow: [],
+      })
-            timeHeaders: datasetObject.timeHeaders,
-            dateHeaders: datasetObject.dateHeaders,
-            dateTimeHeaders: datasetObject.dateTimeHeaders,
-            xlRules: datasetObject.xlRules
-          })
-          .then((parseResult: ParseResult | undefined) => {
-            console.log('parseResult', parseResult)
 
-            if (parseResult && parseResult.data) {
+      this.workbookLoaded().then((workbook) => {
-              let datasource: any[] = []
+        for (let parsedDataset of this.parsedDatasets) {
+          this.spreadsheetService
+            .parseExcelFile({
+              file: this.selectedFile!,
+              workbook: workbook,
+              password: this.selectedFile!.password || undefined,
+              dcValidator: parsedDataset.datasetInfo.dcValidator!,
+              headerPks: parsedDataset.datasetInfo.headerPks,
+              headerArray: parsedDataset.datasetInfo.headerArray,
+              headerShow: [],
+              timeHeaders: parsedDataset.datasetInfo.timeHeaders,
+              dateHeaders: parsedDataset.datasetInfo.dateHeaders,
+              dateTimeHeaders: parsedDataset.datasetInfo.dateTimeHeaders,
+              xlRules: parsedDataset.datasetInfo.xlRules
+            })
+            .then((parseResult: ParseResult | undefined) => {
+              this.fileLoadingState = FileLoadingState.parsed
 
-              parseResult.data.map((item) => {
+              if (parseResult && parseResult.data) {
-                let itemObject: any = {}
+                let datasource: any[] = []
 
-                parseResult.headerShow!.map((header: any, index: number) => {
+                parseResult.data.map((item) => {
-                  itemObject[header] = item[index]
+                  let itemObject: any = {}
+
+                  parseResult.headerShow!.map((header: any, index: number) => {
+                    itemObject[header] = item[index]
+                  })
+
+                  // If Delete? column is not set in the file, we set it to NO
+                  if (!itemObject['_____DELETE__THIS__RECORD_____'])
+                    itemObject['_____DELETE__THIS__RECORD_____'] = 'No'
+
+                  datasource.push(itemObject)
                 })
 
-                // If Delete? column is not set in the file, we set it to NO
+                parsedDataset.datasource = datasource
-                if (!itemObject['_____DELETE__THIS__RECORD_____'])
+                parsedDataset.parseResult = parseResult
-                  itemObject['_____DELETE__THIS__RECORD_____'] = 'No'
+                parsedDataset.parsingTable = false
-
+              }
-                datasource.push(itemObject)
-              })
-
-              this.parsedDatasets.push({
-                libds: datasetObject.libds,
-                parseResult: parseResult,
-                includeInSubmission: true,
-                datasetInfo: datasetObject,
-                datasource: datasource
-              })
-            }
-          })
-          .catch((error: string) => {
-            console.warn('Parsing excel file error.', error)
-
-            this.parsedDatasets.push({
-              libds: datasetObject.libds,
-              includeInSubmission: false,
-              datasetInfo: datasetObject
             })
-          })
+            .catch((error: string) => {
-      }
+              console.warn('Parsing excel file error.', error)
+
+              parsedDataset.datasource = []
+              parsedDataset.includeInSubmission = false
+              parsedDataset.parsingTable = false
+            })
+        }
+      })
     })
   }
 
@@ -349,13 +392,17 @@ export class MultiDatasetComponent implements OnInit {
     setTimeout(() => {
       this.hotInstance = this.hotRegisterer.getInstance('hotInstance')
 
+      // Set height of parsed data to full height of the page content area
+      const contentAreaHeight = this.contentAreaRef.nativeElement.clientHeight
+      const hotHeight = `${contentAreaHeight - 160}px`
+
       if (this.activeParsedDataset) {
         this.hotInstance.updateSettings({
           data: this.activeParsedDataset.datasource || [],
           colHeaders: this.activeParsedDataset.datasetInfo.headerColumns,
           columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(),
           readOnly: true,
-          height: '300px',
+          height: hotHeight || '300px',
           className: 'htDark'
         })
       }
@@ -507,13 +554,32 @@ export class MultiDatasetComponent implements OnInit {
    * convention. {@link isValidDatasetFormat}
    */
   async onAutoDetectColumns() {
-    const sheetNames = await this.parseExcelSheetNames()
+    let passwordError = false
 
-    if (sheetNames) {
+    await this.parseExcelSheetNames()
+      .then((sheetNames) => {
+        this.sheetNames = sheetNames
+      })
+      .catch((err) => {
+        if (err.includes('password')) {
+          passwordError = true
+        }
+      })
+
+    if (passwordError) {
+      this.onDiscardFile()
+      this.eventService.showInfoModal(
+        'Locked file',
+        'We failed to unlock the file.'
+      )
+      return
+    }
+
+    if (this.sheetNames) {
       this.matchedDatasets = []
       this.userInputDatasets = ''
 
-      sheetNames.forEach((sheetName: string, index: number) => {
+      this.sheetNames.forEach((sheetName: string, index: number) => {
         const trimmedSheetname = sheetName.trim()
 
         if (
@@ -561,8 +627,8 @@ export class MultiDatasetComponent implements OnInit {
     this.initHot()
   }
 
-  onSubmittedDatasetClick(submittedDataset: SubmittedDatasetResult) {
+  onSubmittedCsvDatasetClick(submittedDataset: SubmittedCsvDatasetResult) {
-    this.deselectAllSubmittedDatasets()
+    this.deselectAllSubmittedCsvDatasets()
 
     submittedDataset.active = true
   }
@@ -571,8 +637,10 @@ export class MultiDatasetComponent implements OnInit {
     return this.parsedDatasets.find((dataset) => dataset.active)
   }
 
-  public get activeSubmittedDataset(): SubmittedDatasetResult | undefined {
+  public get activeSubmittedCsvDataset():
-    return this.submittedDatasets.find((dataset) => dataset.active)
+    | SubmittedCsvDatasetResult
+    | undefined {
+    return this.submittedCsvDatasets.find((dataset) => dataset.active)
   }
 
   public get notFoundDatasets(): string[] {
@@ -606,8 +674,12 @@ export class MultiDatasetComponent implements OnInit {
     return this.csvFiles.length > 0
   }
 
+  public get excelsSubmitted(): boolean {
+    return !!this.parsedDatasets.filter((ds) => ds.submitResult).length
+  }
+
   public downloadFile(response: any) {
-    const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log`
+    const filename = `stagedata-${this.activeSubmittedCsvDataset?.libds}-log`
     this.helperService.downloadTextFile(filename, JSON.stringify(response))
   }
 
@@ -615,7 +687,7 @@ export class MultiDatasetComponent implements OnInit {
    * Submits attached CSVs which are matched with existing datasets
    */
   async submitCsvFiles() {
-    let requestsResults: SubmittedDatasetResult[] = []
+    let requestsResults: SubmittedCsvDatasetResult[] = []
 
     for (let file of this.csvFiles) {
       const libds = this.parseDatasetFromCsvName(file.fileName)
@@ -626,17 +698,17 @@ export class MultiDatasetComponent implements OnInit {
       await this.sasService
         .uploadFile('services/editors/loadfile', [file], { table: libds })
         .then(
-          (res: any) => {
+          (res: UploadFileResponse) => {
-            if (typeof res.sasjsAbort !== 'undefined') {
+            if (typeof res.adapterResponse.sasjsAbort !== 'undefined') {
-              error = res.sasjsAbort
+              error = res.adapterResponse.sasjsAbort
             } else {
-              success = res
+              success = res.adapterResponse
             }
           },
           (err: any) => {
             console.error('err', err)
 
-            error = err
+            error = err.adapterResponse
           }
         )
 
@@ -647,7 +719,7 @@ export class MultiDatasetComponent implements OnInit {
       })
     }
 
-    this.submittedDatasets = requestsResults
+    this.submittedCsvDatasets = requestsResults
   }
 
   /**
@@ -678,7 +750,7 @@ export class MultiDatasetComponent implements OnInit {
   }
 
   /**
-   * Sends tables to the SAS sequentially
+   * Sends tables found in an excel to the SAS sequentially
    *
    * @param explicitDatasets if empty all datasets will be sent, otherwise only datasets
    * in the array will be sent. eg. ['lib1.table_1', 'lib1.table_2']
@@ -688,10 +760,6 @@ export class MultiDatasetComponent implements OnInit {
 
     this.submitLoading = true
 
-    let requestsResults: SubmittedDatasetResult[] = explicitDatasets
-      ? this.submittedDatasets
-      : []
-
     for (let table of this.parsedDatasets) {
       // Skip the table if no data inside
       if (!table.parseResult || !table.datasource) continue
@@ -733,6 +801,7 @@ export class MultiDatasetComponent implements OnInit {
 
         let error
         let success
+        let log
 
         await this.sasStoreService
           .updateTable(
@@ -741,59 +810,67 @@ export class MultiDatasetComponent implements OnInit {
             'SASControlTable',
             'editors/stagedata',
             table.datasetInfo.data.$sasdata,
-            true
+            true,
+            {
+              debug: true
+            }
           )
-          .then((res: EditorsStageDataSASResponse) => {
+          .then((res: RequestWrapperResponse<EditorsStageDataSASResponse>) => {
-            success = res
+            success = res.adapterResponse
+            log = res.log
           })
           .catch((err: any) => {
             console.error('err', err)
 
-            error = err
+            error = err.adapterResponse
+            log = err.log
           })
 
         const requestResult = {
           success,
           error,
+          log,
           parseResult: table.parseResult,
           libds: table.libds
         }
 
-        // If explicit datasets are set don't just push to th array
+        table.submitResult = requestResult
-        // instead replace if result already exist from before (this might be re-submit)
-        if (explicitDatasets) {
-          const existingResultIndex = requestsResults.findIndex(
-            (result) => result.libds === table.libds
-          )
-
-          if (existingResultIndex > -1) {
-            requestsResults[existingResultIndex] = requestResult
-          } else {
-            requestsResults.push(requestResult)
-          }
-        } else {
-          requestsResults.push(requestResult)
-        }
       }
     }
-
-    this.submittedDatasets = requestsResults
     this.showSubmitReasonModal = false
     this.submitLoading = false
     this.deselectAllParsedDatasets()
   }
 
-  async reSubmitTable(activeSubmittedDataset: SubmittedDatasetResult) {
+  async reSubmitTable(activeSubmittedDataset: ParsedDataset) {
     // Submit only particular table
     await this.submitTables([activeSubmittedDataset.libds])
 
     // Activate new resubmitted table
-    const newSubmittedDataset = this.submittedDatasets.find(
+    const newSubmittedDataset = this.parsedDatasets.find(
       (sd) => sd.libds === activeSubmittedDataset.libds
     )
+
     if (newSubmittedDataset) newSubmittedDataset.active = true
   }
 
+  /**
+   *
+   * @returns Promise once workbook is loaded because use XLSX.read in the background
+   */
+  private workbookLoaded(): Promise<XLSX.WorkBook> {
+    return new Promise((resolve, reject) => {
+      if (!this.selectedFile) reject('No file selected')
+
+      this.workbookInterval = setInterval(() => {
+        if (this.selectedFile!.workbook) {
+          clearInterval(this.workbookInterval)
+          resolve(this.selectedFile!.workbook)
+        }
+      }, 500)
+    })
+  }
+
   private parseDatasetFromCsvName(fileName: string) {
     const fileNameArr = fileName.split('.')
     fileNameArr.pop()
@@ -815,40 +892,26 @@ export class MultiDatasetComponent implements OnInit {
       .map((row) => (row ? `${row[0]}.${row[1]}` : ''))
   }
 
-  private parseExcelSheetNames(): Promise<string[]> {
+  /**
+   * Read the file minimally just to get the sheet names, not reading full file
+   * to help boost the performance
+   *
+   * @returns sheet names in string array
+   */
+  private async parseExcelSheetNames(): Promise<string[]> {
     return new Promise((resolve, reject) => {
-      const reader = new FileReader()
+      if (!this.selectedFile) return resolve([])
 
-      if (!this.selectedFile) {
+      this.spreadsheetService
-        console.warn('selectedFile is missing')
+        .parseExcelSheetNames(this.selectedFile)
-        return resolve([])
+        .then((parsed) => {
-      }
+          if (parsed.password) this.selectedFile!.password = parsed.password
 
-      reader.onload = (event: ProgressEvent<FileReader>) => {
+          return resolve(parsed.sheetNames)
-        if (!event?.target) {
+        })
-          console.warn('File reader event.target is missing')
+        .catch((err) => {
-          return
+          return reject(err)
-        }
-
-        const data = event.target.result
-        const workbook = XLSX.read(data, {
-          type: 'binary'
         })
-
-        try {
-          const sheet_names_list = workbook.SheetNames
-
-          return resolve(sheet_names_list)
-        } catch (e) {
-          console.error(e)
-        }
-      }
-
-      reader.onerror = function (ex) {
-        console.log(ex)
-      }
-
-      reader.readAsBinaryString(this.selectedFile)
     })
   }
 
@@ -977,8 +1040,8 @@ export class MultiDatasetComponent implements OnInit {
     }
   }
 
-  private deselectAllSubmittedDatasets() {
+  private deselectAllSubmittedCsvDatasets() {
-    for (let submittedDataset of this.submittedDatasets) {
+    for (let submittedDataset of this.submittedCsvDatasets) {
       submittedDataset.active = false
     }
   }
@@ -1002,16 +1065,29 @@ export interface ParsedDataset {
   libds: string
   parseResult?: ParseResult
   datasetInfo: DatasetsObject
+  submitResult?: SubmittedDatasetResult
   datasource?: any[]
   includeInSubmission: boolean
   status?: 'success' | 'error'
   active?: boolean
+  parsingTable?: boolean
 }
 
 export interface SubmittedDatasetResult {
+  success: EditorsStageDataSASResponse | undefined
+  error: any
+  log?: string
+}
+
+export interface SubmittedCsvDatasetResult {
   libds: string
   success: EditorsStageDataSASResponse | undefined
   error: any
-  parseResult?: ParseResult
   active?: boolean
 }
+
+export interface SelectedFile extends File {
+  sizeMB?: number
+  password?: string
+  workbook?: XLSX.WorkBook
+}

client/src/app/role/role.component.ts

@@ -4,6 +4,7 @@ import { HelperService } from '../services/helper.service'
 import { Location } from '@angular/common'
 import { Router, ActivatedRoute } from '@angular/router'
 import { SasService } from '../services/sas.service'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-role',
@@ -47,61 +48,69 @@ export class RoleComponent implements OnInit {
     } else {
       if (globals.usernav.roleList === undefined) {
         this.loading = true
-        this.sasService.request('usernav/userroles', null).then((res: any) => {
+        this.sasService
-          this.loading = false
+          .request('usernav/userroles', null)
-          this.roles = res.roles
+          .then((res: RequestWrapperResponse) => {
-          globals.usernav.roleList = res.roles
+            this.loading = false
-          if (this.paramPresent) {
+            this.roles = res.adapterResponse.roles
-            if (this.roles !== undefined) {
+            globals.usernav.roleList = res.adapterResponse.roles
-              let validRole = this.findRole(this.roles, this.paramURI)
+            if (this.paramPresent) {
-              if (validRole !== false) {
+              if (this.roles !== undefined) {
-                this.loading = true
+                let validRole = this.findRole(this.roles, this.paramURI)
-                let data = { iwant: [{ roleid: this.paramURI }] }
+                if (validRole !== false) {
-                this.sasService
+                  this.loading = true
-                  .request('usernav/usermembersbyrole', data)
+                  let data = { iwant: [{ roleid: this.paramURI }] }
-                  .then((res: any) => {
+                  this.sasService
-                    this.loading = false
+                    .request('usernav/usermembersbyrole', data)
-                    this.roleMembers = res.sasmembers
+                    .then((res: RequestWrapperResponse) => {
-                    this.roleMembersCount = res.sasmembers.length
+                      this.loading = false
-                    this.roleGroups = res.sasgroups
+                      this.roleMembers = res.adapterResponse.sasmembers
-                    this.roleGroupsCount = res.sasgroups.length
+                      this.roleMembersCount =
-                    this.roleUri = validRole.ROLEURI
+                        res.adapterResponse.sasmembers.length
-                    this.roleName = validRole.ROLENAME
+                      this.roleGroups = res.adapterResponse.sasgroups
-                    this.roleDesc = validRole.ROLEDESC
+                      this.roleGroupsCount =
-                  })
+                        res.adapterResponse.sasgroups.length
+                      this.roleUri = validRole.ROLEURI
+                      this.roleName = validRole.ROLENAME
+                      this.roleDesc = validRole.ROLEDESC
+                    })
+                }
               }
             }
-          }
+          })
-        })
       } else {
         this.roles = globals.usernav.roleList
         this.roleSearch = globals.usernav.roleSearch
-        this.sasService.request('usernav/userroles', null).then((res: any) => {
+        this.sasService
-          this.roles = res.roles
+          .request('usernav/userroles', null)
-          globals.usernav.roleList = res.roles
+          .then((res: RequestWrapperResponse) => {
+            this.roles = res.adapterResponse.roles
+            globals.usernav.roleList = res.adapterResponse.roles
 
-          if (this.paramPresent) {
+            if (this.paramPresent) {
-            if (this.roles !== undefined) {
+              if (this.roles !== undefined) {
-              let validRole = this.findRole(this.roles, this.paramURI)
+                let validRole = this.findRole(this.roles, this.paramURI)
-              if (validRole !== false) {
+                if (validRole !== false) {
-                this.loading = true
+                  this.loading = true
-                let data = { iwant: [{ roleid: this.paramURI }] }
+                  let data = { iwant: [{ roleid: this.paramURI }] }
-                this.sasService
+                  this.sasService
-                  .request('usernav/usermembersbyrole', data)
+                    .request('usernav/usermembersbyrole', data)
-                  .then((res: any) => {
+                    .then((res: RequestWrapperResponse) => {
-                    this.loading = false
+                      this.loading = false
-                    this.roleMembers = res.sasmembers
+                      this.roleMembers = res.adapterResponse.sasmembers
-                    this.roleMembersCount = res.sasmembers.length
+                      this.roleMembersCount =
-                    this.roleGroups = res.sasgroups
+                        res.adapterResponse.sasmembers.length
-                    this.roleGroupsCount = res.sasgroups.length
+                      this.roleGroups = res.adapterResponse.sasgroups
-                    this.roleUri = validRole.ROLEURI
+                      this.roleGroupsCount =
-                    this.roleName = validRole.ROLENAME
+                        res.adapterResponse.sasgroups.length
-                    this.roleDesc = validRole.ROLEDESC
+                      this.roleUri = validRole.ROLEURI
-                  })
+                      this.roleName = validRole.ROLENAME
+                      this.roleDesc = validRole.ROLEDESC
+                    })
+                }
               }
             }
-          }
+          })
-        })
       }
     }
   }
@@ -125,12 +134,12 @@ export class RoleComponent implements OnInit {
     let data = { iwant: [{ roleid: role.ROLEURI }] }
     this.sasService
       .request('usernav/usermembersbyrole', data)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
         this.loading = false
-        this.roleMembers = res.sasmembers
+        this.roleMembers = res.adapterResponse.sasmembers
-        this.roleMembersCount = res.sasmembers.length
+        this.roleMembersCount = res.adapterResponse.sasmembers.length
-        this.roleGroups = res.sasgroups
+        this.roleGroups = res.adapterResponse.sasgroups
-        this.roleGroupsCount = res.sasgroups.length
+        this.roleGroupsCount = res.adapterResponse.sasgroups.length
         this.roleUri = role.ROLEURI
         this.roleName = role.ROLENAME
         this.roleDesc = role.ROLEDESC

client/src/app/services/app.service.ts

@@ -9,11 +9,13 @@ import { EnvironmentInfo } from '../system/models/environment-info.model'
 import { LicenceService } from './licence.service'
 import { AppSettingsService } from './app-settings.service'
 import { AppThemes } from '../models/AppSettings'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { AppStoreService } from './app-store.service'
 
 @Injectable()
 export class AppService {
   public syssite = new BehaviorSubject<string[] | null>(null)
-  private environmentInfo: EnvironmentInfo | null = null
+  private environmentInfo: EnvironmentInfo = {}
 
   constructor(
     private licenceService: LicenceService,
@@ -21,7 +23,8 @@ export class AppService {
     private sasService: SasService,
     private loggerService: LoggerService,
     private appSettingsService: AppSettingsService,
-    private router: Router
+    private router: Router,
+    private appStoreService: AppStoreService
   ) {
     this.subscribe()
 
@@ -81,16 +84,19 @@ export class AppService {
 
     await this.sasService
       .request('public/startupservice', null)
-      .then(async (res: any) => {
+      .then(async (res: RequestWrapperResponse) => {
-        this.syssite.next([res.SYSSITE])
+        this.syssite.next([res.adapterResponse.SYSSITE])
 
         let missingProps: string[] = []
 
-        if (!res.globvars || (res.globvars && !res.globvars[0]))
+        if (
+          !res.adapterResponse.globvars ||
+          (res.adapterResponse.globvars && !res.adapterResponse.globvars[0])
+        )
           missingProps.push('Globvars')
-        if (!res.sasdatasets) missingProps.push('Sasdatasets')
+        if (!res.adapterResponse.sasdatasets) missingProps.push('Sasdatasets')
-        if (!res.saslibs) missingProps.push('Saslibs')
+        if (!res.adapterResponse.saslibs) missingProps.push('Saslibs')
-        if (!res.xlmaps) missingProps.push('XLMaps')
+        if (!res.adapterResponse.xlmaps) missingProps.push('XLMaps')
 
         if (missingProps.length > 0) {
           startupServiceError = true
@@ -103,23 +109,26 @@ export class AppService {
           return
         }
 
+        const dcAdapterSettings = this.appStoreService.getDcAdapterSettings()
+
         this.environmentInfo = {
-          SYSSITE: res.SYSSITE,
+          SYSSITE: res.adapterResponse.SYSSITE,
-          SYSSCPL: res.SYSSCPL,
+          SYSSCPL: res.adapterResponse.SYSSCPL,
-          SYSTCPIPHOSTNAME: res.SYSTCPIPHOSTNAME,
+          SYSTCPIPHOSTNAME: res.adapterResponse.SYSTCPIPHOSTNAME,
-          SYSVLONG: res.SYSVLONG,
+          SYSVLONG: res.adapterResponse.SYSVLONG,
-          MEMSIZE: res.MEMSIZE,
+          MEMSIZE: res.adapterResponse.MEMSIZE,
-          SYSPROCESSMODE: res.SYSPROCESSMODE,
+          SYSPROCESSMODE: res.adapterResponse.SYSPROCESSMODE,
-          SYSHOSTNAME: res.SYSHOSTNAME,
+          SYSHOSTNAME: res.adapterResponse.SYSHOSTNAME,
-          SYSUSERID: res.SYSUSERID,
+          SYSUSERID: res.adapterResponse.SYSUSERID,
-          SYSHOSTINFOLONG: res.SYSHOSTINFOLONG,
+          SYSHOSTINFOLONG: res.adapterResponse.SYSHOSTINFOLONG,
-          SYSENCODING: res.SYSENCODING,
+          SYSENCODING: res.adapterResponse.SYSENCODING,
-          AUTOEXEC: res.AUTOEXEC,
+          AUTOEXEC: res.adapterResponse.AUTOEXEC,
-          ISADMIN: res.globvars[0].ISADMIN,
+          ISADMIN: res.adapterResponse.globvars[0].ISADMIN,
-          DC_ADMIN_GROUP: res.globvars[0].DC_ADMIN_GROUP
+          DC_ADMIN_GROUP: res.adapterResponse.globvars[0].DC_ADMIN_GROUP,
+          APP_LOC: dcAdapterSettings?.appLoc
         }
 
-        let libs = res.sasdatasets
+        let libs = res.adapterResponse.sasdatasets
         let libGroup: any = {}
         let libsAndTables
         let libraries
@@ -153,7 +162,7 @@ export class AppService {
           globals.editor.libsAndTables = libsAndTables
         }
 
-        globals.xlmaps = res.xlmaps.map((xlmap: any) => ({
+        globals.xlmaps = res.adapterResponse.xlmaps.map((xlmap: any) => ({
           id: xlmap[0],
           description: xlmap[1],
           targetDS: xlmap[2]
@@ -162,9 +171,9 @@ export class AppService {
         globals.editor.libraries = libraries
         globals.editor.startupSet = true
 
-        globals.dcLib = res.globvars[0].DCLIB
+        globals.dcLib = res.adapterResponse.globvars[0].DCLIB
 
-        await this.licenceService.activation(res)
+        await this.licenceService.activation(res.adapterResponse)
       })
       .catch((err: any) => {
         startupServiceError = true

client/src/app/services/sas-store.service.ts

@@ -18,6 +18,7 @@ import { isSpecialMissing } from '@sasjs/utils/input/validators'
 import { Col } from '../shared/dc-validator/models/col.model'
 import { get } from 'lodash-es'
 import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Injectable()
 export class SasStoreService {
@@ -58,12 +59,10 @@ export class SasStoreService {
   ) {
     const tables: any = {}
     tables[tableName] = [tableData]
-    const res: EditorsGetDataSASResponse = await this.sasService.request(
+    const res: RequestWrapperResponse<EditorsGetDataSASResponse> =
-      program,
+      await this.sasService.request(program, tables)
-      tables
-    )
     const response: EditorsGetDataServiceResponse = {
-      data: res,
+      data: res.adapterResponse,
       libds: libds
     }
     return response
@@ -84,8 +83,9 @@ export class SasStoreService {
     tableName: string,
     program: string,
     $dataFormats: $DataFormats | null,
-    suppressErrorSuccessMessages?: boolean
+    suppressErrorSuccessMessages?: boolean,
-  ): Promise<EditorsStageDataSASResponse> {
+    adapterConfig?: any
+  ): Promise<RequestWrapperResponse<EditorsStageDataSASResponse>> {
     // add sp as third argument of createData call
 
     let tables: any = {
@@ -103,7 +103,7 @@ export class SasStoreService {
     let res = await this.sasService.request<EditorsStageDataSASResponse>(
       program,
       tables,
-      null,
+      adapterConfig,
       {
         suppressErrorAbortModal: suppressErrorSuccessMessages,
         suppressSuccessAbortModal: suppressErrorSuccessMessages
@@ -127,8 +127,7 @@ export class SasStoreService {
   ) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables)
+    return (await this.sasService.request(program, tables)).adapterResponse
-    return res
   }
 
   /**
@@ -152,8 +151,8 @@ export class SasStoreService {
    * @returns All submits
    */
   public async getSubmitts() {
-    let res: any = await this.sasService.request('editors/getsubmits', null)
+    return (await this.sasService.request('editors/getsubmits', null))
-    return res
+      .adapterResponse
   }
 
   /**
@@ -161,7 +160,8 @@ export class SasStoreService {
    * @returns All libraries
    */
   public async viewLibs() {
-    return this.sasService.request('public/viewlibs', null)
+    return (await this.sasService.request('public/viewlibs', null))
+      .adapterResponse
   }
 
   public async refreshLibInfo(libref: string) {
@@ -169,30 +169,22 @@ export class SasStoreService {
       lib2refresh: [{ libref }]
     }
 
-    return this.sasService.request('public/refreshlibinfo', data)
+    return (await this.sasService.request('public/refreshlibinfo', data))
-  }
+      .adapterResponse
-
-  public async versionHistory(libDataset: any) {
-    const data = { iwant: [{ LIBDS: libDataset }] }
-    let res: any = await this.sasService.request(
-      'public/getversionhistory',
-      data
-    )
-    return res
   }
 
   public async viewTables(lib: any) {
     let tables = { SASControlTable: [{ MPLIB: lib }] }
-    let res: any = await this.sasService.request('public/viewtables', tables)
+    return (await this.sasService.request('public/viewtables', tables))
-    return res
+      .adapterResponse
   }
 
   public async viewData(libDataset: any, filter_pk: any) {
     let tables = {
       SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }]
     }
-    let res: any = await this.sasService.request('public/viewdata', tables)
+    return (await this.sasService.request('public/viewdata', tables))
-    return res
+      .adapterResponse
   }
 
   public async viewDataSearch(
@@ -213,41 +205,36 @@ export class SasStoreService {
         }
       ]
     }
-    let res: any = await this.sasService.request('public/viewdata', tables)
+
-    return res
+    return (await this.sasService.request('public/viewdata', tables))
+      .adapterResponse
   }
 
   public async getXLMapRules(id: string) {
     const tables = {
       getxlmaps_in: [{ XLMAP_ID: id }]
     }
-    const res: any = await this.sasService.request('editors/getxlmaps', tables)
+    return (await this.sasService.request('editors/getxlmaps', tables))
-    return res
+      .adapterResponse
-  }
-
-  public async getDetails(tableData: any, tableName: string, program: string) {
-    let tables: any = {}
-    tables[tableName] = [tableData]
-
-    let res: any = await this.sasService.request(program, tables)
-    return res
   }
 
   public async showDiffs(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables, {
+    return (
-      useComputeApi: false
+      await this.sasService.request(program, tables, {
-    })
+        useComputeApi: false
-    return res
+      })
+    ).adapterResponse
   }
   public async rejecting(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables, {
+    return (
-      useComputeApi: false
+      await this.sasService.request(program, tables, {
-    })
+        useComputeApi: false
-    return res
+      })
+    ).adapterResponse
   }
 
   public async approveTable(
@@ -258,15 +245,13 @@ export class SasStoreService {
     let tables: any = {}
     tables[tableName] = [tableData]
 
-    let res: any = await this.sasService.request(program, tables)
+    return (await this.sasService.request(program, tables)).adapterResponse
-    return res
   }
 
   public async getHistory(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables)
+    return (await this.sasService.request(program, tables)).adapterResponse
-    return res
   }
 
   setQueryVariables(dataset: string, cols: any) {
@@ -278,8 +263,8 @@ export class SasStoreService {
   public async getChangeInfo(tableId: any) {
     let obj = { TABLE: tableId }
     let table = { SASControlTable: [obj] }
-    let res: any = await this.sasService.request('public/getchangeinfo', table)
+    return (await this.sasService.request('public/getchangeinfo', table))
-    return res
+      .adapterResponse
   }
 
   public async getQueryValues(
@@ -304,13 +289,13 @@ export class SasStoreService {
       tables.FILTERQUERY = filterQuery
     }
 
-    let res: any = await this.sasService
+    return (
-      .request('public/getcolvals', tables)
+      await this.sasService
-      .catch((er: any) => {
+        .request('public/getcolvals', tables)
-        throw er
+        .catch((er: any) => {
-      })
+          throw er
-
+        })
-    return res
+    ).adapterResponse
   }
 
   public async saveQuery(libds: string, filterQuery: QueryClause[]) {
@@ -319,21 +304,17 @@ export class SasStoreService {
       filterquery: filterQuery
     }
 
-    let res: any = await this.sasService.request(
+    const res = await this.sasService.request('public/validatefilter', tables)
-      'public/validatefilter',
+
-      tables
-    )
     this.filter.next(res)
-    return res
+
+    return res.adapterResponse
   }
 
   public async openTable(tableId: string) {
     let tables = { iwant: [{ table_id: tableId }] }
-    let res: any = await this.sasService.request(
+    return (await this.sasService.request('auditors/getstagetable', tables))
-      'auditors/getstagetable',
+      .adapterResponse
-      tables
-    )
-    return res
   }
 
   public checkOperator(operator: any, value: any, type: string) {

client/src/app/services/sas.service.ts

@@ -12,8 +12,10 @@ import { ServerType } from '@sasjs/utils/types/serverType'
 import { DcAdapterSettings } from '../models/DcAdapterSettings'
 import { AppStoreService } from './app-store.service'
 import { LoggerService } from './logger.service'
-import { RequestWrapperOptions } from '../models/RequestWrapperOptions'
+import { RequestWrapperOptions } from '../models/request-wrapper/RequestWrapperOptions'
 import { ErrorBody } from '../models/ErrorBody'
+import { UploadFileResponse } from '../models/UploadFile'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Injectable({
   providedIn: 'root'
@@ -95,14 +97,16 @@ export class SasService {
    * @param config additional parameters to force eg. { debug: false }
    * @param wrapperOptions used to provide options to the request wrapper function
    * for example to suppress error or success abort modals after request is finished
-   * @returns
+   * @returns adapter response or an error. It will return the `log` as well.
+   * The log could be potentially be wrong if multiple requests happen because the log this
+   * function return is the last request in the Adapter Array for the given URL.
    */
   public request<responseType = any>(
     url: string,
     data: any,
     config?: any,
     wrapperOptions?: RequestWrapperOptions
-  ): Promise<responseType> {
+  ): Promise<RequestWrapperResponse<responseType>> {
     url = 'services/' + url
 
     if (!wrapperOptions) wrapperOptions = {}
@@ -117,9 +121,16 @@ export class SasService {
         })
         .then(
           (res: any) => {
+            const sasRequest = this.sasjsAdapter
+              .getSasRequests()
+              .find((rq) => rq.serviceLink === url)
+
             if (res.login === false) {
               this.shouldLogin.next(true)
-              reject(false)
+              reject({
+                adapterResponse: false,
+                log: sasRequest?.logFile
+              })
             }
 
             if (!this.userService.user && res.MF_GETUSER) {
@@ -141,7 +152,12 @@ export class SasService {
             }
 
             if (res.status === 404) {
-              reject({ MESSAGE: res.body || 'SAS Responded with error' })
+              reject({
+                adapterResponse: {
+                  MESSAGE: res.body || 'SAS Responded with error'
+                },
+                log: sasRequest?.logFile
+              })
             }
 
             if (typeof res.sasjsAbort !== 'undefined') {
@@ -157,7 +173,12 @@ export class SasService {
                 this.eventService.startupDataLoaded()
                 this.router.navigateByUrl('/deploy')
 
-                reject({ error: abortMsg })
+                reject({
+                  adapterResponse: {
+                    error: abortMsg
+                  },
+                  log: sasRequest?.logFile
+                })
 
                 return
               }
@@ -174,14 +195,26 @@ export class SasService {
                 )
               }
 
-              reject({ error: abortMsg })
+              reject({
+                adapterResponse: {
+                  error: abortMsg
+                },
+                log: sasRequest?.logFile
+              })
             }
 
-            resolve(res)
+            resolve({
+              adapterResponse: res,
+              log: sasRequest?.logFile
+            })
           },
           (err: { error: ErrorBody | undefined }) => {
             console.error(err)
 
+            const sasRequest = this.sasjsAdapter
+              .getSasRequests()
+              .find((rq) => rq.serviceLink === url)
+
             if (err.error) {
               let errorMessage: string | undefined = err.error.message
               let log: string | undefined
@@ -205,10 +238,18 @@ export class SasService {
                   'Request error'
                 )
               }
-              reject({ error: errorMessage })
+              reject({
+                adapterResponse: {
+                  error: errorMessage
+                },
+                log: sasRequest?.logFile
+              })
             }
 
-            reject(err)
+            reject({
+              adapterResponse: err,
+              log: sasRequest?.logFile
+            })
           }
         )
     })
@@ -222,8 +263,35 @@ export class SasService {
    * @param params Aditional parameters eg. { debug: false }
    * @returns HTTP Response
    */
-  public uploadFile(sasService: string, files: UploadFile[], params: any) {
+  public uploadFile(
-    return this.sasjsAdapter.uploadFile(sasService, files, params)
+    sasService: string,
+    files: UploadFile[],
+    params: any
+  ): Promise<UploadFileResponse> {
+    return new Promise((resolve, reject) => {
+      this.sasjsAdapter.uploadFile(sasService, files, params).then(
+        (res) => {
+          const sasRequest = this.sasjsAdapter
+            .getSasRequests()
+            .find((rq) => rq.serviceLink === 'services/editors/loadfile')
+
+          resolve({
+            adapterResponse: res,
+            log: sasRequest?.logFile
+          })
+        },
+        (err) => {
+          const sasRequest = this.sasjsAdapter
+            .getSasRequests()
+            .find((rq) => rq.serviceLink === 'services/editors/loadfile')
+
+          reject({
+            response: err,
+            log: sasRequest?.logFile
+          })
+        }
+      )
+    })
   }
 
   public async login(username: string, password: string) {

client/src/app/shared/dc-validator/tests/getHotDataSchema.spec.ts

@@ -2,7 +2,7 @@ import { getHotDataSchema } from '../utils/getHotDataSchema'
 
 describe('DC Validator - hot data schema', () => {
   it('should return correct value for col type', () => {
-    expect(getHotDataSchema('numeric')).toEqual(0)
+    expect(getHotDataSchema('numeric')).toEqual('')
     expect(
       getHotDataSchema('autocomplete', { data: '', source: [1, 2, 3] })
     ).toEqual(1)

client/src/app/shared/dc-validator/utils/getHotDataSchema.ts

@@ -1,7 +1,7 @@
 import { DcValidation } from '../models/dc-validation.model'
 
 const schemaTypeMap: { [key: string]: any } = {
-  numeric: 0,
+  numeric: '',
   default: ''
 }
 

client/src/app/shared/excel-password-modal/excel-password-modal.component.html

@@ -1,4 +1,4 @@
-<ng-container *ngIf="options$ | async as options">
+<div *ngIf="options$ | async as options" class="excel-password-root">
   <clr-modal
     [clrModalOpen]="options.open"
     [clrModalSize]="'md'"
@@ -11,6 +11,7 @@
       <p class="m-0">Please enter password:</p>
       <input
         #filePasswordInput
+        [(ngModel)]="passwordInput"
         data-lpignore="true"
         autocomplete="off"
         id="filePasswordInput"
@@ -19,20 +20,25 @@
       />
     </div>
     <div class="modal-footer">
-      <p *ngIf="options.error" class="m-0 color-red">
+      <div>
-        Sorry that didn't work, try again.
+        <p *ngIf="options.error" class="m-0 color-red">
-      </p>
+          Sorry that didn't work, try again.
-      <button type="button" class="btn btn-sm btn-outline" (click)="close()">
+        </p>
-        Cancel
+      </div>
-      </button>
+
-      <button
+      <div class="buttons">
-        type="button"
+        <button type="button" class="btn btn-sm btn-outline" (click)="close()">
-        class="btn btn-sm btn-success-outline"
+          Cancel
-        [disabled]="filePasswordInput.value.length < 1"
+        </button>
-        (click)="close(filePasswordInput.value)"
+        <button
-      >
+          type="button"
-        Unlock
+          class="btn btn-sm btn-success-outline"
-      </button>
+          [disabled]="filePasswordInput.value.length < 1"
+          (click)="close(filePasswordInput.value)"
+        >
+          Unlock
+        </button>
+      </div>
     </div>
   </clr-modal>
-</ng-container>
+</div>

client/src/app/shared/excel-password-modal/excel-password-modal.component.scss

@@ -0,0 +1,18 @@
+.excel-password-root {
+  ::ng-deep {
+    .modal {
+      z-index: 1060;
+    }
+  }
+}
+
+.modal-footer {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+
+  .buttons {
+    display: flex;
+    gap: 5px;
+  }
+}

client/src/app/shared/excel-password-modal/excel-password-modal.component.ts

@@ -1,24 +1,24 @@
 import { Component } from '@angular/core'
 import { Observable } from 'rxjs'
-import {
+import { ExcelPasswordModalService } from './excel-password-modal.service'
-  ExcelPasswordModalService,
+import { Options } from './models/options.interface'
-  Options
-} from './excel-password-modal.service'
 
 @Component({
   selector: 'app-excel-password-modal',
+  styleUrls: ['./excel-password-modal.component.scss'],
   templateUrl: './excel-password-modal.component.html'
 })
 export class ExcelPasswordModalComponent {
-  options$: Observable<Options>
+  options$: Observable<Options> = this.excelPasswordModalService.optionsSubject$
 
   fileUnlockError: boolean = false
 
-  constructor(private excelPasswordModalService: ExcelPasswordModalService) {
+  passwordInput: string = ''
-    this.options$ = this.excelPasswordModalService.optionsSubject$
+
-  }
+  constructor(private excelPasswordModalService: ExcelPasswordModalService) {}
 
   close(password?: string) {
+    this.passwordInput = ''
     this.excelPasswordModalService.close(password)
   }
 }

client/src/app/shared/excel-password-modal/excel-password-modal.service.ts

@@ -1,17 +1,7 @@
 import { Injectable } from '@angular/core'
 import { Subject, Observable } from 'rxjs'
-
+import { OpenOptions, Options } from './models/options.interface'
-export interface Options extends OpenOptions {
+import { Result } from './models/result.interface'
-  open: boolean
-}
-
-export interface Result {
-  password: string | undefined
-}
-
-export interface OpenOptions {
-  error?: boolean
-}
 
 @Injectable({
   providedIn: 'root'

client/src/app/shared/excel-password-modal/models/options.interface.ts

@@ -0,0 +1,7 @@
+export interface OpenOptions {
+  error?: boolean
+}
+
+export interface Options extends OpenOptions {
+  open: boolean
+}

client/src/app/shared/excel-password-modal/models/result.interface.ts

@@ -0,0 +1,3 @@
+export interface Result {
+  password: string | undefined
+}

client/src/app/shared/sidebar/sidebar.component.scss

@@ -2,7 +2,6 @@ $sidebarWidth: 272px;
 
 .clr-vertical-nav .nav-link.active {
     background-color: transparent;
-    padding-left: 5px;
 }
 
 clr-vertical-nav {

client/src/app/shared/spreadsheet-util/random.class.ts

@@ -0,0 +1,3 @@
+export class RandomClass {
+  constructor(jo: string) {}
+}

client/src/app/shared/spreadsheet-util/spreadsheet-util.ts

@@ -0,0 +1,917 @@
+import { isSpecialMissing } from '@sasjs/utils/input/validators'
+import {
+  dateFormat,
+  dateToUtcTime,
+  dateToTime
+} from 'src/app/editor/utils/date.utils'
+import {
+  getMissingHeaders,
+  excelDateToJSDate
+} from 'src/app/editor/utils/grid.utils'
+import {
+  isStringNumber,
+  isStringDecimal
+} from 'src/app/editor/utils/types.utils'
+import { FileUploader } from 'src/app/models/FileUploader.class'
+import SheetInfo from 'src/app/models/SheetInfo'
+import { blobToFile } from 'src/app/xlmap/utils/file.utils'
+import * as XLSX from '@sheet/crypto'
+import { LicenceState } from 'src/app/models/LicenceState'
+import { BehaviorSubject } from 'rxjs'
+import { ParseParams } from 'src/app/models/ParseParams.interface'
+import { ParseResult } from 'src/app/models/ParseResult.interface'
+import { OpenOptions } from '../excel-password-modal/models/options.interface'
+
+/**
+ * Used in combination with buffer
+ */
+import * as iconv from 'iconv-lite'
+/**
+ * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
+ * Which will be send to backend
+ */
+import { Buffer } from 'buffer'
+import { MissingHeaders } from 'src/app/models/RangeInfo'
+import { SearchDataExcelResult } from 'src/app/models/SearchDataExcelResult.interface'
+type AOA = any[][]
+
+export interface ConstructorParams {
+  licenceState: BehaviorSubject<LicenceState>
+}
+
+export class SpreadsheetUtil {
+  private licenceState: BehaviorSubject<LicenceState>
+
+  constructor(params: ConstructorParams) {
+    this.licenceState = params.licenceState
+  }
+
+  /**
+   * Parses attached file and searches for the matching data
+   *
+   * If CSV is provided no searching of the data will be executed, but csv file
+   * returned back in an FileUploader array
+   *
+   * @param promptExcelPassword used to trigger the modal for password input
+   * when provided file is locked
+   * @param parseParams params required for parsing the file
+   * @param onParseStateChange callback used to inform about parsing state
+   * so the user of the function can update the UI with latest info
+   * @param onTableFoundEvent callback fired when table range is found in the file
+   *
+   * @returns parsed list of files to upload and JSON data ready for HOT usage
+   */
+  public parseSpreadsheetFile(
+    parseParams: ParseParams,
+    promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
+    onParseStateChange?: (uploadState: string) => void,
+    onTableFoundEvent?: (info: string) => void
+  ): Promise<ParseResult | undefined> {
+    return new Promise((resolve, reject) => {
+      const uploader: FileUploader = parseParams.uploader || new FileUploader()
+
+      // If workbook is present it means file is already read and we don't need
+      // to read it again, otherwise we will do a XLSX.read()
+      if (parseParams.workbook) {
+        this.parseExcelFile(
+          parseParams,
+          parseParams.workbook,
+          uploader,
+          onTableFoundEvent
+        )
+          .then((response) => {
+            resolve(response)
+          })
+          .catch((err) => {
+            reject(err)
+          })
+      } else {
+        // File is not read so we must do a XLSX.read()
+        let data: any[] = []
+
+        const file: File = parseParams.file
+
+        if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
+
+        if (onParseStateChange)
+          onParseStateChange(`Loading ${file.name} into the browser`)
+
+        let fileType = file.name.slice(
+          file.name.lastIndexOf('.') + 1,
+          file.name.lastIndexOf('.') + 4
+        )
+
+        if (fileType.toLowerCase() === 'xls') {
+          let reader: FileReader = new FileReader()
+
+          const self = this
+
+          reader.onload = async (fileReaderResponse: any) => {
+            const wb = await this.xslxStartReading(
+              fileReaderResponse,
+              promptExcelPassword,
+              parseParams.password
+            )
+
+            if (!wb) {
+              return reject('No workbook found.')
+            }
+
+            this.parseExcelFile(parseParams, wb, uploader, onTableFoundEvent)
+              .then((response) => {
+                resolve(response)
+              })
+              .catch((err) => {
+                reject(err)
+              })
+          }
+
+          reader.readAsArrayBuffer(file)
+        } else if (fileType.toLowerCase() === 'csv') {
+          return this.parseCsvFile(parseParams, uploader, fileType)
+        } else {
+          let abortMsg =
+            'Invalid file type "<b>' +
+            parseParams.file.name +
+            '</b>". Please upload csv or excel file.'
+
+          uploader.queue.pop()
+          return reject(abortMsg)
+        }
+      }
+    })
+  }
+
+  private parseExcelFile(
+    parseParams: ParseParams,
+    workbook: XLSX.WorkBook,
+    uploader: FileUploader,
+    onTableFoundEvent?: (info: string) => void
+  ): Promise<ParseResult | undefined> {
+    return new Promise((resolve, reject) => {
+      /* save data */
+      let isComplete: boolean = false
+      let missingHeaders: MissingHeaders[] = []
+
+      const csvArrayHeaders: string[] = [
+        '_____DELETE__THIS__RECORD_____',
+        ...parseParams.headerArray
+      ]
+
+      const searchResult = this.searchDataInExcel(workbook, parseParams)
+
+      let csvArrayData: any[] = []
+
+      if (searchResult.found) {
+        isComplete = true
+        csvArrayData = searchResult.found.arrayData
+
+        if (
+          !searchResult.found.headers.includes('_____delete__this__record_____')
+        ) {
+          csvArrayData = csvArrayData.map((row: any[]) => {
+            // Add empty val on start of the column to compensate for _____delete__this__record_____
+            // when not found in the file
+            row.unshift({ v: '' })
+
+            return row
+          })
+        }
+
+        if (onTableFoundEvent)
+          onTableFoundEvent(
+            `Sheet: ${searchResult.found.sheetName}\nRange: ${searchResult.found.startAddress}:${searchResult.found.endAddress}`
+          )
+      } else {
+        missingHeaders = searchResult.missing || []
+      }
+
+      if (missingHeaders.length > 0) {
+        let abortMsg = missingHeaders
+          .map((mh) => {
+            return `Sheet: ${mh.sheetName}.\nMissing columns: ${mh.missingHeaders.join(',')}`
+          })
+          .join('\n\n')
+
+        uploader.queue.pop()
+        return reject(abortMsg)
+      }
+
+      // If first row is empty, that means no data has been found
+      if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
+        let abortMsg = 'No relevant data found in File !'
+
+        uploader.queue.pop()
+        return reject(abortMsg)
+      }
+
+      if (
+        parseParams.dateTimeHeaders.length > 0 ||
+        parseParams.dateHeaders.length > 0 ||
+        parseParams.timeHeaders.length > 0
+      ) {
+        csvArrayData = this.updateDateTimeCols(
+          csvArrayHeaders,
+          csvArrayData,
+          parseParams
+        )
+      }
+
+      if (parseParams.xlRules.length > 0) {
+        csvArrayData = this.updateXLRuleCols(
+          csvArrayHeaders,
+          csvArrayData,
+          parseParams
+        )
+      }
+
+      if (!isComplete) {
+        let abortMsg = ''
+
+        if (missingHeaders.length === 0) {
+          abortMsg = 'No relevant data found in File !'
+        } else {
+          abortMsg = missingHeaders
+            .map((mh) => {
+              return `Sheet: ${mh.sheetName}.\nMissing columns: ${mh.missingHeaders.join(',')}`
+            })
+            .join('\n\n')
+        }
+
+        // abort message is fired, return undefined
+        uploader.queue.pop()
+        return reject(abortMsg)
+      } else {
+        parseParams.headerShow = csvArrayHeaders
+
+        // Remove the metadata from the cells, leave only values
+        csvArrayData = csvArrayData.map((row: any) =>
+          row.map((col: any) => {
+            if (col.t === 'n') {
+              return col.v
+            } else {
+              if (col.w) return col.v
+
+              return typeof col.v === 'string' ? col.v.trim() : col.v
+            }
+          })
+        )
+
+        csvArrayData = csvArrayData.map((row: any) => {
+          return row.map((col: any, index: number) => {
+            if (!col && col !== 0) col = ''
+
+            /**
+             * Keeping this for the reference
+             * Code below used to convert JSON to CSV
+             * now the XLSX is converting to CSV
+             */
+            // if (isNaN(col)) {
+            //   // Match and replace the double quotes, ignore the first and last char
+            //   // in case they are double quotes already
+            //   col = col.replace(/(?<!^)"(?!$)/g, '""')
+
+            //   if (col.search(/,/g) > -1 ||
+            //       col.search(/\r|\n/g) > -1
+            //   ) {
+            //     // Missing quotes at the end
+            //     if (col.search(/"$/g) < 0) {
+            //       col = col + '"' // So we add them
+            //     }
+
+            //     // Missing quotes at the start
+            //     if (col.search(/^"/g) < 0) {
+            //       col = '"' + col // So we add them
+            //     }
+            //   }
+            // }
+
+            const colName = parseParams.headerShow[index]
+            const colRule = parseParams.dcValidator?.getRule(colName)
+
+            if (colRule?.type === 'numeric') {
+              if (isSpecialMissing(col) && !col.includes('.')) col = '.' + col
+            }
+
+            return col
+          })
+        })
+
+        // Apply licence rows limitation if exists, it is only affecting data
+        // which will be send to SAS
+        const strippedCsvArrayData = csvArrayData.slice(
+          0,
+          this.licenceState.value.submit_rows_limit
+        )
+        // To submit to sas service, we need clean version of CSV of file
+        // attached. XLSX will do the parsing and heavy lifting
+        // First we create worksheet of json (data we extracted)
+        let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+          skipHeader: true
+        })
+
+        // create CSV to be uploaded from worksheet
+        let csvContentClean = XLSX.utils.sheet_to_csv(ws)
+        // Prepend headers
+        csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
+
+        // Blob from which CSV file will be created depending of the selected
+        // encoding
+        let blob: Blob
+
+        if (parseParams.encoding === 'WLATIN1') {
+          // WLATIN1
+          let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
+          blob = new Blob([encoded], { type: 'application/csv' })
+        } else {
+          // UTF-8
+          blob = new Blob([csvContentClean], { type: 'application/csv' })
+        }
+
+        let newCSVFile: File = blobToFile(blob, parseParams.file.name + '.csv')
+        uploader.addToQueue([newCSVFile])
+      }
+
+      if (csvArrayData.length === 0) {
+        return reject(
+          `Table in the file is empty. Data found on sheet: ${searchResult.found?.sheetName || ''}`
+        )
+      }
+
+      if (!searchResult.found) {
+        return reject(
+          `No relevant data found. 'found' object is empty, unexpected error occurred.`
+        )
+      }
+
+      const rangeStartAddress = searchResult.found.startAddress || ''
+      const rangeEndAddress = searchResult.found.endAddress || ''
+
+      return resolve({
+        uploader,
+        data: csvArrayData,
+        rangeSheetRes: {
+          found: !!searchResult.found,
+          sheetName: searchResult.found.sheetName,
+          rangeStartAddress: rangeStartAddress,
+          rangeEndAddress: rangeEndAddress,
+          rangeAddress: `${rangeStartAddress}:${rangeEndAddress}`,
+          missingHeaders: missingHeaders
+        },
+        headerShow: parseParams.headerShow
+      })
+    })
+  }
+
+  private parseCsvFile(
+    parseParams: ParseParams,
+    uploader: FileUploader,
+    fileType: string
+  ) {
+    return new Promise((resolve, reject) => {
+      if (this.licenceState.value.submit_rows_limit !== Infinity) {
+        uploader.queue.pop()
+        return reject(
+          'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
+        )
+      }
+
+      if (parseParams.encoding === 'WLATIN1') {
+        let reader = new FileReader()
+        const self = this
+        // Closure to capture the file information.
+        reader.onload = (theFile: any) => {
+          let encoded = iconv.decode(
+            Buffer.from(theFile.target.result),
+            'CP-1252'
+          )
+          let blob = new Blob([encoded], { type: fileType })
+          let encodedFile: File = blobToFile(blob, parseParams.file.name)
+          uploader.queue.pop()
+          uploader.addToQueue([encodedFile])
+
+          return resolve({
+            uploader
+          })
+        }
+
+        reader.readAsArrayBuffer(parseParams.file)
+      } else {
+        return resolve({
+          uploader
+        })
+      }
+    })
+  }
+
+  public bytesToMB(size: number): number {
+    return parseFloat((size / (1024 * 1024)).toFixed(2))
+  }
+
+  /**
+   * Wrapper function for XLSX.read() with integrated 'unlock' functionality
+   * Used by multi load component to load the file while user chooses the datasets
+   * to be updated
+   *
+   * @param fileReaderResponse response from the file reader
+   * @param promptExcelPassword password callback
+   * @param password password provided by the user
+   * @returns WorkBook
+   */
+  public xslxStartReading(
+    fileReaderResponse: any,
+    promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
+    password?: string
+  ): Promise<XLSX.WorkBook> {
+    return new Promise(async (resolve, reject) => {
+      /* read workbook */
+      const bstr = this.toBstr(fileReaderResponse.target.result)
+      let wb: XLSX.WorkBook | undefined = undefined
+      let fileUnlocking: boolean = false
+
+      const xlsxOptions: XLSX.ParsingOptions = {
+        type: 'binary',
+        cellDates: false,
+        cellFormula: true,
+        cellStyles: true,
+        cellNF: false,
+        cellText: false,
+        password: password
+      }
+
+      try {
+        wb = await this.xlsxRead(bstr, {
+          ...xlsxOptions
+        })
+      } catch (err: any) {
+        if (err.message.toLowerCase().includes('password')) {
+          fileUnlocking = true
+
+          let passwordError = false
+
+          while (fileUnlocking) {
+            const password = await promptExcelPassword({
+              error: passwordError
+            })
+
+            if (password) {
+              try {
+                wb = await this.xlsxRead(bstr, {
+                  ...xlsxOptions,
+                  password: password
+                })
+
+                fileUnlocking = false
+                passwordError = false
+              } catch (err: any) {
+                passwordError = true
+
+                if (!err.message.toLowerCase().includes('password')) {
+                  fileUnlocking = false
+                }
+              }
+            } else {
+              fileUnlocking = false
+            }
+          }
+        } else {
+          return reject('Error reading the file')
+        }
+      }
+
+      if (!wb) return reject('Failed to parse a workbook')
+
+      return resolve(wb)
+    })
+  }
+
+  /**
+   * XLSX Read wrapper which uses Web Worker to read the file and not block
+   * the UI while reading. It will allow reading bigger files.
+   * If worker fails, fallback is regular file read.
+   * @param data
+   * @param opts
+   * @returns
+   */
+  private xlsxRead(
+    data: any,
+    opts?: XLSX.ParsingOptions | undefined
+  ): Promise<XLSX.WorkBook> {
+    return new Promise((resolve, reject) => {
+      if (opts && opts.password) {
+        console.info('Not using worker to parse the XLSX - has password')
+        // At the moment worker can't use crypto version of SheetJS because of
+        // 'global not defined' issue
+        return resolve(XLSX.read(data, opts))
+      }
+
+      if (typeof Worker === 'undefined') {
+        console.info(
+          'Not using worker to parse the XLSX - no Worker available in this environment'
+        )
+        // Web workers are not supported in this environment.
+        // You should add a fallback so that your program still executes correctly.
+        return resolve(XLSX.read(data, opts))
+      }
+
+      // Ultimately use Web Worker to parse the excel
+      console.info('Using worker to parse the XLSX')
+
+      const worker = new Worker(
+        new URL('../../spreadsheet.worker', import.meta.url)
+      )
+
+      worker.onmessage = ({ data }) => {
+        if (data.event === 'reading_end') {
+          resolve(data.workbook)
+        } else if (data.error) {
+          reject(data.error)
+        } else {
+          console.info(
+            'Worker failed to parse the XLSX - fallback to non worker parsing'
+          )
+          // Fallback to reading without Worker
+          resolve(XLSX.read(data, opts))
+        }
+      }
+
+      worker.postMessage({
+        data,
+        opts
+      })
+
+      // Big timeout (10 minutes) in case Worker fails and no response
+      // and read the file with fallback method without worker
+      setTimeout(() => {
+        return resolve(XLSX.read(data, opts))
+      }, 600 * 1000) // 10 minutes
+    })
+  }
+
+  private searchDataInExcel(
+    wb: XLSX.WorkBook,
+    parseParams: ParseParams
+  ): SearchDataExcelResult {
+    let headerStartAddress: string | undefined
+    let headerEndAddress: string | undefined
+    let rangeStartAddress: string | undefined
+    let rangeEndAddress: string | undefined
+    let sheetName: string = ''
+    let missing: MissingHeaders[] = []
+
+    const csvArrayHeaders: string[] = [
+      '_____DELETE__THIS__RECORD_____',
+      ...parseParams.headerArray
+    ]
+    let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
+    let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
+      (map: any, obj: string) => {
+        map[obj] = -1
+        return map
+      },
+      {}
+    )
+
+    for (let sheet of wb.SheetNames) {
+      headerStartAddress = undefined
+      headerEndAddress = undefined
+
+      csvArrayHeadersMap = csvArrayHeadersLower.reduce(
+        (map: any, obj: string) => {
+          map[obj] = -1
+          return map
+        },
+        {}
+      )
+
+      let missingHeaders = []
+      sheetName = sheet
+      const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
+
+      // Find the first header
+      Object.keys(ws).forEach((wsKey) => {
+        const cellValue = ws[wsKey].v
+
+        // If the cell does not have `v` property we ignore it, those are metadata properties
+        if (cellValue && typeof cellValue === 'string') {
+          const potentialHeader = cellValue.toLowerCase()
+          const headerIndex = csvArrayHeadersLower.indexOf(potentialHeader)
+
+          if (headerIndex > -1) {
+            csvArrayHeadersMap[potentialHeader] = wsKey
+
+            if (!headerStartAddress) headerStartAddress = wsKey
+
+            // Update on every found header, until the end, which will leave
+            // last found header address
+            headerEndAddress = wsKey
+          }
+        }
+      })
+
+      // If _____delete__this__record_____ is not found in the file, remove it from the array
+      if (csvArrayHeadersMap['_____delete__this__record_____'] === -1)
+        delete csvArrayHeadersMap['_____delete__this__record_____']
+
+      // Parse missing headers, if any, abort the search and jump to next sheet
+      missingHeaders = Object.keys(csvArrayHeadersMap).filter(
+        (header) => csvArrayHeadersMap[header] === -1
+      )
+
+      if (missingHeaders.length > 0) {
+        missing.push({
+          sheetName: sheetName,
+          missingHeaders: missingHeaders.map((header) => header.toUpperCase())
+        })
+
+        continue
+      }
+
+      // If no headers are missing, start parsing the data column by column
+      const foundHeaders = Object.keys(csvArrayHeadersMap)
+
+      let json: any = []
+      let arrayData: any = []
+      let endRow: number
+
+      // Sort the headers so first headers are primary key columns
+      const foundHeadersSorted = foundHeaders.sort((a: string, b: string) => {
+        const aIsPk = parseParams.headerPks.includes(a) ? 1 : 0
+        const bIsPk = parseParams.headerPks.includes(b) ? 1 : 0
+
+        return bIsPk - aIsPk
+      })
+
+      foundHeadersSorted.forEach((header) => {
+        const headerAddress = csvArrayHeadersMap[header]
+        const headerAddressLetterRegex = headerAddress.match(/\D+/)
+        const headerAddressNumberRegex = headerAddress.match(/\d+/)
+
+        const headerAddressLetter =
+          (headerAddressLetterRegex ? headerAddressLetterRegex[0] : -1) || -1
+        const headerAddressNumber = parseInt(
+          (headerAddressNumberRegex ? headerAddressNumberRegex[0] : -1) || -1
+        )
+
+        const firstDataRow = headerAddressNumber + 1
+
+        let jsonRow = 0
+
+        // If end row found, use it as a limit
+        if (endRow) {
+          for (let row = firstDataRow; row <= endRow; row++) {
+            const address = `${headerAddressLetter}${row}`
+            const cell = ws[address]
+
+            if (parseParams.headerPks.includes(header)) {
+              // If this column is primary key and has less rows, set new endRow
+              if (cell === undefined || cell.v === undefined) {
+                endRow = row
+                break
+              }
+            }
+
+            // Push to array of objects
+            if (!json[jsonRow]) json.push({})
+            if (cell)
+              json[jsonRow][header] =
+                typeof cell.v === 'string' ? cell.v.trim() : cell.v
+
+            // Push to array of arrays, but with all cell meta info
+            if (!arrayData[jsonRow]) arrayData.push([])
+            arrayData[jsonRow].push(cell ?? { v: '' })
+
+            jsonRow++
+          }
+        } else {
+          // If end row not found, go trough rows until empty PK row appears
+          let cellsRow = firstDataRow
+
+          while (endRow === undefined) {
+            const address = `${headerAddressLetter}${cellsRow}`
+            const cell = ws[address]
+
+            if (!cell || cell.v === undefined) {
+              // This is an empty row, row before this one is the last row with data
+              endRow = cellsRow - 1
+              break
+            }
+
+            // Push to array of objects
+            if (!json[jsonRow]) json.push({})
+            if (ws[address].t === 'n') {
+              // If type is number, use the Underlying value, otherwise use Formatted text
+              // https://docs.sheetjs.com/docs/csf/cell
+              json[jsonRow][header] = ws[address].v
+            } else {
+              if (ws[address].w) {
+                json[jsonRow][header] = ws[address].w
+              } else {
+                json[jsonRow][header] =
+                  typeof ws[address].v === 'string'
+                    ? ws[address].v.trim()
+                    : ws[address].v
+              }
+            }
+
+            // Push to array of arrays, but with all cell meta info
+            if (!arrayData[jsonRow]) arrayData.push([])
+            arrayData[jsonRow].push(cell ?? { v: '' })
+
+            cellsRow++
+            jsonRow++
+          }
+        }
+
+        if (headerStartAddress && headerEndAddress) {
+          const endHeaderAddressLetterRegex = headerEndAddress.match(/\D+/)
+
+          rangeStartAddress = headerStartAddress
+          rangeEndAddress = `${endHeaderAddressLetterRegex}${endRow}`
+        }
+      })
+
+      // Remove leftover elements with missing pk values
+      const rowsWithMissingPk: number[] = []
+      let firstRowIndexMissingPk: number | undefined
+
+      json.forEach((row: any, rowIndex: number) => {
+        let missingPk = false
+
+        parseParams.headerPks.forEach((pkHeader) => {
+          if (row[pkHeader.toLowerCase()] === undefined) missingPk = true
+        })
+
+        if (missingPk) {
+          rowsWithMissingPk.push(rowIndex)
+
+          if (!firstRowIndexMissingPk) firstRowIndexMissingPk = rowIndex
+        }
+      })
+
+      // Remove all rows after the first row with missing PK column even if some
+      // columns after has populated PK
+      if (firstRowIndexMissingPk) {
+        json.splice(firstRowIndexMissingPk, Infinity)
+        arrayData.splice(firstRowIndexMissingPk, Infinity)
+      } else {
+        // Fallback: Remove only rows with missing PK
+        rowsWithMissingPk
+          .sort((a, b) => b - a)
+          .forEach((index) => {
+            json.splice(index, 1)
+            arrayData.splice(index, 1)
+          })
+      }
+
+      if (!arrayData.length) {
+        return {}
+      }
+
+      // If we got to this point it means headers are matched
+      return {
+        found: {
+          data: json,
+          arrayData: arrayData,
+          sheetName: sheetName,
+          startAddress: rangeStartAddress,
+          endAddress: rangeEndAddress,
+          headers: foundHeaders
+        }
+      }
+    }
+
+    // No complete data found
+    return {
+      missing: missing
+    }
+  }
+
+  private updateDateTimeCols(
+    headers: any,
+    data: any,
+    parseParams: ParseParams
+  ) {
+    if (parseParams.dateHeaders.length > 0) {
+      const dateCols: number[] = []
+      parseParams.dateHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          dateCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        dateCols.forEach((element) => {
+          const obj = row[element]
+          if (isStringNumber(obj.v)) {
+            const date = excelDateToJSDate(Number(obj.v))
+
+            obj.v =
+              date.getFullYear() +
+              '-' +
+              ('0' + (date.getMonth() + 1)).slice(-2) +
+              '-' +
+              ('0' + date.getDate()).slice(-2)
+          } else {
+            if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
+              const date = new Date(obj.v)
+              if (date.toUTCString() !== 'Invalid Date') {
+                obj.v = dateFormat(date)
+              }
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    if (parseParams.timeHeaders.length > 0) {
+      let timeCols: number[] = []
+      parseParams.timeHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          timeCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        timeCols.forEach((element) => {
+          const obj = row[element]
+          if (
+            isStringNumber(obj.v) ||
+            isStringDecimal(obj.v) ||
+            obj.v.includes('E-')
+          ) {
+            const date = excelDateToJSDate(Number(obj.v))
+
+            obj.v = dateToUtcTime(date)
+          }
+          row[element] = obj
+        })
+      })
+    }
+    if (parseParams.dateTimeHeaders.length > 0) {
+      let dateTimeCols: number[] = []
+      parseParams.dateTimeHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          dateTimeCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        dateTimeCols.forEach((element) => {
+          const obj = row[element]
+          if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
+            const date = excelDateToJSDate(Number(obj.v))
+            obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
+          } else {
+            if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
+              let str = obj.v.substring(0, obj.v.indexOf(':'))
+              str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
+              obj.v = str
+            }
+            const date = new Date(obj.v)
+            if (date.toUTCString() !== 'Invalid Date') {
+              obj.v = dateFormat(date) + ' ' + dateToTime(date)
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    return data
+  }
+
+  private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
+    if (parseParams.xlRules.length > 0) {
+      const xlRuleCols: any = []
+      parseParams.xlRules.forEach((element: any) => {
+        if (headers.indexOf(element.XL_COLUMN) !== -1) {
+          element['index'] = headers.indexOf(element.XL_COLUMN)
+          xlRuleCols.push(element)
+        }
+      })
+      data.forEach((row: any[]) => {
+        xlRuleCols.forEach((element: any) => {
+          const obj = row[element.index]
+          if (element.XL_RULE === 'FORMULA') {
+            if ('f' in obj) {
+              if (obj['t'] === 'n') {
+                obj['v'] = '=' + obj['f']
+              } else {
+                obj['w'] = '=' + obj['f']
+              }
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    return data
+  }
+
+  private toBstr(res: any) {
+    let bytes = new Uint8Array(res)
+    let binary = ''
+    let length = bytes.byteLength
+    for (let i = 0; i < length; i++) {
+      binary += String.fromCharCode(bytes[i])
+    }
+    return binary
+  }
+}

client/src/app/shared/terms/terms.component.ts

@@ -8,6 +8,7 @@ import {
 import { SasService } from '../../services/sas.service'
 import * as marked from 'marked'
 import { EULA } from 'src/environments/_eula'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-terms',
@@ -51,8 +52,12 @@ export class TermsComponent implements OnInit, AfterViewInit {
 
     this.sasService
       .request(`public/registeruser`, table)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
+        if (
+          res.adapterResponse.return &&
+          res.adapterResponse.return[0] &&
+          res.adapterResponse.return[0].MSG === 'SUCCESS'
+        ) {
           location.reload()
         }
       })

client/src/app/spreadsheet.worker.ts

@@ -0,0 +1,30 @@
+/// <reference lib="webworker" />
+
+/**
+ * We use normal version of the XLSX (SheetJS)
+ * Because at the moment "@sheet/crypto" can't work in the Web Worker environment
+ * Because of the missing "global" variable.
+ */
+import * as XLSX from 'xlsx'
+
+addEventListener('message', ({ data }) => {
+  const { data: xldata, opts: xlopts } = data as {
+    data: any
+    opts?: any
+  }
+
+  try {
+    const workbook = XLSX.read(xldata, xlopts)
+
+    postMessage({
+      event: 'reading_end',
+      workbook: workbook
+    })
+  } catch (err: any) {
+    if (err.message.toLowerCase().includes('password')) {
+      postMessage({
+        error: err
+      })
+    }
+  }
+})

client/src/app/stage/stage.component.ts

@@ -8,6 +8,7 @@ import { HotTableInterface } from '../models/HotTable.interface'
 import { LicenceService } from '../services/licence.service'
 import { globals } from '../_globals'
 import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-stage',
@@ -180,10 +181,12 @@ export class StageComponent implements OnInit {
 
     this.sasService
       .request('editors/restore', data)
-      .then((res: EditorsRestoreServiceResponse) => {
+      .then((res: RequestWrapperResponse<EditorsRestoreServiceResponse>) => {
-        if (res.restore_out) {
+        if (res.adapterResponse.restore_out) {
           this.route.navigate([`/stage`]).then(() => {
-            this.route.navigate([`/stage/${res.restore_out[0].LOADREF}`])
+            this.route.navigate([
+              `/stage/${res.adapterResponse.restore_out[0].LOADREF}`
+            ])
           })
         }
       })

client/src/app/system/models/environment-info.model.ts

@@ -1,15 +1,16 @@
 export interface EnvironmentInfo {
-  SYSSITE: string
+  SYSSITE?: string
-  SYSSCPL: string
+  SYSSCPL?: string
-  SYSTCPIPHOSTNAME: string
+  SYSTCPIPHOSTNAME?: string
-  SYSVLONG: string
+  SYSVLONG?: string
-  MEMSIZE: string
+  MEMSIZE?: string
-  SYSPROCESSMODE: string
+  SYSPROCESSMODE?: string
-  SYSHOSTNAME: string
+  SYSHOSTNAME?: string
-  SYSUSERID: string
+  SYSUSERID?: string
-  SYSHOSTINFOLONG: string
+  SYSHOSTINFOLONG?: string
-  SYSENCODING: string
+  SYSENCODING?: string
-  AUTOEXEC: string
+  AUTOEXEC?: string
-  ISADMIN: number
+  ISADMIN?: number
-  DC_ADMIN_GROUP: string
+  DC_ADMIN_GROUP?: string
+  APP_LOC?: string
 }

client/src/app/system/system.component.html

@@ -77,6 +77,10 @@
               DC Admin Group:
               <span class="dark">{{ environmentInfo?.DC_ADMIN_GROUP }}</span>
             </p>
+            <p cds-text="label" class="m-0">
+              App Location:
+              <span class="dark">{{ environmentInfo?.APP_LOC }}</span>
+            </p>
           </div>
         </div>
 

client/src/app/system/system.component.ts

@@ -9,6 +9,7 @@ import { AppInfo } from './models/app-info.model'
 import { EnvironmentInfo } from './models/environment-info.model'
 import { AppSettingsService } from '../services/app-settings.service'
 import { AppSettings } from '../models/AppSettings'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-system',
@@ -51,7 +52,7 @@ export class SystemComponent implements OnInit {
     this.environmentInfo = this.appService.getEnvironmentInfo()
     this.settings = this.appSettingsService.settings.value
 
-    if (this.environmentInfo) {
+    if (this.environmentInfo.AUTOEXEC) {
       this.environmentInfo.AUTOEXEC = decodeURIComponent(
         this.environmentInfo.AUTOEXEC
       )
@@ -83,12 +84,12 @@ export class SystemComponent implements OnInit {
 
     this.sasService
       .request('admin/refreshcatalog', null)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        this.response = this.parseResponse(res)
+        this.response = this.parseResponse(res.adapterResponse)
         this.responseModal = true
       })
       .catch((err: any) => {
-        this.response = this.parseResponse(err)
+        this.response = this.parseResponse(err.adapterResponse)
         this.responseModal = true
       })
       .finally(() => {
@@ -101,12 +102,12 @@ export class SystemComponent implements OnInit {
 
     this.sasService
       .request('admin/refreshtablelineage', null)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
-        this.response = this.parseResponse(res)
+        this.response = this.parseResponse(res.adapterResponse)
         this.responseModal = true
       })
       .catch((err: any) => {
-        this.response = this.parseResponse(err)
+        this.response = this.parseResponse(err.adapterResponse)
         this.responseModal = true
       })
       .finally(() => {

client/src/app/user/user.component.ts

@@ -6,6 +6,7 @@ import { Location } from '@angular/common'
 import { SasService } from '../services/sas.service'
 import { SASjsConfig } from '@sasjs/adapter'
 import { ServerType } from '@sasjs/utils/types/serverType'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-user',
@@ -89,10 +90,10 @@ export class UserComponent implements OnInit {
         } else {
           this.sasService
             .request('usernav/usermembers', null)
-            .then((res: any) => {
+            .then((res: RequestWrapperResponse) => {
               this.loading = false
-              this.users = res.users
+              this.users = res.adapterResponse.users
-              globals.usernav.userList = res.users
+              globals.usernav.userList = res.adapterResponse.users
             })
         }
       } else {
@@ -141,18 +142,18 @@ export class UserComponent implements OnInit {
 
           this.sasService
             .request('usernav/usergroupsbymember', data)
-            .then((res: any) => {
+            .then((res: RequestWrapperResponse) => {
               this.loading = false
 
               switch (this.serverType) {
                 case ServerType.Sas9: {
-                  this.userInfo = res.info[0]
+                  this.userInfo = res.adapterResponse.info[0]
-                  this.userEmails = res.emails
+                  this.userEmails = res.adapterResponse.emails
-                  this.userEmailsCount = res.emails.length
+                  this.userEmailsCount = res.adapterResponse.emails.length
-                  this.userRoles = res.roles
+                  this.userRoles = res.adapterResponse.roles
-                  this.userRolesCount = res.roles.length
+                  this.userRolesCount = res.adapterResponse.roles.length
-                  this.userLogins = res.logins
+                  this.userLogins = res.adapterResponse.logins
-                  this.userLoginsCount = res.logins.length
+                  this.userLoginsCount = res.adapterResponse.logins.length
 
                   break
                 }
@@ -162,7 +163,7 @@ export class UserComponent implements OnInit {
                       (userAr) => userAr.URI === uri
                     )
                   } else {
-                    const group = res.groups[0]
+                    const group = res.adapterResponse.groups[0]
 
                     this.userInfo = {
                       URI: group.ID,
@@ -175,9 +176,9 @@ export class UserComponent implements OnInit {
                 }
               }
 
-              this.userData = res
+              this.userData = res.adapterResponse
-              this.userGroups = res.groups
+              this.userGroups = res.adapterResponse.groups
-              this.userGroupsCount = res.groups.length
+              this.userGroupsCount = res.adapterResponse.groups.length
             })
         }
       }
@@ -238,18 +239,18 @@ export class UserComponent implements OnInit {
       let data = { iwant: [{ uri: user.URI }] }
       this.sasService
         .request('usernav/usergroupsbymember', data)
-        .then((res: any) => {
+        .then((res: RequestWrapperResponse) => {
           this.loading = false
 
           switch (this.serverType) {
             case ServerType.Sas9: {
-              this.userInfo = res.info[0]
+              this.userInfo = res.adapterResponse.info[0]
-              this.userEmails = res.emails
+              this.userEmails = res.adapterResponse.emails
-              this.userEmailsCount = res.emails.length
+              this.userEmailsCount = res.adapterResponse.emails.length
-              this.userRoles = res.roles
+              this.userRoles = res.adapterResponse.roles
-              this.userRolesCount = res.roles.length
+              this.userRolesCount = res.adapterResponse.roles.length
-              this.userLogins = res.logins
+              this.userLogins = res.adapterResponse.logins
-              this.userLoginsCount = res.logins.length
+              this.userLoginsCount = res.adapterResponse.logins.length
 
               break
             }
@@ -259,7 +260,7 @@ export class UserComponent implements OnInit {
                   (userAr) => userAr.URI === user.URI
                 )
               } else {
-                const group = res.groups[0]
+                const group = res.adapterResponse.groups[0]
 
                 this.userInfo = {
                   URI: group.ID,
@@ -272,9 +273,9 @@ export class UserComponent implements OnInit {
             }
           }
 
-          this.userData = res
+          this.userData = res.adapterResponse
-          this.userGroups = res.groups
+          this.userGroups = res.adapterResponse.groups
-          this.userGroupsCount = res.groups.length
+          this.userGroupsCount = res.adapterResponse.groups.length
         })
     }
   }

client/src/app/viewer/viewer.component.ts

@@ -37,6 +37,7 @@ import { DataFormat } from '../models/sas/common/DateFormat'
 import { Libinfo } from '../models/sas/common/Libinfo'
 import { LicenceService } from '../services/licence.service'
 import { Location } from '@angular/common'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-viewer',

client/src/app/xlmap/xlmap.component.ts

@@ -22,6 +22,7 @@ import {
 } from '../services'
 import { getCellAddress, getFinishingCell } from './utils/xl.utils'
 import { blobToFile, byteArrayToBinaryString } from './utils/file.utils'
+import { UploadFileResponse } from '../models/UploadFile'
 
 interface XLMapRule {
   XLMAP_ID: string
@@ -340,9 +341,9 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
       .uploadFile(uploadUrl, filesToUpload, {
         table: this.selectedXLMap.targetDS
       })
-      .then((res: any) => {
+      .then((res: UploadFileResponse) => {
-        if (res.sasjsAbort) {
+        if (res.adapterResponse.sasjsAbort) {
-          const abortRes = res
+          const abortRes = res.adapterResponse
           const abortMsg = abortRes.sasjsAbort[0].MSG
           const macMsg = abortRes.sasjsAbort[0].MAC
 
@@ -351,14 +352,14 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
             SYSERRORTEXT: abortRes.SYSERRORTEXT,
             MAC: macMsg
           })
-        } else if (res.sasparams) {
+        } else if (res.adapterResponse.sasparams) {
-          const params = res.sasparams[0]
+          const params = res.adapterResponse.sasparams[0]
           const tableId = params.DSID
           this.router.navigateByUrl('/stage/' + tableId)
         }
       })
       .catch((err: any) => {
-        this.eventService.catchResponseError('file upload', err)
+        this.eventService.catchResponseError('file upload', err.response)
       })
       .finally(() => {
         this.status = Status.ReadyToSubmit

client/tsconfig.json

@@ -46,5 +46,9 @@
       "strictInjectionParameters": true,
       "strictInputAccessModifiers": true,
       "strictTemplates": true,
-    }
+    },
+    "exclude": [
+      "cypress/**/*.ts",
+      "cypress.config.ts"
+    ]
 }

client/tsconfig.worker.json

@@ -0,0 +1,15 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/worker",
+    "lib": [
+      "es2018",
+      "webworker"
+    ],
+    "types": []
+  },
+  "include": [
+    "src/**/*.worker.ts"
+  ]
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "6.11.0",
+  "version": "6.12.2",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "^4.11.1",
-        "@sasjs/core": "^4.52.4"
+        "@sasjs/core": "^4.52.5"
       }
     },
     "node_modules/@coolaj86/urequest": {
@@ -116,9 +116,9 @@
       "integrity": "sha512-Grwydm5GxBsYk238PZw41XPjXVVQ9vWcvfZ06L2P0bQbvK0sGn7l69JA7H5MGr3QcaLpiD4Kg70cAh7PgE+JOw=="
     },
     "node_modules/@sasjs/core": {
-      "version": "4.52.4",
+      "version": "4.52.5",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.52.4.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.52.5.tgz",
-      "integrity": "sha512-8lf5ixlA312EgA2DorwbpNXXPfLPzUHO67exIV7SjKiU23Tn1au5GD6hT0Ysr2kophOs10Mp1TCXJjhEq7Qk4A=="
+      "integrity": "sha512-fGuLC+DcH2AoIDDU/Eyn7d4ZIrIeQAN3PB9FgjBikiGuXZYooRELKz1WgaQvhI4qSKgczUANaT80ZJOpfH+sQQ=="
     },
     "node_modules/@sasjs/lint": {
       "version": "2.3.1",

sas/package.json

@@ -29,6 +29,6 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "^4.11.1",
-    "@sasjs/core": "^4.52.4"
+    "@sasjs/core": "^4.52.5"
   }
 }

sas/sasjs/macros/bitemporal_dataloader.sas

@@ -100,6 +100,7 @@ Areas for optimisation
   @li mf_getattrn.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
+  @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
   @li mf_getuser.sas
   @li mf_getvarlist.sas
@@ -621,7 +622,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
 %put DCNOTE: Extracting matching observations from &base_lib..&base_dsn;
 
 %if &engine_type=OLEDB %then %do;
-  %let temp_table=##BITEMP_&base_dsn;
+  %let temp_table=##%mf_getuniquefileref(prefix=BTMP);
   %if &loadtype=BITEMPORAL or &loadtype=TXTEMPORAL %then
         %let base_table=(select * from [dbo].&base_dsn
                         where convert(datetime,&SQLNOW) < &tech_to );
@@ -1049,7 +1050,7 @@ run;
   %let cat_string=catx('|' ,&bus_from,&bus_to);
 
   data work.bitemp5a_lkp (keep=&md5_col)
-    %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    %if "%substr(&sysver,1,1)" ne "4" & "%substr(&sysver,1,1)" ne "5" %then %do;
       /nonote2err
     %end;
     ;
@@ -1191,10 +1192,10 @@ run;
   /* if OLEDB then create a temp table for efficiency */
   %local innertable;
   %if &engine_type=OLEDB %then %do;
-    %let innertable=[##BITEMP_&base_dsn];
+    %let innertable=[&temp_table];
     %let top_table=[dbo].&base_dsn;
     %let flexinow=&SQLNOW;
-    create table &base_lib.."##BITEMP_&base_dsn"n as
+    create table &base_lib.."&temp_table"n as
       select * from work.bitemp5d_subquery;
     /* open up a connection for pass through SQL */
     %dc_assignlib(WRITE,&base_lib,passthru=myAlias)

sas/sasjs/services/approvers/gethistory.sas

@@ -105,8 +105,6 @@ create view work.submits as
     ,a.submitted_by_nm  as submitter
     ,put(a.REVIEWED_ON_DTTM,yymmddhhmmss.) as REVIEWED
     ,a.submit_status_cd as status
-    ,a.reviewed_on_dttm
-    ,a.reviewed_by_nm as approver
   from &mpelib..mpe_submit(where=(submit_status_cd ne 'SUBMITTED')) a
 
 %macro gethistory();
@@ -120,8 +118,18 @@ create view work.submits as
 %end;
 %mend gethistory;
 %gethistory()
+;
 
-  order by a.submitted_on_dttm desc;
+/* get latest reason text */
+create table work.reviews as 
+  select a.* 
+    ,b.reviewed_on_dttm
+    ,b.reviewed_by_nm as approver
+    ,b.review_reason_txt 
+  from work.submits a 
+  left join &mpelib..mpe_review b 
+  on a.table_id=b.table_id 
+  order by a.table_id desc, b.reviewed_on_dttm desc;
 
 %mp_abort(iftrue= (&syscc > 0)
   ,mac=&_program
@@ -129,7 +137,9 @@ create view work.submits as
 )
 
 data work.fromsas;
-  set work.submits;
+  set work.reviews;
+  by descending table_id descending reviewed_on_dttm;
+  if first.table_id;
   if _n_ ge &startrow;
   n+1;
   if n>&hist then stop;

sas/sasjs/services/hooks/mpe_tables_postedit.sas

@@ -23,8 +23,11 @@
 /* ensure uppercasing */
 data work.staging_ds;
   set work.staging_ds;
-  LIBREF=upcase(LIBREF);
+  /* PK fields should not be upcased if we are trying to delete records */
-  DSN=upcase(DSN);
+  if upcase(_____DELETE__THIS__RECORD_____) ne "YES" then do;
+    LIBREF=upcase(LIBREF);
+    DSN=upcase(DSN);
+  end;
   loadtype=upcase(loadtype);
   buskey=upcase(buskey);
   var_txfrom=upcase(var_txfrom);

sas/sasjs/targets/sas9/services_meta/admin/configurator.sas

@@ -188,6 +188,7 @@ data _null_;
     put "<li> METAPERSON: " metaperson "</li>";
     put "<li> METAUSER: " metauser "</li>";
     put "<li> SYSUSERID: " sysuserid "</li>";
+    put "<li> SYSSITE: &syssite</li>";
     url='/SASStoredProcess/do';
   end;
   else do;

sas/sasjs/targets/server/services_server/admin/configurator.sas

@@ -222,6 +222,7 @@ data _null_;
   put "<li> SYSUSERID: " sysuserid "</li>";
   put "<li> SYSHOSTNAME: &syshostname</li>";
   put "<li> SYSVLONG: &sysvlong</li>";
+  put "<li> SYSSITE: &syssite</li>";
 run;
 
 data _null_;