chore(release): 6.12.0 [skip ci]

# [6.12.0](https://git.datacontroller.io/dc/dc/compare/v6.11.1...v6.12.0) (2024-09-02)

### Bug Fixes

* added appLoc to the system page ([dd2138a](dd2138ac5e))
* bumping core for passthrough fix, [#124](#124) ([caa9854](caa9854ff0))
* excel with password flow, introducing web worker for XLSX.read ([a3ce367](a3ce367950))
* implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height ([4218da9](4218da91cd))
* **multi load:** xlsx read file ahead of time, while user choose datasets ([6547461](6547461637))
* refactored adapter request wrapper function to return job log as well ([67436f4](67436f4ff9))
* using temporary names for temporary tables ([ce50365](ce503653cd)), closes [#124](#124)

### Features

* searching data in excel files using new algorithm (massive performance improvement) ([bbb725c](bbb725c64c))

.gitea/workflows/release.yaml

@@ -136,7 +136,7 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          npm start & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+          npm start & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

CHANGELOG.md

@@ -1,3 +1,56 @@
+# [6.12.0](https://git.datacontroller.io/dc/dc/compare/v6.11.1...v6.12.0) (2024-09-02)
+
+
+### Bug Fixes
+
+* added appLoc to the system page ([dd2138a](https://git.datacontroller.io/dc/dc/commit/dd2138ac5e6067de310e83d16fccc9b9764ba3ff))
+* bumping core for passthrough fix, [#124](https://git.datacontroller.io/dc/dc/issues/124) ([caa9854](https://git.datacontroller.io/dc/dc/commit/caa9854ff0431ccbb6ff1d6d3509dc877362cceb))
+* excel with password flow, introducing web worker for XLSX.read ([a3ce367](https://git.datacontroller.io/dc/dc/commit/a3ce36795007a4e3b6ac3499ffd119dc3758f387))
+* implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height ([4218da9](https://git.datacontroller.io/dc/dc/commit/4218da91cd193aa45346ad7e34ccc00ca89df4fb))
+* **multi load:** xlsx read file ahead of time, while user choose datasets ([6547461](https://git.datacontroller.io/dc/dc/commit/65474616379e1dacc1329b3bdc5eb14f34428bb1))
+* refactored adapter request wrapper function to return job log as well ([67436f4](https://git.datacontroller.io/dc/dc/commit/67436f4ff9bb4d77d5f897f47a3e3d472981f275))
+* using temporary names for temporary tables ([ce50365](https://git.datacontroller.io/dc/dc/commit/ce503653cd9fc36f72fb172bd14816e07c792e14)), closes [#124](https://git.datacontroller.io/dc/dc/issues/124)
+
+
+### Features
+
+* searching data in excel files using new algorithm (massive performance improvement) ([bbb725c](https://git.datacontroller.io/dc/dc/commit/bbb725c64cc23ed701b189623992408c42fdde8f))
+
+## [6.11.1](https://git.datacontroller.io/dc/dc/compare/v6.11.0...v6.11.1) (2024-07-02)
+
+
+### Bug Fixes
+
+* adding SYSSITE, part of [#116](https://git.datacontroller.io/dc/dc/issues/116) ([a156c01](https://git.datacontroller.io/dc/dc/commit/a156c0111b3de5e3744e38d377d6e9aa09915803))
+* ensuring review_reason_txt in output.  Closes [#117](https://git.datacontroller.io/dc/dc/issues/117) ([e5d93fd](https://git.datacontroller.io/dc/dc/commit/e5d93fd7d6d86bc47ff56664bd812b4d9d0749a5))
+
+# [6.11.0](https://git.datacontroller.io/dc/dc/compare/v6.10.1...v6.11.0) (2024-06-27)
+
+
+### Bug Fixes
+
+* addressing PR comments ([d94df7f](https://git.datacontroller.io/dc/dc/commit/d94df7f0ebae8feab5e1d5cf8011af8c8be2ca18))
+* **multi load:** fixed parsing algorithm reused for the multi load, the fix affects the normal upload as well. ([d4fee79](https://git.datacontroller.io/dc/dc/commit/d4fee791a72021e449cf9680c3e3a525dce41ac1))
+* **multi load:** label rename ([fa04d7b](https://git.datacontroller.io/dc/dc/commit/fa04d7bf4e5ba337146bdaa926c60488f8851449))
+
+
+### Features
+
+* **multi load:** added HOT for user datasets input ([18363bb](https://git.datacontroller.io/dc/dc/commit/18363bbbeb9cf96183ba4841da8134b2f66f735c))
+* **multi load:** implemented matching libds and parsing of the multiple sheets ([efcdc69](https://git.datacontroller.io/dc/dc/commit/efcdc694dd275cdb9a4e19f26e5522b8dadc5fd9))
+* **multi load:** licence submit limits ([cffeab8](https://git.datacontroller.io/dc/dc/commit/cffeab813d8d4b324f82710dfd73953d4cbf8ffe))
+* **multi load:** multiple csv files ([4d27665](https://git.datacontroller.io/dc/dc/commit/4d276657b35a147a2233a03afcb1716348555f52))
+* **multi load:** refactored range find function, unlocking excel with password is reusable ([eb7c443](https://git.datacontroller.io/dc/dc/commit/eb7c44333c865e7f7bbfb54dd7f73bfc110f86a7))
+* **multi load:** submitting multiple found tables at once ([5deba44](https://git.datacontroller.io/dc/dc/commit/5deba44d2b7352866d821b70dbbfbbf54955dc47))
+
+## [6.10.1](https://git.datacontroller.io/dc/dc/compare/v6.10.0...v6.10.1) (2024-06-07)
+
+
+### Bug Fixes
+
+* adding 60 more colours to crayons table. Closes [#112](https://git.datacontroller.io/dc/dc/issues/112) ([3521579](https://git.datacontroller.io/dc/dc/commit/3521579dead089eebf62455686be3aee88bde687))
+* terms and conditions colours, editor on smaller screens show only icons ([e32d44b](https://git.datacontroller.io/dc/dc/commit/e32d44b1bcdfeea43d19b21ec0ddf4af1ce3992a))
+
 # [6.10.0](https://git.datacontroller.io/dc/dc/compare/v6.9.0...v6.10.0) (2024-06-07)
 
 

client/angular.json

@@ -69,7 +69,8 @@
             ],
             "scripts": [
               "node_modules/marked/marked.min.js"
-            ]
+            ],
+            "webWorkerTsConfig": "tsconfig.worker.json"
           },
           "configurations": {
             "production": {
@@ -148,7 +149,8 @@
               "src/styles.scss"
             ],
             "scripts": [],
-            "karmaConfig": "karma.conf.js"
+            "karmaConfig": "karma.conf.js",
+            "webWorkerTsConfig": "tsconfig.worker.json"
           }
         },
         "lint": {

client/cypress/e2e/editor.cy.ts

@@ -15,9 +15,6 @@ context('editor tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
   })
@@ -118,10 +115,6 @@ context('editor tests: ', function () {
       })
     })
   })
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const clickOnEdit = (callback?: any) => {

client/cypress/e2e/excel-multi-load.cy.ts

@@ -0,0 +1,228 @@
+import { Callbacks } from 'cypress/types/jquery/index'
+
+const username = Cypress.env('username')
+const password = Cypress.env('password')
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+const serverType = Cypress.env('serverType')
+const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
+const fixturePath = 'excels_multi_load/'
+
+const library = 'DC996664'
+const mpeXTestTable = 'MPE_X_TEST'
+const mpeTablesTable = 'MPE_TABLES'
+
+context('excel multi load tests: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`)
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    visitPage('home/multi-load')
+
+    colorLog(
+      `TEST START ---> ${
+        Cypress.mocha.getRunner().suite.ctx.currentTest.title
+      }`,
+      '#3498DB'
+    )
+  })
+
+  it('1 | Uploads Excel file with multiple sheets, 3 sheets including data, 2 sheets matched with dataset', (done) => {
+    attachExcelFile('multi_load_test_2.xlsx', () => {
+      checkHotUserDatasetTable('hotTableUserDataset', [
+        [library, mpeXTestTable],
+        [library, mpeTablesTable]
+      ], () => {
+        cy.get('#continue-btn').trigger('click').then(() => {
+          checkIfTreeHasTables([`${library}.${mpeXTestTable}`, `${library}.${mpeTablesTable}`], undefined, (includes: boolean) => {
+            if (includes) {
+              // MPE_TABLES sheet does not have data so 1 error image must be shown
+              hasErrorTables(1, (valid: boolean) => {
+                if (valid) done()
+              })
+            }
+          })
+        })
+      })
+    })
+  })
+
+  it('2 | Uploads Excel file with multiple sheets, 2 sheets matched with dataset, 1 matched sheet does not have data', (done) => {
+    attachExcelFile('multi_load_test_1.xlsx', () => {
+      checkHotUserDatasetTable('hotTableUserDataset', [
+        [library, mpeXTestTable],
+        [library, mpeTablesTable]
+      ], () => {
+        cy.get('#continue-btn').trigger('click').then(() => {
+          checkIfTreeHasTables([`${library}.${mpeXTestTable}`, `${library}.${mpeTablesTable}`], `${library}.${mpeXTestTable}`, (includes: boolean) => {
+            if (includes) {
+              cy.get('#hotTable').should('be.visible').then(() => {
+                checkHotUserDatasetTable('hotTable', [
+                  ['No', '1', 'more dummy data'],
+                  ['No', '1', 'It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told:'],
+                  ['No', '1', 'if you can fill the unforgiving minute']
+                ], () => {
+                  submitTables()
+
+                  hasSuccessSubmits(2, (valid: boolean) => {
+                    if (valid) done()
+                  })
+
+                })
+              })
+            }
+          })
+        })
+      })
+    })
+  })
+
+  it('3 | Uploads Excel file with multiple sheets, 1 sheets has 2 tables', (done) => {
+    attachExcelFile('multi_load_test_1.xlsx', () => {
+      checkHotUserDatasetTable('hotTableUserDataset', [
+        [library, mpeXTestTable],
+        [library, mpeTablesTable]
+      ], () => {
+        cy.get('#continue-btn').trigger('click').then(() => {
+          checkIfTreeHasTables([`${library}.${mpeXTestTable}`, `${library}.${mpeTablesTable}`], `${library}.${mpeXTestTable}`, (includes: boolean) => {
+            if (includes) {
+              cy.get('#hotTable').should('be.visible').then(() => {
+                checkHotUserDatasetTable('hotTable', [
+                  ['No', '1', 'more dummy data'],
+                  ['No', '1', 'It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told:'],
+                  ['No', '1', 'if you can fill the unforgiving minute']
+                ], () => {
+                  clickOnTreeNode('DC996664.MPE_TABLES', () => {
+                    cy.wait(1000).then(() => {
+                      cy.get('#hotTable').should('be.visible').then(() => {
+                        checkHotUserDatasetTable('hotTable', [
+                          ['No', 'DC914286', 'MPE_COLUMN_LEVEL_SECURITY'],
+                          ['No', 'DC914286', 'MPE_XLMAP_INFO'],
+                          ['No', 'DC914286', 'MPE_XLMAP_RULES']
+                        ], () => {
+                          submitTables()
+
+                          hasSuccessSubmits(2, (valid: boolean) => {
+                            if (valid) done()
+                          })
+
+                        })
+                      })
+                    })
+                  })
+                })
+              })
+            }
+          })
+        })
+      })
+    })
+  })
+
+  this.afterEach(() => {
+    colorLog(`TEST END -------------`, '#3498DB')
+  })
+})
+
+const attachExcelFile = (excelFilename: string, callback?: any) => {
+  cy.get('#browse-file')
+    .should('exist')
+    .click()
+    .then(() => {
+      cy.get('input[type="file"]#file-upload')
+        .attachFile(`/${fixturePath}/${excelFilename}`)
+        .then(() => {
+          if (callback) callback()
+        })
+    })
+}
+
+const checkHotUserDatasetTable = (hotId: string, dataToContain: any[][], callback?: () => void) => {
+  cy.get(`#${hotId}`, { timeout: longerCommandTimeout })
+  .find('div.ht_master.handsontable')
+  .find('div.wtHolder')
+  .find('div.wtHider')
+  .find('div.wtSpreader')
+  .find('table.htCore')
+  .find('tbody')
+  .then((data) => {
+    cy.wait(2000).then(() => {
+      for (let rowI = 0; rowI < dataToContain.length; rowI++) {
+        for (let colI = 0; colI < dataToContain[rowI].length; colI++) {
+          expect(data[0].children[rowI].children[colI]).to.contain(dataToContain[rowI][colI])
+        }
+      }
+
+      if (callback) callback()
+    })
+  })
+}
+
+const clickOnTreeNode = (clickOnNode: string, callback?: () => void) => {
+  cy.get('.nav-tree clr-tree > clr-tree-node').then((treeNodes: any) => {
+    for (let node of treeNodes) {
+      if (node.innerText.toUpperCase().trim().includes(clickOnNode)) {
+        cy.get(node).trigger('click')
+        if (callback) callback()
+      }
+    }
+  })
+}
+
+const checkIfTreeHasTables = (tables: string[], clickOnNode?: string, callback?: (includes: boolean) => void) => {
+  cy.get('.nav-tree clr-tree > clr-tree-node').then((treeNodes: any) => {
+    let datasets = tables
+    let nodesCorrect = true
+    let nodeToClick
+
+    for (let node of treeNodes) {
+      if (!datasets.includes(node.innerText.toUpperCase().trim())) {
+        nodesCorrect = false
+      }
+
+      if (clickOnNode) {
+        if (node.innerText.toUpperCase().trim().includes(clickOnNode)) {
+          nodeToClick = node
+        }
+      }
+    }
+
+    if (nodeToClick) {
+      cy.wait(1000)
+      cy.get(nodeToClick).trigger('click')
+    }
+
+    if (callback) callback(nodesCorrect)
+  })
+}
+
+const submitTables = () => {
+  cy.get('#submit-all').trigger('click')
+  cy.get('#submit-tables').trigger('click')
+  cy.wait(1000)
+}
+
+const hasSuccessSubmits = (expectedNoOfSubmits: number, callback: (valid: boolean) => void) => {
+  cy.get('.nav-tree clr-tree > clr-tree-node cds-icon[status="success"]').should('be.visible').then(($nodes) => {
+    callback(expectedNoOfSubmits === $nodes.length)
+  })
+}
+
+const hasErrorTables = (expectedNoOfErrors: number, callback: (valid: boolean) => void) => {
+  cy.get('.nav-tree clr-tree > clr-tree-node cds-icon[status="danger"]').should('be.visible').then(($nodes) => {
+    callback(expectedNoOfErrors === $nodes.length)
+  })
+}
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}
+
+const colorLog = (msg: string, color: string) => {
+  console.log('%c' + msg, 'color:' + color + ';font-weight:bold;')
+}

client/cypress/e2e/excel.cy.ts

@@ -17,9 +17,6 @@ context('excel tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
 
@@ -112,13 +109,8 @@ context('excel tests: ', function () {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
 
     attachExcelFile('duplicate_column_excel.xlsx', () => {
-      cy.get('.abortMsg', { timeout: longerCommandTimeout })
-        .should('exist')
-        .then((elements: any) => {
-          if (elements[0]) {
-            if (elements[0].innerText.toLowerCase().includes('missing')) done()
-          }
-        })
+      submitExcel()
+      rejectExcel(done)
     })
   })
 
@@ -337,7 +329,6 @@ context('excel tests: ', function () {
 
   this.afterEach(() => {
     colorLog(`TEST END -------------`, '#3498DB')
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
   })
 })
 

client/cypress/e2e/filtering.cy.ts

@@ -15,9 +15,7 @@ context('filtering tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation, { timeout: longerCommandTimeout })
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
+
 
     visitPage('home')
   })
@@ -174,10 +172,6 @@ context('filtering tests: ', function () {
   //     })
   //   })
   // })
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const checkInfoBarIncludes = (text: string, callback: any) => {

client/cypress/e2e/licensing.cy.ts

@@ -23,15 +23,12 @@ interface EditConfigTableCells {
 
 context('licensing tests: ', function () {
   this.beforeAll(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
+
     cy.loginAndUpdateValidKey()
   })
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
   })
@@ -375,9 +372,7 @@ context('licensing tests: ', function () {
     })
   }
 
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
+
 })
 
 const logout = (callback?: any) => {

client/cypress/e2e/liveness.cy.ts

@@ -18,9 +18,6 @@ context('liveness tests: ', function () {
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
 
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
   })

client/cypress/e2e/viewbox.cy.ts

@@ -16,7 +16,6 @@ context('editor tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-
     cy.wait(2000)
 
     cy.get('body').then(($body) => {
@@ -393,10 +392,6 @@ context('editor tests: ', function () {
   //       }
   //     )
   // })
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const removeAllColumns = () => {

client/cypress/integration/editor.tests.ts

@@ -15,9 +15,6 @@ context('editor tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
   })
@@ -118,10 +115,6 @@ context('editor tests: ', function () {
       })
     })
   })
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const clickOnEdit = (callback?: any) => {

client/cypress/integration/excel.tests.ts

@@ -19,9 +19,6 @@ context('excel tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
 
@@ -339,7 +336,6 @@ context('excel tests: ', function () {
 
   this.afterEach(() => {
     colorLog(`TEST END -------------`, '#3498DB')
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
   })
 })
 

client/cypress/integration/filtering.tests.ts

@@ -15,9 +15,6 @@ context('filtering tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation, { timeout: longerCommandTimeout })
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
   })
@@ -173,10 +170,6 @@ context('filtering tests: ', function () {
       })
     })
   })
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const checkInfoBarIncludes = (text: string, callback: any) => {

client/cypress/integration/licensing.tests.ts

@@ -23,15 +23,11 @@ interface EditConfigTableCells {
 
 context('licensing tests: ', function () {
   this.beforeAll(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
     cy.loginAndUpdateValidKey()
   })
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
 
     visitPage('home')
   })
@@ -374,10 +370,6 @@ context('licensing tests: ', function () {
       })
     })
   }
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const logout = (callback?: any) => {

client/cypress/integration/liveness.tests.ts

@@ -18,10 +18,6 @@ context('liveness tests: ', function () {
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
 
-    // cy.get('input.username').type(username)
-    // cy.get('input.password').type(password)
-    // cy.get('.login-group button').click()
-
     visitPage('home')
   })
 

client/cypress/integration/viewbox.tests.ts

@@ -17,7 +17,6 @@ context('editor tests: ', function () {
 
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
-
     cy.wait(2000)
 
     cy.get('body').then(($body) => {
@@ -386,10 +385,6 @@ context('editor tests: ', function () {
       }
     )
   })
-
-  this.afterEach(() => {
-    // cy.visit(`${hostUrl}/SASLogon/logout`)
-  })
 })
 
 const checkColumns = (columns: string[], callback: () => void) => {

client/licenseChecker.js

@@ -10,7 +10,7 @@ const check = (cwd) => {
         onlyAllow:
           'AFLv2.1;Apache 2.0;Apache-2.0;Apache*;Artistic-2.0;0BSD;BSD*;BSD-2-Clause;BSD-3-Clause;CC0-1.0;CC-BY-3.0;CC-BY-4.0;ISC;MIT;MPL-2.0;ODC-By-1.0;Python-2.0;Unlicense;',
         excludePackages:
-          '@cds/city@1.1.0;@handsontable/angular@14.3.0;handsontable@14.3.0;hyperformula@2.7.0;jackspeak@2.2.0;path-scurry@1.7.0'
+          '@cds/city@1.1.0;@handsontable/angular@14.4.0;handsontable@14.4.0;hyperformula@2.7.0;jackspeak@2.2.0;path-scurry@1.7.0'
       },
       (error, json) => {
         if (error) {

client/package.json

@@ -53,7 +53,6 @@
     "@sasjs/utils": "^3.4.0",
     "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
     "@types/d3-graphviz": "^2.6.7",
-    "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
     "@types/text-encoding": "0.0.35",
     "base64-arraybuffer": "^0.2.0",
     "buffer": "^5.4.3",
@@ -81,6 +80,7 @@
     "tslib": "^2.3.0",
     "vm": "^0.1.0",
     "webpack": "^5.91.0",
+    "xlsx": "^0.18.5",
     "zone.js": "~0.14.4"
   },
   "devDependencies": {

client/src/app/_globals.ts

@@ -43,6 +43,10 @@ export interface XLMapListItem {
   targetDS: string
 }
 
+export interface HandsontableStaticConfig {
+  darkTableHeaderClass: string
+}
+
 /**
  * Cached filtering values across whole app (editor, viewer, viewboxes)
  * Cached lineage libraries, tables
@@ -62,6 +66,7 @@ export const globals: {
   viyaApi: any
   usernav: any
   operators: any
+  handsontable: HandsontableStaticConfig
   [key: string]: any
 } = {
   rootParam: <string>'',
@@ -140,5 +145,8 @@ export const globals: {
   operators: {
     numOperators: ['=', '<', '>', '<=', '>=', 'BETWEEN', 'IN', 'NOT IN', 'NE'],
     charOperators: ['=', '<', '>', '<=', '>=', 'CONTAINS', 'IN', 'NOT IN', 'NE']
+  },
+  handsontable: {
+    darkTableHeaderClass: 'darkTH'
   }
 }

client/src/app/app.component.html

@@ -245,6 +245,7 @@
 
   <app-alerts *ngIf="!errTop"></app-alerts>
   <app-requests-modal [(opened)]="requestsModal"></app-requests-modal>
+  <app-excel-password-modal></app-excel-password-modal>
 
   <!-- <app-terms *ngIf="showRegistration"></app-terms> -->
 

client/src/app/app.component.scss

@@ -173,6 +173,12 @@ header {
     }
   }
 
+  .btn-primary .btn, .btn.btn-primary {
+    &:disabled {
+      opacity: 0.65;
+    }
+  }
+
   .btn {
     cursor: pointer;
     display: inline-block;

client/src/app/app.component.ts

@@ -18,10 +18,20 @@ import {
   ClarityIcons,
   exclamationTriangleIcon,
   moonIcon,
-  sunIcon
+  processOnVmIcon,
+  sunIcon,
+  tableIcon,
+  trashIcon
 } from '@cds/core/icon'
 
-ClarityIcons.addIcons(moonIcon, sunIcon, exclamationTriangleIcon)
+ClarityIcons.addIcons(
+  moonIcon,
+  sunIcon,
+  exclamationTriangleIcon,
+  tableIcon,
+  trashIcon,
+  processOnVmIcon
+)
 
 @Component({
   selector: 'my-app',

client/src/app/deploy/sections/manual/manual.component.ts

@@ -1,6 +1,7 @@
 import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core'
 import SASjs, { SASjsConfig } from '@sasjs/adapter'
 import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 import { DeployService } from 'src/app/services/deploy.service'
 import { EventService } from 'src/app/services/event.service'
 import { LoggerService } from 'src/app/services/logger.service'
@@ -303,10 +304,10 @@ export class ManualComponent implements OnInit {
 
     this.sasService
       .request('public/startupservice', null)
-      .then((res: any) => {
-        this.loggerService.log(res)
+      .then((res: RequestWrapperResponse) => {
+        this.loggerService.log(res.adapterResponse)
 
-        if (res.saslibs) {
+        if (res.adapterResponse.saslibs) {
           this.validationState = 'success'
         } else {
           this.validationState = 'error'

client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts

@@ -3,6 +3,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'
 import SASjs, { SASjsConfig } from '@sasjs/adapter'
 import { ServerType } from '@sasjs/utils/types/serverType'
 import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 import { SASGroup } from 'src/app/models/sas/public-getgroups.model'
 import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model'
 import { SasService } from 'src/app/services/sas.service'
@@ -68,11 +69,11 @@ export class SasjsConfiguratorComponent implements OnInit {
     this.loading = true
 
     this.sasService.request('usernav/usergroupsbymember', null).then(
-      (res: any) => {
-        this.METAPERSON = res.MF_GETUSER
-        this.SYSUSERID = res.SYSUSERID
-        this.SYSHOSTNAME = res.SYSHOSTNAME
-        this.SYSVLONG = res.SYSVLONG
+      (res: RequestWrapperResponse) => {
+        this.METAPERSON = res.adapterResponse.MF_GETUSER
+        this.SYSUSERID = res.adapterResponse.SYSUSERID
+        this.SYSHOSTNAME = res.adapterResponse.SYSHOSTNAME
+        this.SYSVLONG = res.adapterResponse.SYSVLONG
 
         /*
           We would like to present a default DCPATH (deployment path) to the
@@ -88,12 +89,14 @@ export class SasjsConfiguratorComponent implements OnInit {
         */
         this.dcDirectory =
           this.tmpDirectories[
-            ['L', 'H', 'A', 'S'].includes(res.SYSSCPL.substring(0, 1))
+            ['L', 'H', 'A', 'S'].includes(
+              res.adapterResponse.SYSSCPL.substring(0, 1)
+            )
               ? 'linux'
               : 'windows'
           ]
 
-        this.dcAdminGroupList = res.groups
+        this.dcAdminGroupList = res.adapterResponse.groups
         this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME
 
         this.loading = false

client/src/app/editor/editor.component.html

@@ -3,7 +3,7 @@
     appFileDrop
     (fileOver)="fileOverBase($event)"
     [uploader]="uploader"
-    (fileDrop)="getFileDesc($event, true)"
+    (fileDrop)="attachFile($event, true)"
     [clrModalSize]="'xl'"
     [clrModalStaticBackdrop]="false"
     [clrModalClosable]="excelUploadState === 'Validating-DQ'"
@@ -81,7 +81,7 @@
               type="file"
               appFileSelect
               [uploader]="uploader"
-              (change)="getFileDesc($event)"
+              (change)="attachFile($event)"
             />
           </div>
 
@@ -92,7 +92,7 @@
             <button
               [disabled]="true"
               class="btnView btn btn-sm btn-success profile-buttons w-100"
-              (click)="getFile()"
+              (click)="uploadParsedFiles()"
             >
               Upload
             </button>
@@ -164,22 +164,28 @@
       <div
         class="card-header clr-row buttonBar headerBar clr-flex-md-row clr-justify-content-center clr-justify-content-lg-end"
       >
-        <div *ngIf="tableTrue" class="clr-col-12 clr-col-lg-4 backBtn">
-          <span class="btn btn-sm btn-icon btn-dimmed" [routerLink]="['/home']">
+        <div
+          *ngIf="tableTrue"
+          class="clr-col-12 clr-col-md-3 clr-col-lg-4 backBtn"
+        >
+          <span
+            class="btn icon-collapse btn-sm btn-icon btn-dimmed"
+            [routerLink]="['/home']"
+          >
             <clr-icon shape="caret" dir="left" size="20"></clr-icon>
-            Back to table selection
+            <span class="text">Back to table selection</span>
           </span>
           <span
             (click)="viewboxManager()"
-            class="btn btn-sm btn-icon btn-dimmed viewbox-open"
+            class="btn icon-collapse btn-sm btn-icon btn-dimmed viewbox-open"
           >
             <clr-icon shape="view-cards" size="20"></clr-icon>
-            Viewboxes
+            <span class="text">Viewboxes</span>
           </span>
         </div>
 
         <div
-          class="clr-col-12 clr-col-lg-4 d-flex flex-column align-items-center"
+          class="clr-col-12 clr-col-md-5 clr-col-lg-4 d-flex flex-column align-items-center"
           [class.clr-col-lg-12]="!tableTrue"
         >
           <h4
@@ -234,34 +240,37 @@
             </ng-container>
           </h4>
         </div>
-        <div *ngIf="tableTrue" class="clr-col-12 clr-col-lg-4 btnCtrl">
+        <div
+          *ngIf="tableTrue"
+          class="clr-col-12 clr-col-md-4 clr-col-lg-4 btnCtrl"
+        >
           <ng-container *ngIf="hotTable.readOnly && !uploadPreview">
             <button
               type="button"
-              class="btnView btn btn-sm btn-icon btn-block btn-dimmed"
+              class="btnView btn icon-collapse btn-sm btn-icon btn-block btn-dimmed"
               (click)="openQb()"
             >
               <clr-icon shape="filter"></clr-icon>
-              <span>Filter</span>
+              <span class="text">Filter</span>
             </button>
 
             <button
               type="button"
-              class="btn btn-sm btn-primary btn-block"
+              class="btn icon-collapse btn-sm btn-primary btn-block"
               (click)="editTable()"
             >
               <clr-icon shape="note"></clr-icon>
-              <span>Edit</span>
+              <span class="text">Edit</span>
             </button>
 
             <button
               *ngIf="!columnLevelSecurityFlag"
               (click)="onShowUploadModal()"
               type="button"
-              class="btn btn-sm btn-success btn-block mr-0"
+              class="btn icon-collapse btn-sm btn-success btn-block mr-0"
             >
               <clr-icon shape="upload"></clr-icon>
-              <span>Upload</span>
+              <span class="text">Upload</span>
             </button>
           </ng-container>
 
@@ -567,7 +576,7 @@
               <button
                 type="button"
                 class="btn btn-sm btn-primary"
-                (click)="getFile(); submitLimitNotice = false"
+                (click)="uploadParsedFiles(); submitLimitNotice = false"
               >
                 Submit
               </button>

client/src/app/editor/editor.component.scss

@@ -201,6 +201,7 @@ hot-table {
 
   display: flex;
   justify-content: center;
+  align-items: flex-start;
 
   margin: 1px;
 
@@ -211,7 +212,10 @@ hot-table {
   span {
     font-size: 20px;
     margin-top: 20px;
-    color: #fff;
+    padding: 10px;
+    background: #dbdbdb;
+    border-radius: 5px;
+    color: black;
   }
 }
 
@@ -223,6 +227,16 @@ hot-table {
   font-size: inherit !important;
 }
 
+// When width is smaller remove the text from the buttons
+// keep only the icons
+@media (max-width: 992px) {
+  .icon-collapse {
+    .text {
+      display: none;
+    }
+  }
+}
+
 // FIXME
 // Let's leave it here for a reference if there
 // is an issue with viewboxes/filter modal overlaying

client/src/app/group/group.component.ts

@@ -6,6 +6,7 @@ import { ServerType } from '@sasjs/utils/types/serverType'
 import { HelperService } from '../services/helper.service'
 import { SasService } from '../services/sas.service'
 import { globals } from '../_globals'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-group',
@@ -82,11 +83,13 @@ export class GroupComponent implements OnInit {
               globals.usernav.groupList = groups
             })
         } else {
-          this.sasService.request('public/getgroups', null).then((res: any) => {
-            this.loading = false
-            this.groups = res.groups
-            globals.usernav.groupList = res.groups
-          })
+          this.sasService
+            .request('public/getgroups', null)
+            .then((res: RequestWrapperResponse) => {
+              this.loading = false
+              this.groups = res.adapterResponse.groups
+              globals.usernav.groupList = res.adapterResponse.groups
+            })
         }
       } else {
         this.groups = globals.usernav.groupList
@@ -128,14 +131,15 @@ export class GroupComponent implements OnInit {
           let data = { iwant: [{ groupid: this.paramURI }] }
           this.sasService
             .request('usernav/usermembersbygroup', data)
-            .then((res: any) => {
-              this.groupMembers = res.sasmembers
-              this.groupMemberCount = res.sasmembers.length
-              if (res.sasmembers[0] !== undefined) {
+            .then((res: RequestWrapperResponse) => {
+              this.groupMembers = res.adapterResponse.sasmembers
+              this.groupMemberCount = res.adapterResponse.sasmembers.length
+              if (res.adapterResponse.sasmembers[0] !== undefined) {
                 this.loading = false
-                this.groupUri = res.sasmembers[0].URIMEM || this.paramURI
-                this.groupName = res.sasmembers[0].GROUPNAME
-                this.groupDesc = res.sasmembers[0].GROUPDESC
+                this.groupUri =
+                  res.adapterResponse.sasmembers[0].URIMEM || this.paramURI
+                this.groupName = res.adapterResponse.sasmembers[0].GROUPNAME
+                this.groupDesc = res.adapterResponse.sasmembers[0].GROUPDESC
 
                 if (!this.groupName) {
                   this.groupName = this.paramURI
@@ -202,13 +206,13 @@ export class GroupComponent implements OnInit {
 
       this.sasService
         .request('usernav/usermembersbygroup', data)
-        .then((res: any) => {
+        .then((res: RequestWrapperResponse) => {
           this.loading = false
           this.groupUri = group.GROUPURI
           this.groupName = group.GROUPNAME
           this.groupDesc = group.GROUPDESC
-          this.groupMembers = res.sasmembers
-          this.groupMemberCount = res.sasmembers.length
+          this.groupMembers = res.adapterResponse.sasmembers
+          this.groupMemberCount = res.adapterResponse.sasmembers.length
         })
     }
   }

client/src/app/home/home-routing.module.ts

@@ -3,6 +3,7 @@ import { RouterModule, Routes } from '@angular/router'
 import { HomeRouteComponent } from '../routes/home-route/home-route.component'
 import { HomeComponent } from './home.component'
 import { XLMapModule } from '../xlmap/xlmap.module'
+import { MultiDatasetModule } from '../multi-dataset/multi-dataset.module'
 
 const routes: Routes = [
   {
@@ -11,7 +12,8 @@ const routes: Routes = [
     children: [
       { path: '', pathMatch: 'full', redirectTo: 'tables' },
       { path: 'tables', component: HomeComponent },
-      { path: 'files', loadChildren: () => XLMapModule }
+      { path: 'excel-maps', loadChildren: () => XLMapModule },
+      { path: 'multi-load', loadChildren: () => MultiDatasetModule }
     ]
   }
 ]

client/src/app/licensing/licensing.component.ts

@@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'
 import { ActivatedRoute, Router } from '@angular/router'
 import { AppService, LicenceService, SasService } from '../services'
 import { LicenseKeyData } from '../models/LicenseKeyData'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 enum LicenseActions {
   key = 'key',
@@ -116,8 +117,12 @@ export class LicensingComponent implements OnInit {
 
     this.sasService
       .request('admin/registerkey', table)
-      .then((res: any) => {
-        if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
+      .then((res: RequestWrapperResponse) => {
+        if (
+          res.adapterResponse.return &&
+          res.adapterResponse.return[0] &&
+          res.adapterResponse.return[0].MSG === 'SUCCESS'
+        ) {
           location.replace(location.href.split('#')[0])
         }
       })

client/src/app/lineage/lineage.component.ts

@@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
 import * as saveSvg from 'save-svg-as-png'
 import { LoggerService } from '../services/logger.service'
 import { LicenceService } from '../services/licence.service'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 const moment = require('moment')
 
 @Component({
@@ -115,8 +116,8 @@ export class LineageComponent {
 
     await this.sasService
       .request('lineage/getmetacols', libTable)
-      .then((res: any) => {
-        this.columnsList = res.metacols
+      .then((res: RequestWrapperResponse) => {
+        this.columnsList = res.adapterResponse.metacols
         if (this.columnsList && this.columnsList.length > 0) {
           // this.column = this.columnsList[0]['COLURI']
 
@@ -174,8 +175,8 @@ export class LineageComponent {
     let libTable = { SASControlTable: [{ liburi: $event }] }
     await this.sasService
       .request('lineage/getmetatables', libTable)
-      .then((res: any) => {
-        this.tablesList = res.metatables
+      .then((res: RequestWrapperResponse) => {
+        this.tablesList = res.adapterResponse.metatables
 
         if (this.tablesList && this.tablesList.length > 0) {
           library['tables'] = this.tablesList
@@ -295,8 +296,8 @@ export class LineageComponent {
       } else {
         await this.sasService
           .request('public/viewlibs', null)
-          .then((res: any) => {
-            this.libraryList = res.saslibs
+          .then((res: RequestWrapperResponse) => {
+            this.libraryList = res.adapterResponse.saslibs
             this.helperService.displayLibraries(this.libraryList)
 
             if (this.libraryList) {
@@ -402,8 +403,8 @@ export class LineageComponent {
     return new Promise<void>((resolve, reject) => {
       this.sasService
         .request('lineage/fetchtablelineage', libTable)
-        .then(async (res: any) => {
-          if (res.flatdata.length > 0) {
+        .then(async (res: RequestWrapperResponse) => {
+          if (res.adapterResponse.flatdata.length > 0) {
             if (this.licenceService.checkLineageLimit()) {
               this.eventService.showInfoModal(
                 'Notice',
@@ -421,20 +422,22 @@ export class LineageComponent {
           }
 
           this.lineageTableName =
-            res.info[0].LIBREF + '.' + res.info[0].TABLENAME
+            res.adapterResponse.info[0].LIBREF +
+            '.' +
+            res.adapterResponse.info[0].TABLENAME
 
-          let dotArray = res.finalfinal
+          let dotArray = res.adapterResponse.finalfinal
           let vizTmp: string = ''
 
           for (let i = 0; i < dotArray.length; i++) {
             vizTmp += unescape(dotArray[i].LINE) + '\n'
           }
 
-          this.flatdata = res.flatdata
+          this.flatdata = res.adapterResponse.flatdata
 
           if (this.libraryList) {
             let libraryToSelect = this.libraryList.find((library: any) =>
-              res.info[0].LIBURI.toUpperCase().includes(
+              res.adapterResponse.info[0].LIBURI.toUpperCase().includes(
                 library.LIBRARYID.toUpperCase()
               )
             )
@@ -450,7 +453,7 @@ export class LineageComponent {
               if (libraryToSelect['tables']) {
                 tableToSelect = libraryToSelect['tables'].find((table: any) =>
                   table.TABLEURI.toUpperCase().includes(
-                    res.info[0].TABLEID.toUpperCase()
+                    res.adapterResponse.info[0].TABLEID.toUpperCase()
                   )
                 )
 
@@ -495,10 +498,10 @@ export class LineageComponent {
             .replace(/\sds:/g, '\nds:')
             .replace(/\s\n/g, '\n')
 
-          this.idlookup = res.idlookup
+          this.idlookup = res.adapterResponse.idlookup
 
-          if (res.finalfinal.length > this.largeDotFileLimit) {
-            this.largeDotFileLines = res.finalfinal.length
+          if (res.adapterResponse.finalfinal.length > this.largeDotFileLimit) {
+            this.largeDotFileLines = res.adapterResponse.finalfinal.length
           } else {
             this.buildGraph()
           }
@@ -619,8 +622,8 @@ export class LineageComponent {
     return new Promise<void>((resolve, reject) => {
       this.sasService
         .request('lineage/fetchcollineage', libTable)
-        .then(async (res: any) => {
-          if (res.flatdata.length > 0) {
+        .then(async (res: RequestWrapperResponse) => {
+          if (res.adapterResponse.flatdata.length > 0) {
             if (this.licenceService.checkLineageLimit()) {
               this.eventService.showInfoModal(
                 'Notice',
@@ -631,18 +634,21 @@ export class LineageComponent {
             }
           }
 
-          if (typeof res === 'string') {
+          if (typeof res.adapterResponse === 'string') {
             this.vizInput = 'digraph G {SAS Error}'
             this.buildGraph()
             return
           }
 
-          this.lineageTableName = res.info[0].LIBREF + '.' + res.info[0].TABNAME
-          this.lineageColumnName = res.info[0].COLNAME
+          this.lineageTableName =
+            res.adapterResponse.info[0].LIBREF +
+            '.' +
+            res.adapterResponse.info[0].TABNAME
+          this.lineageColumnName = res.adapterResponse.info[0].COLNAME
 
-          this.idlookup = res.idlookup
+          this.idlookup = res.adapterResponse.idlookup
 
-          let dotArray = res.fromsas
+          let dotArray = res.adapterResponse.fromsas
           let vizTmp: string = ''
           for (let i = 0; i < dotArray.length; i++) {
             vizTmp += unescape(dotArray[i].STRING) + '\n'
@@ -653,11 +659,11 @@ export class LineageComponent {
             .replace(/\sds:/g, '\nds:')
             .replace(/\s\n/g, '\n')
 
-          this.flatdata = res.flatdata
+          this.flatdata = res.adapterResponse.flatdata
 
           if (this.libraryList) {
             let libraryToSelect = this.libraryList.find((library: any) =>
-              res.info[0]?.LIBURI?.toUpperCase()?.includes(
+              res.adapterResponse.info[0]?.LIBURI?.toUpperCase()?.includes(
                 library?.LIBRARYID?.toUpperCase()
               )
             )
@@ -672,7 +678,8 @@ export class LineageComponent {
 
               if (libraryToSelect['tables']) {
                 tableToSelect = libraryToSelect['tables'].find(
-                  (table: any) => table.TABLEURI === res.info[0].TABURI
+                  (table: any) =>
+                    table.TABLEURI === res.adapterResponse.info[0].TABURI
                 )
 
                 if (tableToSelect) {
@@ -714,8 +721,8 @@ export class LineageComponent {
             }
           }
 
-          if (res.fromsas.length > this.largeDotFileLimit) {
-            this.largeDotFileLines = res.fromsas.length
+          if (res.adapterResponse.fromsas.length > this.largeDotFileLimit) {
+            this.largeDotFileLines = res.adapterResponse.fromsas.length
           } else {
             this.buildGraph()
           }

client/src/app/metadata/metadata.component.ts

@@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
 import { globals } from '../_globals'
 
 import { Injectable } from '@angular/core'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 interface MetaData {
   NAME: any
@@ -109,44 +110,52 @@ export class MetadataComponent implements OnInit {
       this.metatypesLoading = false
       this.metaDataSearch = globals.metadata.metaDataSearch
     } else {
-      this.sasService.request('metanav/metatypes', null).then((res: any) => {
-        this.metaDataList = res.types
-        globals.metadata.metaDataList = this.metaDataList
-        this.loading = false
-        this.metatypesLoading = false
-      })
+      this.sasService
+        .request('metanav/metatypes', null)
+        .then((res: RequestWrapperResponse) => {
+          this.metaDataList = res.adapterResponse.types
+          globals.metadata.metaDataList = this.metaDataList
+          this.loading = false
+          this.metatypesLoading = false
+        })
 
-      this.sasService.request('metanav/metarepos', null).then((res: any) => {
-        let foundation = false
-        this.repositories = []
-        for (let index = 0; index < res.outrepos.length; index++) {
-          this.repositories.push(res.outrepos[index].NAME)
-          if (res.outrepos[index].NAME === 'Foundation') {
-            foundation = true
+      this.sasService
+        .request('metanav/metarepos', null)
+        .then((res: RequestWrapperResponse) => {
+          let foundation = false
+          this.repositories = []
+          for (
+            let index = 0;
+            index < res.adapterResponse.outrepos.length;
+            index++
+          ) {
+            this.repositories.push(res.adapterResponse.outrepos[index].NAME)
+            if (res.adapterResponse.outrepos[index].NAME === 'Foundation') {
+              foundation = true
+            }
           }
-        }
-        if (foundation) {
-          this.repository = 'Foundation'
-        } else {
-          this.repository = res.outrepos[0].NAME
-        }
-        globals.metadata.metaRepositories = this.repositories
-        globals.metadata.selectedRepository = this.repository
-        if (this.objectRoute) {
-          this.eventService.closeSidebar()
-          this.showData = true
-          let name = ''
-          let id = this.route.snapshot.params['objectID']
-          // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
-          this.metaObjectList = []
-          this.metaObjectList.push({ ID: id, NAME: name })
-          this.metaObjectShowList = this.metaObjectList
-          this.metaObjectOnClick(
-            this.metaObjectShowList[0].ID,
-            this.metaObjectShowList[0]
-          )
-        }
-      })
+          if (foundation) {
+            this.repository = 'Foundation'
+          } else {
+            this.repository = res.adapterResponse.outrepos[0].NAME
+          }
+          globals.metadata.metaRepositories = this.repositories
+          globals.metadata.selectedRepository = this.repository
+          if (this.objectRoute) {
+            this.eventService.closeSidebar()
+            this.showData = true
+            let name = ''
+            let id = this.route.snapshot.params['objectID']
+            // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
+            this.metaObjectList = []
+            this.metaObjectList.push({ ID: id, NAME: name })
+            this.metaObjectShowList = this.metaObjectList
+            this.metaObjectOnClick(
+              this.metaObjectShowList[0].ID,
+              this.metaObjectShowList[0]
+            )
+          }
+        })
     }
   }
 
@@ -183,56 +192,64 @@ export class MetadataComponent implements OnInit {
     const data: any = {
       SASControlTable: [{ metatype: $event, repo: this.repository }]
     }
-    this.sasService.request('metanav/metaobjects', data).then((res: any) => {
-      this.metaObjectList = res.objects
-      this.getMetaObjectAttributes(this.metaObjectSize)
-      this.loading = false
-      this.assoTypeSelected = $event
-      this.eventService.closeSidebar()
-      this.showData = true
-    })
+    this.sasService
+      .request('metanav/metaobjects', data)
+      .then((res: RequestWrapperResponse) => {
+        this.metaObjectList = res.adapterResponse.objects
+        this.getMetaObjectAttributes(this.metaObjectSize)
+        this.loading = false
+        this.assoTypeSelected = $event
+        this.eventService.closeSidebar()
+        this.showData = true
+      })
   }
 
   public async selectmetaObject($event: any, metaData?: any) {
     let data: any = {
       SASControlTable: [{ objecturi: $event }]
     }
-    this.sasService.request('metanav/metadetails', data).then((res: any) => {
-      this.metaObjectAssociations = res.associations
-      this.root$ = of(this.getAssosiationsCount(res.associations))
-      this.showAcc = true
-      this.showTable = true
-      let metaObjectName = res.attributes.find(
-        (x: any) => x.NAME === 'Name'
-      ).VALUE
-      this.assoObjectSelected = metaObjectName
-      metaData.NAME = metaObjectName
-      let url = this.router.url
-      if (this.objectRoute) {
-        // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
-        this.location.replaceState(
-          url.slice(0, url.lastIndexOf('object')) +
-            'object/' +
-            $event.slice(1 + $event.indexOf('\\'))
+    this.sasService
+      .request('metanav/metadetails', data)
+      .then((res: RequestWrapperResponse) => {
+        this.metaObjectAssociations = res.adapterResponse.associations
+        this.root$ = of(
+          this.getAssosiationsCount(res.adapterResponse.associations)
         )
-      } else {
-        // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
-        this.location.replaceState(
-          url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
-        )
-      }
-      this.metaObjectAttributes = res.attributes
-    })
+        this.showAcc = true
+        this.showTable = true
+        let metaObjectName = res.adapterResponse.attributes.find(
+          (x: any) => x.NAME === 'Name'
+        ).VALUE
+        this.assoObjectSelected = metaObjectName
+        metaData.NAME = metaObjectName
+        let url = this.router.url
+        if (this.objectRoute) {
+          // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+          this.location.replaceState(
+            url.slice(0, url.lastIndexOf('object')) +
+              'object/' +
+              $event.slice(1 + $event.indexOf('\\'))
+          )
+        } else {
+          // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+          this.location.replaceState(
+            url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
+          )
+        }
+        this.metaObjectAttributes = res.adapterResponse.attributes
+      })
   }
 
   public async selectAssosiationsDetails($event: any, metaData?: any) {
     let data: any = {
       SASControlTable: [{ objecturi: $event }]
     }
-    this.sasService.request('metanav/metadetails', data).then((res: any) => {
-      this.metaObjectAttributes = res.attributes
-      this.showTable = true
-    })
+    this.sasService
+      .request('metanav/metadetails', data)
+      .then((res: RequestWrapperResponse) => {
+        this.metaObjectAttributes = res.adapterResponse.attributes
+        this.showTable = true
+      })
   }
 
   public getAssosiationsCount(assosiationList: Array<any>) {
@@ -244,7 +261,7 @@ export class MetadataComponent implements OnInit {
           details: []
         })
       }
-      let assocObj = assosiationsHash.get(assosiation.ASSOC)
+      let assocObj: any = assosiationsHash.get(assosiation.ASSOC)
       assocObj.count++
       assocObj.details.push({
         ASSOCURI: assosiation.ASSOCURI,
@@ -254,7 +271,7 @@ export class MetadataComponent implements OnInit {
       })
     }
     let assocGrouped: Array<any> = []
-    assosiationsHash.forEach(function (val, key) {
+    assosiationsHash.forEach(function (val: any, key) {
       assocGrouped.push({
         ASSOC: key,
         count: val.count,
@@ -294,9 +311,9 @@ export class MetadataComponent implements OnInit {
     }
     return this.sasService
       .request('metanav/metadetails', data)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
         this.showTable = true
-        this.metaObjectAttributes = res.attributes
+        this.metaObjectAttributes = res.adapterResponse.attributes
         this.assoObjectSelected = asso.NAME
         let url = this.router.url
         if (this.objectRoute) {
@@ -314,7 +331,7 @@ export class MetadataComponent implements OnInit {
               asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\'))
           )
         }
-        return this.getAssosiationsCount(res.associations)
+        return this.getAssosiationsCount(res.adapterResponse.associations)
       })
   }
 

client/src/app/models/FileUploadEncoding.ts

@@ -0,0 +1 @@
+export type FileUploadEncoding = 'UTF-8' | 'WLATIN1'

client/src/app/models/ParseParams.interface.ts

@@ -0,0 +1,29 @@
+import { DcValidator } from '../shared/dc-validator/dc-validator'
+import { FileUploadEncoding } from './FileUploadEncoding'
+import { FileUploader } from './FileUploader.class'
+import { ExcelRule } from './TableData'
+import XLSX from 'xlsx'
+
+export interface ParseParams {
+  file: File
+  password?: string
+  dcValidator: DcValidator
+  /**
+   * If workbook is provided, parse function will not run a XLSX.read()
+   * it will use this property instead. So the client must do a file read beforehand
+   */
+  workbook?: XLSX.WorkBook
+  /**
+   * Parse function will manipulate and return the uploader array which can be provided with files already in the queue
+   * Otherwise new empty instance will be created.
+   */
+  uploader?: FileUploader
+  headerPks: string[]
+  headerArray: string[]
+  headerShow: string[]
+  timeHeaders: string[]
+  dateHeaders: string[]
+  dateTimeHeaders: string[]
+  xlRules: ExcelRule[]
+  encoding?: FileUploadEncoding
+}

client/src/app/models/ParseResult.interface.ts

@@ -0,0 +1,15 @@
+import { FileUploader } from './FileUploader.class'
+import FoundRangeInfo from './RangeInfo'
+
+export interface ParseResult {
+  /**
+   * In case of CSV file, won't be returned
+   */
+  data?: any[]
+  /**
+   * In case of CSV file, won't be returned
+   */
+  headerShow?: string[]
+  rangeSheetRes?: FoundRangeInfo
+  uploader: FileUploader
+}

client/src/app/models/RangeInfo.ts

@@ -0,0 +1,13 @@
+export default interface FoundRangeInfo {
+  found: boolean
+  sheetName: string
+  rangeStartAddress: string
+  rangeEndAddress: string
+  rangeAddress: string
+  missingHeaders: MissingHeaders[]
+}
+
+export interface MissingHeaders {
+  sheetName: string
+  missingHeaders: string[]
+}

client/src/app/models/SearchDataExcelResult.interface.ts

@@ -0,0 +1,13 @@
+import { MissingHeaders } from './RangeInfo'
+
+export interface SearchDataExcelResult {
+  missing?: MissingHeaders[]
+  found?: {
+    data: any
+    arrayData: any[]
+    sheetName: string
+    headers: string[]
+    startAddress?: string
+    endAddress?: string
+  }
+}

client/src/app/models/SheetInfo.ts

@@ -7,4 +7,5 @@ export default interface SheetInfo {
   missingHeaders: string[]
   rangeStartRow: number
   rangeStartCol: number
+  rangeAddress?: string
 }

client/src/app/models/UploadFile.ts

@@ -0,0 +1,4 @@
+export interface UploadFileResponse {
+  adapterResponse: any
+  log?: string
+}

client/src/app/models/request-wrapper/RequestWrapperResponse.ts

@@ -0,0 +1,4 @@
+export interface RequestWrapperResponse<responseType = any> {
+  adapterResponse: responseType
+  log?: string
+}

client/src/app/models/sas/editors-stagedata.model.ts

@@ -0,0 +1,34 @@
+import { BaseSASResponse } from './common/BaseSASResponse'
+
+export interface EditorsStageDataSASResponse extends BaseSASResponse {
+  SYSDATE: string
+  SYSTIME: string
+  sasparams: Sasparam[]
+  _DEBUG: string
+  _PROGRAM: string
+  AUTOEXEC: string
+  MF_GETUSER: string
+  SYSCC: string
+  SYSENCODING: string
+  SYSERRORTEXT: string
+  SYSHOSTINFOLONG: string
+  SYSHOSTNAME: string
+  SYSPROCESSID: string
+  SYSPROCESSMODE: string
+  SYSPROCESSNAME: string
+  SYSJOBID: string
+  SYSSCPL: string
+  SYSSITE: string
+  SYSTCPIPHOSTNAME: string
+  SYSUSERID: string
+  SYSVLONG: string
+  SYSWARNINGTEXT: string
+  END_DTTM: string
+  MEMSIZE: string
+}
+
+export interface Sasparam {
+  STATUS: string | 'SUCCESS'
+  DSID: string
+  URL: string
+}

client/src/app/multi-dataset/multi-dataset-routing.module.ts

@@ -0,0 +1,19 @@
+import { NgModule } from '@angular/core'
+import { RouterModule, Routes } from '@angular/router'
+
+import { MultiDatasetRouteComponent } from '../routes/multi-dataset-route/multi-dataset-route.component'
+import { MultiDatasetComponent } from './multi-dataset.component'
+
+const routes: Routes = [
+  {
+    path: '',
+    component: MultiDatasetRouteComponent,
+    children: [{ path: '', component: MultiDatasetComponent }]
+  }
+]
+
+@NgModule({
+  imports: [RouterModule.forChild(routes)],
+  exports: [RouterModule]
+})
+export class MultiDatasetRoutingModule {}

client/src/app/multi-dataset/multi-dataset.component.html

@@ -0,0 +1,543 @@
+<app-sidebar>
+  <div *ngIf="datasetsLoading" class="my-10-mx-auto text-center">
+    <clr-spinner clrMedium></clr-spinner>
+  </div>
+
+  <div *ngIf="!parsedDatasets.length" class="text-center mb-10">
+    <button
+      (click)="fileUploadInput.click()"
+      id="browse-file"
+      class="btn btn-primary btn-sm"
+      [disabled]="selectedFile !== null || submittingCsv"
+    >
+      Browse file
+    </button>
+    <input
+      hidden
+      #fileUploadInput
+      id="file-upload"
+      type="file"
+      (change)="onFileChange($event)"
+      multiple
+    />
+  </div>
+
+  <ng-container *ngIf="parsedDatasets.length && !submittedCsvDatasets.length">
+    <div *ngIf="!excelsSubmitted" class="text-center mb-10">
+      <button (click)="onDiscard()" class="btn btn-danger btn-sm mr-10">
+        Discard
+      </button>
+      <button
+        (click)="onSubmitAll()"
+        id="submit-all"
+        class="btn btn-primary btn-sm"
+      >
+        Submit All
+      </button>
+    </div>
+
+    <p cds-text="caption" class="ml-10 mb-10">Found tables:</p>
+    <clr-tree>
+      <clr-tree-node *ngFor="let dataset of parsedDatasets">
+        <button
+          (click)="onParsedDatasetClick(dataset)"
+          class="clr-treenode-link whitespace-nowrap d-flex clr-align-items-center"
+          [class.active]="dataset.active"
+        >
+          <ng-container *ngIf="dataset.submitResult">
+            <cds-icon
+              *ngIf="dataset.submitResult.error"
+              status="danger"
+              shape="exclamation-circle"
+            ></cds-icon>
+            <cds-icon
+              *ngIf="dataset.submitResult.success"
+              status="success"
+              shape="check-circle"
+            ></cds-icon>
+          </ng-container>
+
+          <ng-container *ngIf="!dataset.submitResult">
+            <ng-container *ngIf="dataset.datasource">
+              <cds-icon
+                *ngIf="!(dataset.datasource.length && dataset.parseResult)"
+                status="danger"
+                shape="exclamation-circle"
+              ></cds-icon>
+              <cds-icon
+                *ngIf="dataset.datasource.length && dataset.parseResult"
+                shape="table"
+              ></cds-icon>
+            </ng-container>
+
+            <ng-container *ngIf="!dataset.datasource">
+              <cds-icon *ngIf="!dataset.parsingTable" shape="table"></cds-icon>
+
+              <clr-spinner *ngIf="dataset.parsingTable" clrSmall></clr-spinner>
+            </ng-container>
+          </ng-container>
+
+          <span class="ml-5"> {{ dataset.libds }} </span>
+        </button>
+      </clr-tree-node>
+    </clr-tree>
+  </ng-container>
+
+  <ng-container *ngIf="submittedCsvDatasets.length">
+    <p cds-text="caption" class="ml-10 mb-10 mt-10">Submitted tables:</p>
+    <clr-tree>
+      <clr-tree-node *ngFor="let dataset of submittedCsvDatasets">
+        <button
+          (click)="onSubmittedCsvDatasetClick(dataset)"
+          class="clr-treenode-link whitespace-nowrap"
+          [class.active]="dataset.active"
+        >
+          <cds-icon
+            *ngIf="dataset.error"
+            status="danger"
+            shape="exclamation-circle"
+          ></cds-icon>
+          <cds-icon
+            *ngIf="dataset.success"
+            status="success"
+            shape="check-circle"
+          ></cds-icon>
+          <cds-icon shape="table"></cds-icon>
+          {{ dataset.libds }}
+        </button>
+      </clr-tree-node>
+    </clr-tree>
+  </ng-container>
+
+  <!-- <div *ngIf="librariesPaging" class="w-100 text-center">
+    <span class="spinner spinner-sm"> Loading... </span>
+  </div> -->
+</app-sidebar>
+
+<div #contentArea class="content-area">
+  <div class="card no-borders h-100 d-flex clr-flex-column">
+    <div
+      class="header-row clr-row justify-content-between clr-justify-content-center w-100 m-0"
+    >
+      <p cds-text="section">Multi Dataset Load</p>
+    </div>
+
+    <div
+      *ngIf="selectedFile === null && !submittingCsv"
+      class="no-table-selected pointer-events-none"
+    >
+      <clr-icon
+        shape="warning-standard"
+        size="40"
+        class="is-info icon-dc-fill"
+      ></clr-icon>
+      <p class="text-center color-gray mt-10" cds-text="section">
+        Please upload a file
+      </p>
+    </div>
+
+    <ng-container *ngIf="selectedFile !== null || submittingCsv">
+      <ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
+        <div class="d-flex clr-justify-content-center mt-15">
+          <div class="dataset-input-wrapper">
+            <p cds-text="secondary regular" class="mb-5">
+              Selected file: <strong>{{ selectedFile.name }}</strong>
+              <clr-tooltip>
+                <cds-icon
+                  clrTooltipTrigger
+                  (click)="onDiscardFile()"
+                  shape="trash"
+                  status="danger"
+                  class="ml-5 cursor-pointer"
+                ></cds-icon>
+                <clr-tooltip-content> Discard the file </clr-tooltip-content>
+              </clr-tooltip>
+            </p>
+            <p cds-text="secondary regular" class="mb-20">
+              File size: <strong>{{ selectedFile.sizeMB }} MB</strong>
+            </p>
+            <p cds-text="secondary regular" class="mb-15">
+              Paste or type the list of datasets to upload:
+            </p>
+
+            <clr-control-helper class="mb-5"
+              >Each row is one dataset. We will automatically detect tables by
+              the sheetname and populate if any.</clr-control-helper
+            >
+
+            <hot-table
+              hotId="hotInstanceUserDataset"
+              id="hotTableUserDataset"
+              class="mt-15"
+              [afterGetColHeader]="afterGetColHeader"
+              [settings]="hotUserDatasets"
+              [licenseKey]="hotTableLicenseKey"
+              stretchH="all"
+            >
+            </hot-table>
+
+            <div class="dataset-selection-actions text-right mt-10">
+              <button
+                (click)="onStartParsingFile()"
+                id="continue-btn"
+                class="btn btn-primary btn-sm"
+                [disabled]="!matchedDatasets.length"
+                [clrLoading]="uploadLoading"
+              >
+                Continue
+              </button>
+            </div>
+          </div>
+        </div>
+      </ng-container>
+
+      <ng-container
+        *ngIf="parsedDatasets.length && !submittedCsvDatasets.length"
+      >
+        <div
+          *ngIf="!activeParsedDataset"
+          class="no-table-selected pointer-events-none"
+        >
+          <ng-container *ngIf="fileLoadingState !== FileLoadingState.parsed">
+            <clr-icon
+              shape="process-on-vm"
+              size="40"
+              class="is-info icon-dc-fill"
+            ></clr-icon>
+
+            <p class="text-center color-gray mt-10" cds-text="section">
+              {{ fileLoadingState }}...
+            </p>
+          </ng-container>
+
+          <ng-container *ngIf="fileLoadingState === FileLoadingState.parsed">
+            <clr-icon
+              shape="warning-standard"
+              size="40"
+              class="is-info icon-dc-fill"
+            ></clr-icon>
+            <p class="text-center color-gray mt-10" cds-text="section">
+              Please select a dataset on the left to review the data
+            </p>
+          </ng-container>
+        </div>
+
+        <ng-container *ngIf="activeParsedDataset">
+          <div
+            *ngIf="activeParsedDataset.submitResult"
+            class="d-flex clr-justify-content-between p-10 mt-15 submission-results"
+          >
+            <div>
+              <p cds-text="secondary regular" class="mb-10">
+                Submit Status:
+                <span
+                  *ngIf="activeParsedDataset.submitResult?.success"
+                  class="color-green"
+                  ><strong>SUCCESS</strong></span
+                >
+                <span
+                  *ngIf="activeParsedDataset.submitResult?.error"
+                  class="color-red"
+                  ><strong>ERROR</strong></span
+                >
+              </p>
+              <p
+                *ngIf="activeParsedDataset.submitResult?.error"
+                cds-text="secondary regular"
+              >
+                Error details:
+              </p>
+            </div>
+
+            <div>
+              <button
+                *ngIf="
+                  !submittingCsv && activeParsedDataset.submitResult?.error
+                "
+                (click)="reSubmitTable(activeParsedDataset)"
+                class="btn btn-primary mt-10"
+                [clrLoading]="submitLoading"
+              >
+                Resubmit
+              </button>
+              <button
+                (click)="
+                  downloadFile(
+                    activeParsedDataset.submitResult.log ||
+                      activeParsedDataset.submitResult.success ||
+                      activeParsedDataset.submitResult.error
+                  )
+                "
+                class="btn btn-primary-outline mt-10"
+              >
+                Download log
+              </button>
+            </div>
+          </div>
+
+          <div
+            *ngIf="activeParsedDataset.submitResult?.error"
+            class="error-field mt-15"
+          >
+            <div class="log-wrapper">
+              {{ activeParsedDataset.submitResult?.error | json }}
+            </div>
+          </div>
+
+          <div class="d-flex clr-justify-content-between p-10 mt-15">
+            <div>
+              <p cds-text="secondary regular" class="mb-10">
+                Found in range:
+
+                <ng-container *ngIf="activeParsedDataset.parseResult">
+                  <strong
+                    >"{{
+                      activeParsedDataset.parseResult.rangeSheetRes?.sheetName
+                    }}"!{{
+                      activeParsedDataset.parseResult.rangeSheetRes
+                        ?.rangeAddress
+                    }}</strong
+                  >
+                </ng-container>
+
+                <ng-container *ngIf="!activeParsedDataset.parseResult">
+                  <strong *ngIf="!activeParsedDataset.parsingTable"
+                    >No data found</strong
+                  >
+
+                  <span
+                    *ngIf="activeParsedDataset.parsingTable"
+                    class="d-flex clr-align-items-center"
+                  >
+                    <strong>Searching for the data...</strong>
+                    <clr-spinner class="ml-5" clrSmall></clr-spinner>
+                  </span>
+                </ng-container>
+              </p>
+              <p cds-text="secondary regular">
+                Dataset:
+                <strong>
+                  <clr-tooltip>
+                    <a
+                      clrTooltipTrigger
+                      [routerLink]="'/editor/' + activeParsedDataset.libds"
+                      >{{ activeParsedDataset.libds }}</a
+                    >
+                    <clr-tooltip-content
+                      [clrPosition]="'top-right'"
+                      [clrSize]="'sm'"
+                    >
+                      Click to edit the table
+                    </clr-tooltip-content>
+                  </clr-tooltip>
+                </strong>
+              </p>
+            </div>
+
+            <div>
+              <clr-toggle-wrapper>
+                <input
+                  type="checkbox"
+                  clrToggle
+                  [(ngModel)]="activeParsedDataset.includeInSubmission"
+                  name="options"
+                  [disabled]="
+                    !(
+                      activeParsedDataset.datasource &&
+                      activeParsedDataset.parseResult
+                    )
+                  "
+                  required
+                  value="option1"
+                />
+                <label>Include in submission</label>
+              </clr-toggle-wrapper>
+            </div>
+          </div>
+
+          <div *ngIf="isHotHidden" class="text-center w-100">
+            <clr-spinner class="spinner-md"></clr-spinner>
+          </div>
+
+          <hot-table
+            hotId="hotInstance"
+            id="hotTable"
+            class="mt-15"
+            [afterGetColHeader]="afterGetColHeader"
+            [className]="['htDark', 'htCustomHidden']"
+            [licenseKey]="hotTableLicenseKey"
+            [multiColumnSorting]="true"
+            [viewportRowRenderingOffset]="50"
+            [manualColumnResize]="true"
+            [filters]="true"
+            stretchH="all"
+          >
+          </hot-table>
+        </ng-container>
+      </ng-container>
+
+      <ng-container *ngIf="submittedCsvDatasets.length">
+        <div
+          *ngIf="!activeSubmittedCsvDataset"
+          class="no-table-selected pointer-events-none"
+        >
+          <clr-icon
+            shape="warning-standard"
+            size="40"
+            class="is-info icon-dc-fill"
+          ></clr-icon>
+          <p class="text-center color-gray mt-10" cds-text="section">
+            Please select a dataset on the left to review the submit results
+          </p>
+        </div>
+      </ng-container>
+
+      <ng-container *ngIf="activeSubmittedCsvDataset">
+        <div class="d-flex clr-justify-content-between p-10">
+          <div>
+            <p cds-text="secondary regular" class="mb-10">
+              Matched with dataset:
+              <strong>
+                <clr-tooltip>
+                  <a
+                    clrTooltipTrigger
+                    [routerLink]="'/editor/' + activeSubmittedCsvDataset.libds"
+                    >{{ activeSubmittedCsvDataset.libds }}</a
+                  >
+                  <clr-tooltip-content
+                    [clrPosition]="'top-right'"
+                    [clrSize]="'sm'"
+                  >
+                    Click to edit the table
+                  </clr-tooltip-content>
+                </clr-tooltip>
+              </strong>
+            </p>
+            <p cds-text="secondary regular" class="mb-10">
+              Status:
+              <span
+                *ngIf="activeSubmittedCsvDataset.success"
+                class="color-green"
+                ><strong>SUCCESS</strong></span
+              >
+              <span *ngIf="activeSubmittedCsvDataset.error" class="color-red"
+                ><strong>ERROR</strong></span
+              >
+            </p>
+            <p
+              *ngIf="activeSubmittedCsvDataset.error"
+              cds-text="secondary regular"
+            >
+              Error details:
+            </p>
+          </div>
+
+          <div>
+            <button
+              (click)="
+                downloadFile(
+                  activeSubmittedCsvDataset.success ||
+                    activeSubmittedCsvDataset.error
+                )
+              "
+              class="btn btn-primary-outline mt-10"
+            >
+              Download log
+            </button>
+          </div>
+        </div>
+
+        <div *ngIf="activeSubmittedCsvDataset.error" class="error-field mt-15">
+          <div class="log-wrapper">
+            {{ activeSubmittedCsvDataset.error | json }}
+          </div>
+        </div>
+      </ng-container>
+    </ng-container>
+
+    <!-- <div>
+      <p
+        *ngIf="
+          licenceState.value.viewer_rows_allowed !== Infinity &&
+          hotTable.data &&
+          hotTable.data.length > licenceState.value.viewer_rows_allowed
+        "
+        class="mt-2-i w-100 text-center"
+      >
+        To display more than {{ licenceState.value.viewer_rows_allowed }} rows,
+        contact <contact-link />
+      </p>
+    </div> -->
+  </div>
+</div>
+
+<clr-modal [(clrModalOpen)]="showSubmitReasonModal" [clrModalClosable]="false">
+  <h3 class="modal-title">
+    Submit {{ tablesToSubmit.length }}
+    {{ tablesToSubmit.length === 1 ? 'table' : 'tables' }} for approval
+  </h3>
+  <div class="modal-body">
+    <p
+      *ngIf="licenceState.value.submit_rows_limit !== Infinity"
+      cds-text="body"
+      class="licence-limit-notice mt-0 mb-15"
+    >
+      Due to current licence, only
+      {{ licenceState.value.submit_rows_limit }} rows in each file will be
+      submitted. To remove the restriction, contact
+      support&#64;datacontroller.io.
+    </p>
+
+    <div class="text-area-full-width">
+      <label for="formFields_8" class="mb-5 d-block">Message</label>
+      <textarea
+        clrTextarea
+        [(ngModel)]="submitReasonMessage"
+        tabindex="0"
+        class="submit-reason"
+        type="text"
+        id="formFields_8"
+      ></textarea>
+    </div>
+
+    <p cds-text="caption_clean" class="mt-10">
+      Tables will be sent sequentially, logs will be available after all tables
+      are submitted.
+    </p>
+  </div>
+  <div class="modal-footer">
+    <button
+      type="button"
+      class="btn btn-outline"
+      [disabled]="submitLoading"
+      (click)="showSubmitReasonModal = false"
+    >
+      Cancel
+    </button>
+    <button
+      type="button"
+      id="submit-tables"
+      class="btn btn-primary"
+      [clrLoading]="submitLoading"
+      (click)="submitTables()"
+    >
+      Submit
+    </button>
+  </div>
+</clr-modal>
+
+<clr-modal [(clrModalOpen)]="csvSubmitting" [clrModalClosable]="false">
+  <h3 class="modal-title">
+    Submitting {{ csvFiles.length }} CSV
+    {{ csvFiles.length === 1 ? 'file' : 'files' }}
+  </h3>
+  <div class="modal-body">
+    <div class="text-center">
+      <clr-spinner clrMedium></clr-spinner>
+    </div>
+
+    <p cds-text="caption_clean" class="mt-10 text-center">
+      This will take few moments
+    </p>
+  </div>
+</clr-modal>

client/src/app/multi-dataset/multi-dataset.component.scss

@@ -0,0 +1,54 @@
+.no-table-selected {
+  display: flex;
+  justify-content: center;
+  flex-direction: column;
+  align-items: center;
+  position: absolute;
+  background: var(--clr-vertical-nav-bg-color);
+  z-index: 10;
+  width: 100%;
+  height: 100%;
+  top: 0;
+}
+
+.header-row {
+  padding: 15px 0;
+  border-bottom: 1px solid #d3d3d3;
+}
+
+.dataset-input-wrapper {
+  max-width: 500px;
+  width: 100%;
+
+  textarea {
+    min-height: 200px;
+    height: 200px;
+  }
+}
+
+.submit-reason {
+  min-height: 70px;
+  max-height: 70px;
+  height: 70px;
+}
+
+.log-wrapper {
+  margin: 0 10px;
+  height: auto;
+}
+
+::ng-deep td.not-matched {
+  background-color: #ff000054;
+}
+
+.dataset-selection-actions {
+  border-top: 1px solid #d3d3d3;
+}
+
+.licence-limit-notice {
+  color: var(--cds-alias-status-warning-dark);
+}
+
+.submission-results {
+  border-bottom: 1px solid #d3d3d3;
+}

client/src/app/multi-dataset/multi-dataset.module.ts

@@ -0,0 +1,31 @@
+import { CommonModule } from '@angular/common'
+import { NgModule } from '@angular/core'
+import { FormsModule } from '@angular/forms'
+import { ClarityModule } from '@clr/angular'
+import { HotTableModule } from '@handsontable/angular'
+import { registerAllModules } from 'handsontable/registry'
+import { AppSharedModule } from '../app-shared.module'
+import { DirectivesModule } from '../directives/directives.module'
+import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
+import { MultiDatasetComponent } from './multi-dataset.component'
+import { MultiDatasetRoutingModule } from './multi-dataset-routing.module'
+import { MultiDatasetRouteComponent } from '../routes/multi-dataset-route/multi-dataset-route.component'
+
+// register Handsontable's modules
+registerAllModules()
+
+@NgModule({
+  declarations: [MultiDatasetRouteComponent, MultiDatasetComponent],
+  imports: [
+    HotTableModule,
+    MultiDatasetRoutingModule,
+    FormsModule,
+    ClarityModule,
+    AppSharedModule,
+    CommonModule,
+    DcTreeModule,
+    DirectivesModule
+  ],
+  exports: [MultiDatasetComponent]
+})
+export class MultiDatasetModule {}

client/src/app/role/role.component.ts

@@ -4,6 +4,7 @@ import { HelperService } from '../services/helper.service'
 import { Location } from '@angular/common'
 import { Router, ActivatedRoute } from '@angular/router'
 import { SasService } from '../services/sas.service'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-role',
@@ -47,61 +48,69 @@ export class RoleComponent implements OnInit {
     } else {
       if (globals.usernav.roleList === undefined) {
         this.loading = true
-        this.sasService.request('usernav/userroles', null).then((res: any) => {
-          this.loading = false
-          this.roles = res.roles
-          globals.usernav.roleList = res.roles
-          if (this.paramPresent) {
-            if (this.roles !== undefined) {
-              let validRole = this.findRole(this.roles, this.paramURI)
-              if (validRole !== false) {
-                this.loading = true
-                let data = { iwant: [{ roleid: this.paramURI }] }
-                this.sasService
-                  .request('usernav/usermembersbyrole', data)
-                  .then((res: any) => {
-                    this.loading = false
-                    this.roleMembers = res.sasmembers
-                    this.roleMembersCount = res.sasmembers.length
-                    this.roleGroups = res.sasgroups
-                    this.roleGroupsCount = res.sasgroups.length
-                    this.roleUri = validRole.ROLEURI
-                    this.roleName = validRole.ROLENAME
-                    this.roleDesc = validRole.ROLEDESC
-                  })
+        this.sasService
+          .request('usernav/userroles', null)
+          .then((res: RequestWrapperResponse) => {
+            this.loading = false
+            this.roles = res.adapterResponse.roles
+            globals.usernav.roleList = res.adapterResponse.roles
+            if (this.paramPresent) {
+              if (this.roles !== undefined) {
+                let validRole = this.findRole(this.roles, this.paramURI)
+                if (validRole !== false) {
+                  this.loading = true
+                  let data = { iwant: [{ roleid: this.paramURI }] }
+                  this.sasService
+                    .request('usernav/usermembersbyrole', data)
+                    .then((res: RequestWrapperResponse) => {
+                      this.loading = false
+                      this.roleMembers = res.adapterResponse.sasmembers
+                      this.roleMembersCount =
+                        res.adapterResponse.sasmembers.length
+                      this.roleGroups = res.adapterResponse.sasgroups
+                      this.roleGroupsCount =
+                        res.adapterResponse.sasgroups.length
+                      this.roleUri = validRole.ROLEURI
+                      this.roleName = validRole.ROLENAME
+                      this.roleDesc = validRole.ROLEDESC
+                    })
+                }
               }
             }
-          }
-        })
+          })
       } else {
         this.roles = globals.usernav.roleList
         this.roleSearch = globals.usernav.roleSearch
-        this.sasService.request('usernav/userroles', null).then((res: any) => {
-          this.roles = res.roles
-          globals.usernav.roleList = res.roles
+        this.sasService
+          .request('usernav/userroles', null)
+          .then((res: RequestWrapperResponse) => {
+            this.roles = res.adapterResponse.roles
+            globals.usernav.roleList = res.adapterResponse.roles
 
-          if (this.paramPresent) {
-            if (this.roles !== undefined) {
-              let validRole = this.findRole(this.roles, this.paramURI)
-              if (validRole !== false) {
-                this.loading = true
-                let data = { iwant: [{ roleid: this.paramURI }] }
-                this.sasService
-                  .request('usernav/usermembersbyrole', data)
-                  .then((res: any) => {
-                    this.loading = false
-                    this.roleMembers = res.sasmembers
-                    this.roleMembersCount = res.sasmembers.length
-                    this.roleGroups = res.sasgroups
-                    this.roleGroupsCount = res.sasgroups.length
-                    this.roleUri = validRole.ROLEURI
-                    this.roleName = validRole.ROLENAME
-                    this.roleDesc = validRole.ROLEDESC
-                  })
+            if (this.paramPresent) {
+              if (this.roles !== undefined) {
+                let validRole = this.findRole(this.roles, this.paramURI)
+                if (validRole !== false) {
+                  this.loading = true
+                  let data = { iwant: [{ roleid: this.paramURI }] }
+                  this.sasService
+                    .request('usernav/usermembersbyrole', data)
+                    .then((res: RequestWrapperResponse) => {
+                      this.loading = false
+                      this.roleMembers = res.adapterResponse.sasmembers
+                      this.roleMembersCount =
+                        res.adapterResponse.sasmembers.length
+                      this.roleGroups = res.adapterResponse.sasgroups
+                      this.roleGroupsCount =
+                        res.adapterResponse.sasgroups.length
+                      this.roleUri = validRole.ROLEURI
+                      this.roleName = validRole.ROLENAME
+                      this.roleDesc = validRole.ROLEDESC
+                    })
+                }
               }
             }
-          }
-        })
+          })
       }
     }
   }
@@ -125,12 +134,12 @@ export class RoleComponent implements OnInit {
     let data = { iwant: [{ roleid: role.ROLEURI }] }
     this.sasService
       .request('usernav/usermembersbyrole', data)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
         this.loading = false
-        this.roleMembers = res.sasmembers
-        this.roleMembersCount = res.sasmembers.length
-        this.roleGroups = res.sasgroups
-        this.roleGroupsCount = res.sasgroups.length
+        this.roleMembers = res.adapterResponse.sasmembers
+        this.roleMembersCount = res.adapterResponse.sasmembers.length
+        this.roleGroups = res.adapterResponse.sasgroups
+        this.roleGroupsCount = res.adapterResponse.sasgroups.length
         this.roleUri = role.ROLEURI
         this.roleName = role.ROLENAME
         this.roleDesc = role.ROLEDESC

client/src/app/routes/multi-dataset-route/multi-dataset-route.component.html

@@ -0,0 +1 @@
+<router-outlet></router-outlet>

client/src/app/routes/multi-dataset-route/multi-dataset-route.component.ts

@@ -0,0 +1,16 @@
+import { Component, OnInit, OnDestroy } from '@angular/core'
+
+@Component({
+  selector: 'app-multi-dataset-route',
+  templateUrl: './multi-dataset-route.component.html',
+  host: {
+    class: 'content-container'
+  }
+})
+export class MultiDatasetRouteComponent implements OnInit, OnDestroy {
+  constructor() {}
+
+  ngOnInit() {}
+
+  ngOnDestroy() {}
+}

client/src/app/services/app.service.ts

@@ -9,11 +9,13 @@ import { EnvironmentInfo } from '../system/models/environment-info.model'
 import { LicenceService } from './licence.service'
 import { AppSettingsService } from './app-settings.service'
 import { AppThemes } from '../models/AppSettings'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { AppStoreService } from './app-store.service'
 
 @Injectable()
 export class AppService {
   public syssite = new BehaviorSubject<string[] | null>(null)
-  private environmentInfo: EnvironmentInfo | null = null
+  private environmentInfo: EnvironmentInfo = {}
 
   constructor(
     private licenceService: LicenceService,
@@ -21,7 +23,8 @@ export class AppService {
     private sasService: SasService,
     private loggerService: LoggerService,
     private appSettingsService: AppSettingsService,
-    private router: Router
+    private router: Router,
+    private appStoreService: AppStoreService
   ) {
     this.subscribe()
 
@@ -81,16 +84,19 @@ export class AppService {
 
     await this.sasService
       .request('public/startupservice', null)
-      .then(async (res: any) => {
-        this.syssite.next([res.SYSSITE])
+      .then(async (res: RequestWrapperResponse) => {
+        this.syssite.next([res.adapterResponse.SYSSITE])
 
         let missingProps: string[] = []
 
-        if (!res.globvars || (res.globvars && !res.globvars[0]))
+        if (
+          !res.adapterResponse.globvars ||
+          (res.adapterResponse.globvars && !res.adapterResponse.globvars[0])
+        )
           missingProps.push('Globvars')
-        if (!res.sasdatasets) missingProps.push('Sasdatasets')
-        if (!res.saslibs) missingProps.push('Saslibs')
-        if (!res.xlmaps) missingProps.push('XLMaps')
+        if (!res.adapterResponse.sasdatasets) missingProps.push('Sasdatasets')
+        if (!res.adapterResponse.saslibs) missingProps.push('Saslibs')
+        if (!res.adapterResponse.xlmaps) missingProps.push('XLMaps')
 
         if (missingProps.length > 0) {
           startupServiceError = true
@@ -103,23 +109,26 @@ export class AppService {
           return
         }
 
+        const dcAdapterSettings = this.appStoreService.getDcAdapterSettings()
+
         this.environmentInfo = {
-          SYSSITE: res.SYSSITE,
-          SYSSCPL: res.SYSSCPL,
-          SYSTCPIPHOSTNAME: res.SYSTCPIPHOSTNAME,
-          SYSVLONG: res.SYSVLONG,
-          MEMSIZE: res.MEMSIZE,
-          SYSPROCESSMODE: res.SYSPROCESSMODE,
-          SYSHOSTNAME: res.SYSHOSTNAME,
-          SYSUSERID: res.SYSUSERID,
-          SYSHOSTINFOLONG: res.SYSHOSTINFOLONG,
-          SYSENCODING: res.SYSENCODING,
-          AUTOEXEC: res.AUTOEXEC,
-          ISADMIN: res.globvars[0].ISADMIN,
-          DC_ADMIN_GROUP: res.globvars[0].DC_ADMIN_GROUP
+          SYSSITE: res.adapterResponse.SYSSITE,
+          SYSSCPL: res.adapterResponse.SYSSCPL,
+          SYSTCPIPHOSTNAME: res.adapterResponse.SYSTCPIPHOSTNAME,
+          SYSVLONG: res.adapterResponse.SYSVLONG,
+          MEMSIZE: res.adapterResponse.MEMSIZE,
+          SYSPROCESSMODE: res.adapterResponse.SYSPROCESSMODE,
+          SYSHOSTNAME: res.adapterResponse.SYSHOSTNAME,
+          SYSUSERID: res.adapterResponse.SYSUSERID,
+          SYSHOSTINFOLONG: res.adapterResponse.SYSHOSTINFOLONG,
+          SYSENCODING: res.adapterResponse.SYSENCODING,
+          AUTOEXEC: res.adapterResponse.AUTOEXEC,
+          ISADMIN: res.adapterResponse.globvars[0].ISADMIN,
+          DC_ADMIN_GROUP: res.adapterResponse.globvars[0].DC_ADMIN_GROUP,
+          APP_LOC: dcAdapterSettings?.appLoc
         }
 
-        let libs = res.sasdatasets
+        let libs = res.adapterResponse.sasdatasets
         let libGroup: any = {}
         let libsAndTables
         let libraries
@@ -153,7 +162,7 @@ export class AppService {
           globals.editor.libsAndTables = libsAndTables
         }
 
-        globals.xlmaps = res.xlmaps.map((xlmap: any) => ({
+        globals.xlmaps = res.adapterResponse.xlmaps.map((xlmap: any) => ({
           id: xlmap[0],
           description: xlmap[1],
           targetDS: xlmap[2]
@@ -162,9 +171,9 @@ export class AppService {
         globals.editor.libraries = libraries
         globals.editor.startupSet = true
 
-        globals.dcLib = res.globvars[0].DCLIB
+        globals.dcLib = res.adapterResponse.globvars[0].DCLIB
 
-        await this.licenceService.activation(res)
+        await this.licenceService.activation(res.adapterResponse)
       })
       .catch((err: any) => {
         startupServiceError = true

client/src/app/services/sas-store.service.ts

@@ -17,10 +17,11 @@ import { LoggerService } from './logger.service'
 import { isSpecialMissing } from '@sasjs/utils/input/validators'
 import { Col } from '../shared/dc-validator/models/col.model'
 import { get } from 'lodash-es'
+import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Injectable()
 export class SasStoreService {
-  public libds!: string
   public response: Subject<any> = new Subject<any>()
   public changedTable: Subject<any> = new Subject<any>()
   public details: Subject<any> = new Subject<any>()
@@ -56,16 +57,13 @@ export class SasStoreService {
     program: string,
     libds: string
   ) {
-    this.libds = libds
     const tables: any = {}
     tables[tableName] = [tableData]
-    const res: EditorsGetDataSASResponse = await this.sasService.request(
-      program,
-      tables
-    )
+    const res: RequestWrapperResponse<EditorsGetDataSASResponse> =
+      await this.sasService.request(program, tables)
     const response: EditorsGetDataServiceResponse = {
-      data: res,
-      libds: this.libds
+      data: res.adapterResponse,
+      libds: libds
     }
     return response
   }
@@ -84,8 +82,10 @@ export class SasStoreService {
     tableData: any,
     tableName: string,
     program: string,
-    $dataFormats: $DataFormats | null
-  ) {
+    $dataFormats: $DataFormats | null,
+    suppressErrorSuccessMessages?: boolean,
+    adapterConfig?: any
+  ): Promise<RequestWrapperResponse<EditorsStageDataSASResponse>> {
     // add sp as third argument of createData call
 
     let tables: any = {
@@ -100,7 +100,15 @@ export class SasStoreService {
 
     tables[tableName] = [tableParams]
 
-    let res: any = await this.sasService.request(program, tables)
+    let res = await this.sasService.request<EditorsStageDataSASResponse>(
+      program,
+      tables,
+      adapterConfig,
+      {
+        suppressErrorAbortModal: suppressErrorSuccessMessages,
+        suppressSuccessAbortModal: suppressErrorSuccessMessages
+      }
+    )
 
     return res
   }
@@ -119,8 +127,7 @@ export class SasStoreService {
   ) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables)
-    return res
+    return (await this.sasService.request(program, tables)).adapterResponse
   }
 
   /**
@@ -144,8 +151,8 @@ export class SasStoreService {
    * @returns All submits
    */
   public async getSubmitts() {
-    let res: any = await this.sasService.request('editors/getsubmits', null)
-    return res
+    return (await this.sasService.request('editors/getsubmits', null))
+      .adapterResponse
   }
 
   /**
@@ -153,7 +160,8 @@ export class SasStoreService {
    * @returns All libraries
    */
   public async viewLibs() {
-    return this.sasService.request('public/viewlibs', null)
+    return (await this.sasService.request('public/viewlibs', null))
+      .adapterResponse
   }
 
   public async refreshLibInfo(libref: string) {
@@ -161,30 +169,22 @@ export class SasStoreService {
       lib2refresh: [{ libref }]
     }
 
-    return this.sasService.request('public/refreshlibinfo', data)
-  }
-
-  public async versionHistory(libDataset: any) {
-    const data = { iwant: [{ LIBDS: libDataset }] }
-    let res: any = await this.sasService.request(
-      'public/getversionhistory',
-      data
-    )
-    return res
+    return (await this.sasService.request('public/refreshlibinfo', data))
+      .adapterResponse
   }
 
   public async viewTables(lib: any) {
     let tables = { SASControlTable: [{ MPLIB: lib }] }
-    let res: any = await this.sasService.request('public/viewtables', tables)
-    return res
+    return (await this.sasService.request('public/viewtables', tables))
+      .adapterResponse
   }
 
   public async viewData(libDataset: any, filter_pk: any) {
     let tables = {
       SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }]
     }
-    let res: any = await this.sasService.request('public/viewdata', tables)
-    return res
+    return (await this.sasService.request('public/viewdata', tables))
+      .adapterResponse
   }
 
   public async viewDataSearch(
@@ -205,41 +205,36 @@ export class SasStoreService {
         }
       ]
     }
-    let res: any = await this.sasService.request('public/viewdata', tables)
-    return res
+
+    return (await this.sasService.request('public/viewdata', tables))
+      .adapterResponse
   }
 
   public async getXLMapRules(id: string) {
     const tables = {
       getxlmaps_in: [{ XLMAP_ID: id }]
     }
-    const res: any = await this.sasService.request('editors/getxlmaps', tables)
-    return res
-  }
-
-  public async getDetails(tableData: any, tableName: string, program: string) {
-    let tables: any = {}
-    tables[tableName] = [tableData]
-
-    let res: any = await this.sasService.request(program, tables)
-    return res
+    return (await this.sasService.request('editors/getxlmaps', tables))
+      .adapterResponse
   }
 
   public async showDiffs(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables, {
-      useComputeApi: false
-    })
-    return res
+    return (
+      await this.sasService.request(program, tables, {
+        useComputeApi: false
+      })
+    ).adapterResponse
   }
   public async rejecting(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables, {
-      useComputeApi: false
-    })
-    return res
+    return (
+      await this.sasService.request(program, tables, {
+        useComputeApi: false
+      })
+    ).adapterResponse
   }
 
   public async approveTable(
@@ -250,15 +245,13 @@ export class SasStoreService {
     let tables: any = {}
     tables[tableName] = [tableData]
 
-    let res: any = await this.sasService.request(program, tables)
-    return res
+    return (await this.sasService.request(program, tables)).adapterResponse
   }
 
   public async getHistory(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
-    let res: any = await this.sasService.request(program, tables)
-    return res
+    return (await this.sasService.request(program, tables)).adapterResponse
   }
 
   setQueryVariables(dataset: string, cols: any) {
@@ -270,8 +263,8 @@ export class SasStoreService {
   public async getChangeInfo(tableId: any) {
     let obj = { TABLE: tableId }
     let table = { SASControlTable: [obj] }
-    let res: any = await this.sasService.request('public/getchangeinfo', table)
-    return res
+    return (await this.sasService.request('public/getchangeinfo', table))
+      .adapterResponse
   }
 
   public async getQueryValues(
@@ -296,13 +289,13 @@ export class SasStoreService {
       tables.FILTERQUERY = filterQuery
     }
 
-    let res: any = await this.sasService
-      .request('public/getcolvals', tables)
-      .catch((er: any) => {
-        throw er
-      })
-
-    return res
+    return (
+      await this.sasService
+        .request('public/getcolvals', tables)
+        .catch((er: any) => {
+          throw er
+        })
+    ).adapterResponse
   }
 
   public async saveQuery(libds: string, filterQuery: QueryClause[]) {
@@ -311,21 +304,17 @@ export class SasStoreService {
       filterquery: filterQuery
     }
 
-    let res: any = await this.sasService.request(
-      'public/validatefilter',
-      tables
-    )
+    const res = await this.sasService.request('public/validatefilter', tables)
+
     this.filter.next(res)
-    return res
+
+    return res.adapterResponse
   }
 
   public async openTable(tableId: string) {
     let tables = { iwant: [{ table_id: tableId }] }
-    let res: any = await this.sasService.request(
-      'auditors/getstagetable',
-      tables
-    )
-    return res
+    return (await this.sasService.request('auditors/getstagetable', tables))
+      .adapterResponse
   }
 
   public checkOperator(operator: any, value: any, type: string) {

client/src/app/services/sas.service.ts

@@ -12,8 +12,10 @@ import { ServerType } from '@sasjs/utils/types/serverType'
 import { DcAdapterSettings } from '../models/DcAdapterSettings'
 import { AppStoreService } from './app-store.service'
 import { LoggerService } from './logger.service'
-import { RequestWrapperOptions } from '../models/RequestWrapperOptions'
+import { RequestWrapperOptions } from '../models/request-wrapper/RequestWrapperOptions'
 import { ErrorBody } from '../models/ErrorBody'
+import { UploadFileResponse } from '../models/UploadFile'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Injectable({
   providedIn: 'root'
@@ -93,15 +95,18 @@ export class SasService {
    * @param url service to run reuqest against
    * @param data to be sent to backend service
    * @param config additional parameters to force eg. { debug: false }
-   * @param wrapperOptions used to suppress error or success abort modals after request is finished
-   * @returns
+   * @param wrapperOptions used to provide options to the request wrapper function
+   * for example to suppress error or success abort modals after request is finished
+   * @returns adapter response or an error. It will return the `log` as well.
+   * The log could be potentially be wrong if multiple requests happen because the log this
+   * function return is the last request in the Adapter Array for the given URL.
    */
-  public request(
+  public request<responseType = any>(
     url: string,
     data: any,
     config?: any,
     wrapperOptions?: RequestWrapperOptions
-  ): Promise<any> {
+  ): Promise<RequestWrapperResponse<responseType>> {
     url = 'services/' + url
 
     if (!wrapperOptions) wrapperOptions = {}
@@ -116,9 +121,16 @@ export class SasService {
         })
         .then(
           (res: any) => {
+            const sasRequest = this.sasjsAdapter
+              .getSasRequests()
+              .find((rq) => rq.serviceLink === url)
+
             if (res.login === false) {
               this.shouldLogin.next(true)
-              reject(false)
+              reject({
+                adapterResponse: false,
+                log: sasRequest?.logFile
+              })
             }
 
             if (!this.userService.user && res.MF_GETUSER) {
@@ -140,7 +152,12 @@ export class SasService {
             }
 
             if (res.status === 404) {
-              reject({ MESSAGE: res.body || 'SAS Responded with error' })
+              reject({
+                adapterResponse: {
+                  MESSAGE: res.body || 'SAS Responded with error'
+                },
+                log: sasRequest?.logFile
+              })
             }
 
             if (typeof res.sasjsAbort !== 'undefined') {
@@ -156,7 +173,12 @@ export class SasService {
                 this.eventService.startupDataLoaded()
                 this.router.navigateByUrl('/deploy')
 
-                reject({ error: abortMsg })
+                reject({
+                  adapterResponse: {
+                    error: abortMsg
+                  },
+                  log: sasRequest?.logFile
+                })
 
                 return
               }
@@ -173,14 +195,26 @@ export class SasService {
                 )
               }
 
-              reject({ error: abortMsg })
+              reject({
+                adapterResponse: {
+                  error: abortMsg
+                },
+                log: sasRequest?.logFile
+              })
             }
 
-            resolve(res)
+            resolve({
+              adapterResponse: res,
+              log: sasRequest?.logFile
+            })
           },
           (err: { error: ErrorBody | undefined }) => {
             console.error(err)
 
+            const sasRequest = this.sasjsAdapter
+              .getSasRequests()
+              .find((rq) => rq.serviceLink === url)
+
             if (err.error) {
               let errorMessage: string | undefined = err.error.message
               let log: string | undefined
@@ -204,10 +238,18 @@ export class SasService {
                   'Request error'
                 )
               }
-              reject({ error: errorMessage })
+              reject({
+                adapterResponse: {
+                  error: errorMessage
+                },
+                log: sasRequest?.logFile
+              })
             }
 
-            reject(err)
+            reject({
+              adapterResponse: err,
+              log: sasRequest?.logFile
+            })
           }
         )
     })
@@ -221,8 +263,35 @@ export class SasService {
    * @param params Aditional parameters eg. { debug: false }
    * @returns HTTP Response
    */
-  public uploadFile(sasService: string, files: UploadFile[], params: any) {
-    return this.sasjsAdapter.uploadFile(sasService, files, params)
+  public uploadFile(
+    sasService: string,
+    files: UploadFile[],
+    params: any
+  ): Promise<UploadFileResponse> {
+    return new Promise((resolve, reject) => {
+      this.sasjsAdapter.uploadFile(sasService, files, params).then(
+        (res) => {
+          const sasRequest = this.sasjsAdapter
+            .getSasRequests()
+            .find((rq) => rq.serviceLink === 'services/editors/loadfile')
+
+          resolve({
+            adapterResponse: res,
+            log: sasRequest?.logFile
+          })
+        },
+        (err) => {
+          const sasRequest = this.sasjsAdapter
+            .getSasRequests()
+            .find((rq) => rq.serviceLink === 'services/editors/loadfile')
+
+          reject({
+            response: err,
+            log: sasRequest?.logFile
+          })
+        }
+      )
+    })
   }
 
   public async login(username: string, password: string) {

client/src/app/services/spreadsheet.service.ts

@@ -0,0 +1,190 @@
+import { Injectable } from '@angular/core'
+import { ExcelPasswordModalService } from '../shared/excel-password-modal/excel-password-modal.service'
+import { EventService } from './event.service'
+import { LicenceService } from './licence.service'
+import { SpreadsheetUtil } from '../shared/spreadsheet-util/spreadsheet-util'
+import { ParseParams } from '../models/ParseParams.interface'
+import { ParseResult } from '../models/ParseResult.interface'
+import { OpenOptions } from '../shared/excel-password-modal/models/options.interface'
+import { Result } from '../shared/excel-password-modal/models/result.interface'
+import * as XLSX from '@sheet/crypto'
+
+@Injectable({
+  providedIn: 'root'
+})
+export class SpreadsheetService {
+  private licenceState = this.licenceService.licenceState
+
+  constructor(
+    private excelPasswordModalService: ExcelPasswordModalService,
+    private eventService: EventService,
+    private licenceService: LicenceService
+  ) {}
+
+  public parseExcelFile(
+    parseParams: ParseParams,
+    onParseStateChange?: (uploadState: string) => void,
+    onTableFoundEvent?: (info: string) => void
+  ): Promise<ParseResult | undefined> {
+    const spreadSheetUtil = new SpreadsheetUtil({
+      licenceState: this.licenceState
+    })
+
+    return spreadSheetUtil.parseSpreadsheetFile(
+      parseParams,
+      this.promptExcelPassword,
+      onParseStateChange,
+      onTableFoundEvent
+    )
+  }
+
+  /**
+   * Reads the excel file using the XLSX.read() function
+   * If possible, function will use the web worker to read it in background thread
+   * otherwise fallback method will be used
+   *
+   * @param file selected in an <input>
+   * @returns WorkBook
+   */
+  public xlsxReadFile(file: any): Promise<XLSX.WorkBook> {
+    return new Promise((resolve, reject) => {
+      const spreadSheetUtil = new SpreadsheetUtil({
+        licenceState: this.licenceState
+      })
+
+      let reader: FileReader = new FileReader()
+
+      reader.onload = (fileReaderResponse: any) => {
+        spreadSheetUtil
+          .xslxStartReading(fileReaderResponse, this.promptExcelPassword)
+          .then((response) => {
+            resolve(response)
+          })
+          .catch((err) => {
+            reject(err)
+          })
+      }
+
+      reader.readAsArrayBuffer(file)
+    })
+  }
+
+  /**
+   * Read the file minimally just to get the sheet names, not reading full file
+   * to help boost the performance
+   *
+   * @returns sheet names in string array
+   */
+  public async parseExcelSheetNames(file: File): Promise<{
+    sheetNames: string[]
+    password?: string
+  }> {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader()
+
+      if (!file) {
+        console.warn('file is missing')
+        return resolve({ sheetNames: [] })
+      }
+
+      reader.onload = async (event: ProgressEvent<FileReader>) => {
+        if (!event?.target) {
+          console.warn('File reader event.target is missing')
+          return
+        }
+
+        let wb: XLSX.WorkBook | undefined = undefined
+        let fileUnlocking: boolean = false
+        let password: string | undefined
+        const data = event.target.result
+
+        try {
+          wb = XLSX.read(data, {
+            // Load file minimally to parse sheets
+            bookSheets: true,
+            type: 'binary'
+          })
+        } catch (err: any) {
+          if (err.message.toLowerCase().includes('password')) {
+            fileUnlocking = true
+
+            let passwordError = false
+
+            while (fileUnlocking) {
+              password = await this.promptExcelPassword({
+                error: passwordError
+              })
+
+              if (password) {
+                try {
+                  wb = XLSX.read(data, {
+                    // Load file minimally to parse sheets
+                    bookSheets: true,
+                    type: 'binary',
+                    password: password
+                  })
+
+                  fileUnlocking = false
+                  passwordError = false
+                } catch (err: any) {
+                  passwordError = true
+
+                  if (!err.message.toLowerCase().includes('password')) {
+                    fileUnlocking = false
+                  }
+                }
+
+                if (!password)
+                  return reject('Invalid password, failed to decrypt the file')
+              } else {
+                fileUnlocking = false
+                return reject('No password provided')
+              }
+            }
+          } else {
+            return reject('Error reading the file')
+          }
+        }
+
+        if (!wb) return reject('Error parsing the workbook')
+
+        try {
+          const sheetNames = wb.SheetNames
+
+          return resolve({
+            sheetNames: sheetNames,
+            password: password
+          })
+        } catch (e) {
+          console.error(e)
+        }
+      }
+
+      reader.onerror = function (ex) {
+        console.log(ex)
+      }
+
+      reader.readAsBinaryString(file)
+    })
+  }
+
+  public bytesToMB(size: number): number {
+    return parseFloat((size / (1024 * 1024)).toFixed(2))
+  }
+
+  /**
+   * When excel is password protected we will display the password prompt for user to type password in.
+   * @returns Password user input or undefined if discarded by user
+   */
+  private promptExcelPassword = (
+    options?: OpenOptions
+  ): Promise<string | undefined> => {
+    return new Promise((resolve, reject) => {
+      this.excelPasswordModalService
+        .open(options)
+        .subscribe((result: Result) => {
+          resolve(result.password)
+        })
+    })
+  }
+}

client/src/app/shared/excel-password-modal/excel-password-modal.component.html

@@ -0,0 +1,44 @@
+<div *ngIf="options$ | async as options" class="excel-password-root">
+  <clr-modal
+    [clrModalOpen]="options.open"
+    [clrModalSize]="'md'"
+    [clrModalClosable]="false"
+  >
+    <h3 class="modal-title center text-center color-darker-gray">
+      Password Protected File
+    </h3>
+    <div class="modal-body d-flex clr-justify-content-center">
+      <p class="m-0">Please enter password:</p>
+      <input
+        #filePasswordInput
+        [(ngModel)]="passwordInput"
+        data-lpignore="true"
+        autocomplete="off"
+        id="filePasswordInput"
+        type="text"
+        class="clr-input disable-password-manager"
+      />
+    </div>
+    <div class="modal-footer">
+      <div>
+        <p *ngIf="options.error" class="m-0 color-red">
+          Sorry that didn't work, try again.
+        </p>
+      </div>
+
+      <div class="buttons">
+        <button type="button" class="btn btn-sm btn-outline" (click)="close()">
+          Cancel
+        </button>
+        <button
+          type="button"
+          class="btn btn-sm btn-success-outline"
+          [disabled]="filePasswordInput.value.length < 1"
+          (click)="close(filePasswordInput.value)"
+        >
+          Unlock
+        </button>
+      </div>
+    </div>
+  </clr-modal>
+</div>

client/src/app/shared/excel-password-modal/excel-password-modal.component.scss

@@ -0,0 +1,18 @@
+.excel-password-root {
+  ::ng-deep {
+    .modal {
+      z-index: 1060;
+    }
+  }
+}
+
+.modal-footer {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+
+  .buttons {
+    display: flex;
+    gap: 5px;
+  }
+}

client/src/app/shared/excel-password-modal/excel-password-modal.component.ts

@@ -0,0 +1,24 @@
+import { Component } from '@angular/core'
+import { Observable } from 'rxjs'
+import { ExcelPasswordModalService } from './excel-password-modal.service'
+import { Options } from './models/options.interface'
+
+@Component({
+  selector: 'app-excel-password-modal',
+  styleUrls: ['./excel-password-modal.component.scss'],
+  templateUrl: './excel-password-modal.component.html'
+})
+export class ExcelPasswordModalComponent {
+  options$: Observable<Options> = this.excelPasswordModalService.optionsSubject$
+
+  fileUnlockError: boolean = false
+
+  passwordInput: string = ''
+
+  constructor(private excelPasswordModalService: ExcelPasswordModalService) {}
+
+  close(password?: string) {
+    this.passwordInput = ''
+    this.excelPasswordModalService.close(password)
+  }
+}

client/src/app/shared/excel-password-modal/excel-password-modal.service.ts

@@ -0,0 +1,35 @@
+import { Injectable } from '@angular/core'
+import { Subject, Observable } from 'rxjs'
+import { OpenOptions, Options } from './models/options.interface'
+import { Result } from './models/result.interface'
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ExcelPasswordModalService {
+  public optionsSubject$: Subject<Options> = new Subject()
+  public resultChange$: Subject<Result> = new Subject()
+
+  constructor() {}
+
+  public open(openOptions?: OpenOptions): Observable<Result> {
+    this.optionsSubject$.next({
+      open: true,
+      ...openOptions
+    })
+
+    this.resultChange$ = new Subject<Result>()
+    return this.resultChange$.asObservable()
+  }
+
+  close(password?: string) {
+    this.optionsSubject$.next({
+      open: false
+    })
+
+    this.resultChange$.next({
+      password
+    })
+    this.resultChange$.complete()
+  }
+}

client/src/app/shared/excel-password-modal/models/options.interface.ts

@@ -0,0 +1,7 @@
+export interface OpenOptions {
+  error?: boolean
+}
+
+export interface Options extends OpenOptions {
+  open: boolean
+}

client/src/app/shared/excel-password-modal/models/result.interface.ts

@@ -0,0 +1,3 @@
+export interface Result {
+  password: string | undefined
+}

client/src/app/shared/requests-modal/requests-modal.component.scss

@@ -61,20 +61,6 @@
     }
 }
 
-.log-wrapper {
-    min-height: 50px;
-    padding: 10px;
-    margin-top: 10px;
-
-    white-space: pre-wrap;
-    border-radius: 3px;
-
-    border: 1px solid #e2e2e2;
-
-    height: 48vh;
-    overflow: auto;
-}
-
 .no-reqs {
     border-top: 1px solid #0000001a;
     padding-top: 5px;

client/src/app/shared/shared.module.ts

@@ -14,6 +14,7 @@ import { TermsComponent } from './terms/terms.component'
 import { DirectivesModule } from '../directives/directives.module'
 import { DatasetInfoComponent } from './dataset-info/dataset-info.component'
 import { ContactLinkComponent } from './contact-link/contact-link.component'
+import { ExcelPasswordModalComponent } from './excel-password-modal/excel-password-modal.component'
 
 @NgModule({
   imports: [
@@ -30,7 +31,8 @@ import { ContactLinkComponent } from './contact-link/contact-link.component'
     AlertsComponent,
     TermsComponent,
     DatasetInfoComponent,
-    ContactLinkComponent
+    ContactLinkComponent,
+    ExcelPasswordModalComponent
   ],
   exports: [
     LoadingIndicatorComponent,
@@ -39,7 +41,8 @@ import { ContactLinkComponent } from './contact-link/contact-link.component'
     AlertsComponent,
     TermsComponent,
     DatasetInfoComponent,
-    ContactLinkComponent
+    ContactLinkComponent,
+    ExcelPasswordModalComponent
   ],
   providers: [UserService, AlertsService]
 })

client/src/app/shared/sidebar/sidebar.component.html

@@ -124,8 +124,17 @@
           routerLinkActive="active"
           >Tables</a
         >
-        <a clrVerticalNavLink routerLink="/home/files" routerLinkActive="active"
-          >Files</a
+        <a
+          clrVerticalNavLink
+          routerLink="/home/excel-maps"
+          routerLinkActive="active"
+          >Excel Maps</a
+        >
+        <a
+          clrVerticalNavLink
+          routerLink="/home/multi-load"
+          routerLinkActive="active"
+          >Multi Load</a
         >
       </clr-dropdown-menu>
     </clr-dropdown>

client/src/app/shared/sidebar/sidebar.component.scss

@@ -2,7 +2,6 @@ $sidebarWidth: 272px;
 
 .clr-vertical-nav .nav-link.active {
     background-color: transparent;
-    padding-left: 5px;
 }
 
 clr-vertical-nav {

client/src/app/shared/spreadsheet-util/random.class.ts

@@ -0,0 +1,3 @@
+export class RandomClass {
+  constructor(jo: string) {}
+}

client/src/app/shared/spreadsheet-util/spreadsheet-util.ts

@@ -0,0 +1,917 @@
+import { isSpecialMissing } from '@sasjs/utils/input/validators'
+import {
+  dateFormat,
+  dateToUtcTime,
+  dateToTime
+} from 'src/app/editor/utils/date.utils'
+import {
+  getMissingHeaders,
+  excelDateToJSDate
+} from 'src/app/editor/utils/grid.utils'
+import {
+  isStringNumber,
+  isStringDecimal
+} from 'src/app/editor/utils/types.utils'
+import { FileUploader } from 'src/app/models/FileUploader.class'
+import SheetInfo from 'src/app/models/SheetInfo'
+import { blobToFile } from 'src/app/xlmap/utils/file.utils'
+import * as XLSX from '@sheet/crypto'
+import { LicenceState } from 'src/app/models/LicenceState'
+import { BehaviorSubject } from 'rxjs'
+import { ParseParams } from 'src/app/models/ParseParams.interface'
+import { ParseResult } from 'src/app/models/ParseResult.interface'
+import { OpenOptions } from '../excel-password-modal/models/options.interface'
+
+/**
+ * Used in combination with buffer
+ */
+import * as iconv from 'iconv-lite'
+/**
+ * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
+ * Which will be send to backend
+ */
+import { Buffer } from 'buffer'
+import { MissingHeaders } from 'src/app/models/RangeInfo'
+import { SearchDataExcelResult } from 'src/app/models/SearchDataExcelResult.interface'
+type AOA = any[][]
+
+export interface ConstructorParams {
+  licenceState: BehaviorSubject<LicenceState>
+}
+
+export class SpreadsheetUtil {
+  private licenceState: BehaviorSubject<LicenceState>
+
+  constructor(params: ConstructorParams) {
+    this.licenceState = params.licenceState
+  }
+
+  /**
+   * Parses attached file and searches for the matching data
+   *
+   * If CSV is provided no searching of the data will be executed, but csv file
+   * returned back in an FileUploader array
+   *
+   * @param promptExcelPassword used to trigger the modal for password input
+   * when provided file is locked
+   * @param parseParams params required for parsing the file
+   * @param onParseStateChange callback used to inform about parsing state
+   * so the user of the function can update the UI with latest info
+   * @param onTableFoundEvent callback fired when table range is found in the file
+   *
+   * @returns parsed list of files to upload and JSON data ready for HOT usage
+   */
+  public parseSpreadsheetFile(
+    parseParams: ParseParams,
+    promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
+    onParseStateChange?: (uploadState: string) => void,
+    onTableFoundEvent?: (info: string) => void
+  ): Promise<ParseResult | undefined> {
+    return new Promise((resolve, reject) => {
+      const uploader: FileUploader = parseParams.uploader || new FileUploader()
+
+      // If workbook is present it means file is already read and we don't need
+      // to read it again, otherwise we will do a XLSX.read()
+      if (parseParams.workbook) {
+        this.parseExcelFile(
+          parseParams,
+          parseParams.workbook,
+          uploader,
+          onTableFoundEvent
+        )
+          .then((response) => {
+            resolve(response)
+          })
+          .catch((err) => {
+            reject(err)
+          })
+      } else {
+        // File is not read so we must do a XLSX.read()
+        let data: any[] = []
+
+        const file: File = parseParams.file
+
+        if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
+
+        if (onParseStateChange)
+          onParseStateChange(`Loading ${file.name} into the browser`)
+
+        let fileType = file.name.slice(
+          file.name.lastIndexOf('.') + 1,
+          file.name.lastIndexOf('.') + 4
+        )
+
+        if (fileType.toLowerCase() === 'xls') {
+          let reader: FileReader = new FileReader()
+
+          const self = this
+
+          reader.onload = async (fileReaderResponse: any) => {
+            const wb = await this.xslxStartReading(
+              fileReaderResponse,
+              promptExcelPassword,
+              parseParams.password
+            )
+
+            if (!wb) {
+              return reject('No workbook found.')
+            }
+
+            this.parseExcelFile(parseParams, wb, uploader, onTableFoundEvent)
+              .then((response) => {
+                resolve(response)
+              })
+              .catch((err) => {
+                reject(err)
+              })
+          }
+
+          reader.readAsArrayBuffer(file)
+        } else if (fileType.toLowerCase() === 'csv') {
+          return this.parseCsvFile(parseParams, uploader, fileType)
+        } else {
+          let abortMsg =
+            'Invalid file type "<b>' +
+            parseParams.file.name +
+            '</b>". Please upload csv or excel file.'
+
+          uploader.queue.pop()
+          return reject(abortMsg)
+        }
+      }
+    })
+  }
+
+  private parseExcelFile(
+    parseParams: ParseParams,
+    workbook: XLSX.WorkBook,
+    uploader: FileUploader,
+    onTableFoundEvent?: (info: string) => void
+  ): Promise<ParseResult | undefined> {
+    return new Promise((resolve, reject) => {
+      /* save data */
+      let isComplete: boolean = false
+      let missingHeaders: MissingHeaders[] = []
+
+      const csvArrayHeaders: string[] = [
+        '_____DELETE__THIS__RECORD_____',
+        ...parseParams.headerArray
+      ]
+
+      const searchResult = this.searchDataInExcel(workbook, parseParams)
+
+      let csvArrayData: any[] = []
+
+      if (searchResult.found) {
+        isComplete = true
+        csvArrayData = searchResult.found.arrayData
+
+        if (
+          !searchResult.found.headers.includes('_____delete__this__record_____')
+        ) {
+          csvArrayData = csvArrayData.map((row: any[]) => {
+            // Add empty val on start of the column to compensate for _____delete__this__record_____
+            // when not found in the file
+            row.unshift({ v: '' })
+
+            return row
+          })
+        }
+
+        if (onTableFoundEvent)
+          onTableFoundEvent(
+            `Sheet: ${searchResult.found.sheetName}\nRange: ${searchResult.found.startAddress}:${searchResult.found.endAddress}`
+          )
+      } else {
+        missingHeaders = searchResult.missing || []
+      }
+
+      if (missingHeaders.length > 0) {
+        let abortMsg = missingHeaders
+          .map((mh) => {
+            return `Sheet: ${mh.sheetName}.\nMissing columns: ${mh.missingHeaders.join(',')}`
+          })
+          .join('\n\n')
+
+        uploader.queue.pop()
+        return reject(abortMsg)
+      }
+
+      // If first row is empty, that means no data has been found
+      if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
+        let abortMsg = 'No relevant data found in File !'
+
+        uploader.queue.pop()
+        return reject(abortMsg)
+      }
+
+      if (
+        parseParams.dateTimeHeaders.length > 0 ||
+        parseParams.dateHeaders.length > 0 ||
+        parseParams.timeHeaders.length > 0
+      ) {
+        csvArrayData = this.updateDateTimeCols(
+          csvArrayHeaders,
+          csvArrayData,
+          parseParams
+        )
+      }
+
+      if (parseParams.xlRules.length > 0) {
+        csvArrayData = this.updateXLRuleCols(
+          csvArrayHeaders,
+          csvArrayData,
+          parseParams
+        )
+      }
+
+      if (!isComplete) {
+        let abortMsg = ''
+
+        if (missingHeaders.length === 0) {
+          abortMsg = 'No relevant data found in File !'
+        } else {
+          abortMsg = missingHeaders
+            .map((mh) => {
+              return `Sheet: ${mh.sheetName}.\nMissing columns: ${mh.missingHeaders.join(',')}`
+            })
+            .join('\n\n')
+        }
+
+        // abort message is fired, return undefined
+        uploader.queue.pop()
+        return reject(abortMsg)
+      } else {
+        parseParams.headerShow = csvArrayHeaders
+
+        // Remove the metadata from the cells, leave only values
+        csvArrayData = csvArrayData.map((row: any) =>
+          row.map((col: any) => {
+            if (col.t === 'n') {
+              return col.v
+            } else {
+              if (col.w) return col.v
+
+              return typeof col.v === 'string' ? col.v.trim() : col.v
+            }
+          })
+        )
+
+        csvArrayData = csvArrayData.map((row: any) => {
+          return row.map((col: any, index: number) => {
+            if (!col && col !== 0) col = ''
+
+            /**
+             * Keeping this for the reference
+             * Code below used to convert JSON to CSV
+             * now the XLSX is converting to CSV
+             */
+            // if (isNaN(col)) {
+            //   // Match and replace the double quotes, ignore the first and last char
+            //   // in case they are double quotes already
+            //   col = col.replace(/(?<!^)"(?!$)/g, '""')
+
+            //   if (col.search(/,/g) > -1 ||
+            //       col.search(/\r|\n/g) > -1
+            //   ) {
+            //     // Missing quotes at the end
+            //     if (col.search(/"$/g) < 0) {
+            //       col = col + '"' // So we add them
+            //     }
+
+            //     // Missing quotes at the start
+            //     if (col.search(/^"/g) < 0) {
+            //       col = '"' + col // So we add them
+            //     }
+            //   }
+            // }
+
+            const colName = parseParams.headerShow[index]
+            const colRule = parseParams.dcValidator?.getRule(colName)
+
+            if (colRule?.type === 'numeric') {
+              if (isSpecialMissing(col) && !col.includes('.')) col = '.' + col
+            }
+
+            return col
+          })
+        })
+
+        // Apply licence rows limitation if exists, it is only affecting data
+        // which will be send to SAS
+        const strippedCsvArrayData = csvArrayData.slice(
+          0,
+          this.licenceState.value.submit_rows_limit
+        )
+        // To submit to sas service, we need clean version of CSV of file
+        // attached. XLSX will do the parsing and heavy lifting
+        // First we create worksheet of json (data we extracted)
+        let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+          skipHeader: true
+        })
+
+        // create CSV to be uploaded from worksheet
+        let csvContentClean = XLSX.utils.sheet_to_csv(ws)
+        // Prepend headers
+        csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
+
+        // Blob from which CSV file will be created depending of the selected
+        // encoding
+        let blob: Blob
+
+        if (parseParams.encoding === 'WLATIN1') {
+          // WLATIN1
+          let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
+          blob = new Blob([encoded], { type: 'application/csv' })
+        } else {
+          // UTF-8
+          blob = new Blob([csvContentClean], { type: 'application/csv' })
+        }
+
+        let newCSVFile: File = blobToFile(blob, parseParams.file.name + '.csv')
+        uploader.addToQueue([newCSVFile])
+      }
+
+      if (csvArrayData.length === 0) {
+        return reject(
+          `Table in the file is empty. Data found on sheet: ${searchResult.found?.sheetName || ''}`
+        )
+      }
+
+      if (!searchResult.found) {
+        return reject(
+          `No relevant data found. 'found' object is empty, unexpected error occurred.`
+        )
+      }
+
+      const rangeStartAddress = searchResult.found.startAddress || ''
+      const rangeEndAddress = searchResult.found.endAddress || ''
+
+      return resolve({
+        uploader,
+        data: csvArrayData,
+        rangeSheetRes: {
+          found: !!searchResult.found,
+          sheetName: searchResult.found.sheetName,
+          rangeStartAddress: rangeStartAddress,
+          rangeEndAddress: rangeEndAddress,
+          rangeAddress: `${rangeStartAddress}:${rangeEndAddress}`,
+          missingHeaders: missingHeaders
+        },
+        headerShow: parseParams.headerShow
+      })
+    })
+  }
+
+  private parseCsvFile(
+    parseParams: ParseParams,
+    uploader: FileUploader,
+    fileType: string
+  ) {
+    return new Promise((resolve, reject) => {
+      if (this.licenceState.value.submit_rows_limit !== Infinity) {
+        uploader.queue.pop()
+        return reject(
+          'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
+        )
+      }
+
+      if (parseParams.encoding === 'WLATIN1') {
+        let reader = new FileReader()
+        const self = this
+        // Closure to capture the file information.
+        reader.onload = (theFile: any) => {
+          let encoded = iconv.decode(
+            Buffer.from(theFile.target.result),
+            'CP-1252'
+          )
+          let blob = new Blob([encoded], { type: fileType })
+          let encodedFile: File = blobToFile(blob, parseParams.file.name)
+          uploader.queue.pop()
+          uploader.addToQueue([encodedFile])
+
+          return resolve({
+            uploader
+          })
+        }
+
+        reader.readAsArrayBuffer(parseParams.file)
+      } else {
+        return resolve({
+          uploader
+        })
+      }
+    })
+  }
+
+  public bytesToMB(size: number): number {
+    return parseFloat((size / (1024 * 1024)).toFixed(2))
+  }
+
+  /**
+   * Wrapper function for XLSX.read() with integrated 'unlock' functionality
+   * Used by multi load component to load the file while user chooses the datasets
+   * to be updated
+   *
+   * @param fileReaderResponse response from the file reader
+   * @param promptExcelPassword password callback
+   * @param password password provided by the user
+   * @returns WorkBook
+   */
+  public xslxStartReading(
+    fileReaderResponse: any,
+    promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
+    password?: string
+  ): Promise<XLSX.WorkBook> {
+    return new Promise(async (resolve, reject) => {
+      /* read workbook */
+      const bstr = this.toBstr(fileReaderResponse.target.result)
+      let wb: XLSX.WorkBook | undefined = undefined
+      let fileUnlocking: boolean = false
+
+      const xlsxOptions: XLSX.ParsingOptions = {
+        type: 'binary',
+        cellDates: false,
+        cellFormula: true,
+        cellStyles: true,
+        cellNF: false,
+        cellText: false,
+        password: password
+      }
+
+      try {
+        wb = await this.xlsxRead(bstr, {
+          ...xlsxOptions
+        })
+      } catch (err: any) {
+        if (err.message.toLowerCase().includes('password')) {
+          fileUnlocking = true
+
+          let passwordError = false
+
+          while (fileUnlocking) {
+            const password = await promptExcelPassword({
+              error: passwordError
+            })
+
+            if (password) {
+              try {
+                wb = await this.xlsxRead(bstr, {
+                  ...xlsxOptions,
+                  password: password
+                })
+
+                fileUnlocking = false
+                passwordError = false
+              } catch (err: any) {
+                passwordError = true
+
+                if (!err.message.toLowerCase().includes('password')) {
+                  fileUnlocking = false
+                }
+              }
+            } else {
+              fileUnlocking = false
+            }
+          }
+        } else {
+          return reject('Error reading the file')
+        }
+      }
+
+      if (!wb) return reject('Failed to parse a workbook')
+
+      return resolve(wb)
+    })
+  }
+
+  /**
+   * XLSX Read wrapper which uses Web Worker to read the file and not block
+   * the UI while reading. It will allow reading bigger files.
+   * If worker fails, fallback is regular file read.
+   * @param data
+   * @param opts
+   * @returns
+   */
+  private xlsxRead(
+    data: any,
+    opts?: XLSX.ParsingOptions | undefined
+  ): Promise<XLSX.WorkBook> {
+    return new Promise((resolve, reject) => {
+      if (opts && opts.password) {
+        console.info('Not using worker to parse the XLSX - has password')
+        // At the moment worker can't use crypto version of SheetJS because of
+        // 'global not defined' issue
+        return resolve(XLSX.read(data, opts))
+      }
+
+      if (typeof Worker === 'undefined') {
+        console.info(
+          'Not using worker to parse the XLSX - no Worker available in this environment'
+        )
+        // Web workers are not supported in this environment.
+        // You should add a fallback so that your program still executes correctly.
+        return resolve(XLSX.read(data, opts))
+      }
+
+      // Ultimately use Web Worker to parse the excel
+      console.info('Using worker to parse the XLSX')
+
+      const worker = new Worker(
+        new URL('../../spreadsheet.worker', import.meta.url)
+      )
+
+      worker.onmessage = ({ data }) => {
+        if (data.event === 'reading_end') {
+          resolve(data.workbook)
+        } else if (data.error) {
+          reject(data.error)
+        } else {
+          console.info(
+            'Worker failed to parse the XLSX - fallback to non worker parsing'
+          )
+          // Fallback to reading without Worker
+          resolve(XLSX.read(data, opts))
+        }
+      }
+
+      worker.postMessage({
+        data,
+        opts
+      })
+
+      // Big timeout (10 minutes) in case Worker fails and no response
+      // and read the file with fallback method without worker
+      setTimeout(() => {
+        return resolve(XLSX.read(data, opts))
+      }, 600 * 1000) // 10 minutes
+    })
+  }
+
+  private searchDataInExcel(
+    wb: XLSX.WorkBook,
+    parseParams: ParseParams
+  ): SearchDataExcelResult {
+    let headerStartAddress: string | undefined
+    let headerEndAddress: string | undefined
+    let rangeStartAddress: string | undefined
+    let rangeEndAddress: string | undefined
+    let sheetName: string = ''
+    let missing: MissingHeaders[] = []
+
+    const csvArrayHeaders: string[] = [
+      '_____DELETE__THIS__RECORD_____',
+      ...parseParams.headerArray
+    ]
+    let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
+    let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
+      (map: any, obj: string) => {
+        map[obj] = -1
+        return map
+      },
+      {}
+    )
+
+    for (let sheet of wb.SheetNames) {
+      headerStartAddress = undefined
+      headerEndAddress = undefined
+
+      csvArrayHeadersMap = csvArrayHeadersLower.reduce(
+        (map: any, obj: string) => {
+          map[obj] = -1
+          return map
+        },
+        {}
+      )
+
+      let missingHeaders = []
+      sheetName = sheet
+      const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
+
+      // Find the first header
+      Object.keys(ws).forEach((wsKey) => {
+        const cellValue = ws[wsKey].v
+
+        // If the cell does not have `v` property we ignore it, those are metadata properties
+        if (cellValue && typeof cellValue === 'string') {
+          const potentialHeader = cellValue.toLowerCase()
+          const headerIndex = csvArrayHeadersLower.indexOf(potentialHeader)
+
+          if (headerIndex > -1) {
+            csvArrayHeadersMap[potentialHeader] = wsKey
+
+            if (!headerStartAddress) headerStartAddress = wsKey
+
+            // Update on every found header, until the end, which will leave
+            // last found header address
+            headerEndAddress = wsKey
+          }
+        }
+      })
+
+      // If _____delete__this__record_____ is not found in the file, remove it from the array
+      if (csvArrayHeadersMap['_____delete__this__record_____'] === -1)
+        delete csvArrayHeadersMap['_____delete__this__record_____']
+
+      // Parse missing headers, if any, abort the search and jump to next sheet
+      missingHeaders = Object.keys(csvArrayHeadersMap).filter(
+        (header) => csvArrayHeadersMap[header] === -1
+      )
+
+      if (missingHeaders.length > 0) {
+        missing.push({
+          sheetName: sheetName,
+          missingHeaders: missingHeaders.map((header) => header.toUpperCase())
+        })
+
+        continue
+      }
+
+      // If no headers are missing, start parsing the data column by column
+      const foundHeaders = Object.keys(csvArrayHeadersMap)
+
+      let json: any = []
+      let arrayData: any = []
+      let endRow: number
+
+      // Sort the headers so first headers are primary key columns
+      const foundHeadersSorted = foundHeaders.sort((a: string, b: string) => {
+        const aIsPk = parseParams.headerPks.includes(a) ? 1 : 0
+        const bIsPk = parseParams.headerPks.includes(b) ? 1 : 0
+
+        return bIsPk - aIsPk
+      })
+
+      foundHeadersSorted.forEach((header) => {
+        const headerAddress = csvArrayHeadersMap[header]
+        const headerAddressLetterRegex = headerAddress.match(/\D+/)
+        const headerAddressNumberRegex = headerAddress.match(/\d+/)
+
+        const headerAddressLetter =
+          (headerAddressLetterRegex ? headerAddressLetterRegex[0] : -1) || -1
+        const headerAddressNumber = parseInt(
+          (headerAddressNumberRegex ? headerAddressNumberRegex[0] : -1) || -1
+        )
+
+        const firstDataRow = headerAddressNumber + 1
+
+        let jsonRow = 0
+
+        // If end row found, use it as a limit
+        if (endRow) {
+          for (let row = firstDataRow; row <= endRow; row++) {
+            const address = `${headerAddressLetter}${row}`
+            const cell = ws[address]
+
+            if (parseParams.headerPks.includes(header)) {
+              // If this column is primary key and has less rows, set new endRow
+              if (cell === undefined || cell.v === undefined) {
+                endRow = row
+                break
+              }
+            }
+
+            // Push to array of objects
+            if (!json[jsonRow]) json.push({})
+            if (cell)
+              json[jsonRow][header] =
+                typeof cell.v === 'string' ? cell.v.trim() : cell.v
+
+            // Push to array of arrays, but with all cell meta info
+            if (!arrayData[jsonRow]) arrayData.push([])
+            arrayData[jsonRow].push(cell ?? { v: '' })
+
+            jsonRow++
+          }
+        } else {
+          // If end row not found, go trough rows until empty PK row appears
+          let cellsRow = firstDataRow
+
+          while (endRow === undefined) {
+            const address = `${headerAddressLetter}${cellsRow}`
+            const cell = ws[address]
+
+            if (!cell || cell.v === undefined) {
+              // This is an empty row, row before this one is the last row with data
+              endRow = cellsRow - 1
+              break
+            }
+
+            // Push to array of objects
+            if (!json[jsonRow]) json.push({})
+            if (ws[address].t === 'n') {
+              // If type is number, use the Underlying value, otherwise use Formatted text
+              // https://docs.sheetjs.com/docs/csf/cell
+              json[jsonRow][header] = ws[address].v
+            } else {
+              if (ws[address].w) {
+                json[jsonRow][header] = ws[address].w
+              } else {
+                json[jsonRow][header] =
+                  typeof ws[address].v === 'string'
+                    ? ws[address].v.trim()
+                    : ws[address].v
+              }
+            }
+
+            // Push to array of arrays, but with all cell meta info
+            if (!arrayData[jsonRow]) arrayData.push([])
+            arrayData[jsonRow].push(cell ?? { v: '' })
+
+            cellsRow++
+            jsonRow++
+          }
+        }
+
+        if (headerStartAddress && headerEndAddress) {
+          const endHeaderAddressLetterRegex = headerEndAddress.match(/\D+/)
+
+          rangeStartAddress = headerStartAddress
+          rangeEndAddress = `${endHeaderAddressLetterRegex}${endRow}`
+        }
+      })
+
+      // Remove leftover elements with missing pk values
+      const rowsWithMissingPk: number[] = []
+      let firstRowIndexMissingPk: number | undefined
+
+      json.forEach((row: any, rowIndex: number) => {
+        let missingPk = false
+
+        parseParams.headerPks.forEach((pkHeader) => {
+          if (row[pkHeader.toLowerCase()] === undefined) missingPk = true
+        })
+
+        if (missingPk) {
+          rowsWithMissingPk.push(rowIndex)
+
+          if (!firstRowIndexMissingPk) firstRowIndexMissingPk = rowIndex
+        }
+      })
+
+      // Remove all rows after the first row with missing PK column even if some
+      // columns after has populated PK
+      if (firstRowIndexMissingPk) {
+        json.splice(firstRowIndexMissingPk, Infinity)
+        arrayData.splice(firstRowIndexMissingPk, Infinity)
+      } else {
+        // Fallback: Remove only rows with missing PK
+        rowsWithMissingPk
+          .sort((a, b) => b - a)
+          .forEach((index) => {
+            json.splice(index, 1)
+            arrayData.splice(index, 1)
+          })
+      }
+
+      if (!arrayData.length) {
+        return {}
+      }
+
+      // If we got to this point it means headers are matched
+      return {
+        found: {
+          data: json,
+          arrayData: arrayData,
+          sheetName: sheetName,
+          startAddress: rangeStartAddress,
+          endAddress: rangeEndAddress,
+          headers: foundHeaders
+        }
+      }
+    }
+
+    // No complete data found
+    return {
+      missing: missing
+    }
+  }
+
+  private updateDateTimeCols(
+    headers: any,
+    data: any,
+    parseParams: ParseParams
+  ) {
+    if (parseParams.dateHeaders.length > 0) {
+      const dateCols: number[] = []
+      parseParams.dateHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          dateCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        dateCols.forEach((element) => {
+          const obj = row[element]
+          if (isStringNumber(obj.v)) {
+            const date = excelDateToJSDate(Number(obj.v))
+
+            obj.v =
+              date.getFullYear() +
+              '-' +
+              ('0' + (date.getMonth() + 1)).slice(-2) +
+              '-' +
+              ('0' + date.getDate()).slice(-2)
+          } else {
+            if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
+              const date = new Date(obj.v)
+              if (date.toUTCString() !== 'Invalid Date') {
+                obj.v = dateFormat(date)
+              }
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    if (parseParams.timeHeaders.length > 0) {
+      let timeCols: number[] = []
+      parseParams.timeHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          timeCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        timeCols.forEach((element) => {
+          const obj = row[element]
+          if (
+            isStringNumber(obj.v) ||
+            isStringDecimal(obj.v) ||
+            obj.v.includes('E-')
+          ) {
+            const date = excelDateToJSDate(Number(obj.v))
+
+            obj.v = dateToUtcTime(date)
+          }
+          row[element] = obj
+        })
+      })
+    }
+    if (parseParams.dateTimeHeaders.length > 0) {
+      let dateTimeCols: number[] = []
+      parseParams.dateTimeHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          dateTimeCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        dateTimeCols.forEach((element) => {
+          const obj = row[element]
+          if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
+            const date = excelDateToJSDate(Number(obj.v))
+            obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
+          } else {
+            if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
+              let str = obj.v.substring(0, obj.v.indexOf(':'))
+              str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
+              obj.v = str
+            }
+            const date = new Date(obj.v)
+            if (date.toUTCString() !== 'Invalid Date') {
+              obj.v = dateFormat(date) + ' ' + dateToTime(date)
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    return data
+  }
+
+  private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
+    if (parseParams.xlRules.length > 0) {
+      const xlRuleCols: any = []
+      parseParams.xlRules.forEach((element: any) => {
+        if (headers.indexOf(element.XL_COLUMN) !== -1) {
+          element['index'] = headers.indexOf(element.XL_COLUMN)
+          xlRuleCols.push(element)
+        }
+      })
+      data.forEach((row: any[]) => {
+        xlRuleCols.forEach((element: any) => {
+          const obj = row[element.index]
+          if (element.XL_RULE === 'FORMULA') {
+            if ('f' in obj) {
+              if (obj['t'] === 'n') {
+                obj['v'] = '=' + obj['f']
+              } else {
+                obj['w'] = '=' + obj['f']
+              }
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    return data
+  }
+
+  private toBstr(res: any) {
+    let bytes = new Uint8Array(res)
+    let binary = ''
+    let length = bytes.byteLength
+    for (let i = 0; i < length; i++) {
+      binary += String.fromCharCode(bytes[i])
+    }
+    return binary
+  }
+}

client/src/app/shared/terms/terms.component.scss

@@ -19,8 +19,6 @@
   .card-block {
     flex: 1;
     overflow: auto;
-
-    background-color: #f4f4f4;
   }
 }
 

client/src/app/shared/terms/terms.component.ts

@@ -8,6 +8,7 @@ import {
 import { SasService } from '../../services/sas.service'
 import * as marked from 'marked'
 import { EULA } from 'src/environments/_eula'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-terms',
@@ -51,8 +52,12 @@ export class TermsComponent implements OnInit, AfterViewInit {
 
     this.sasService
       .request(`public/registeruser`, table)
-      .then((res: any) => {
-        if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
+      .then((res: RequestWrapperResponse) => {
+        if (
+          res.adapterResponse.return &&
+          res.adapterResponse.return[0] &&
+          res.adapterResponse.return[0].MSG === 'SUCCESS'
+        ) {
           location.reload()
         }
       })

client/src/app/shared/utils/hot.utils.ts

@@ -0,0 +1,13 @@
+import { globals } from '../../_globals'
+
+/**
+ * Function reused in HOT instances to add a class used for dark mode
+ */
+export const baseAfterGetColHeader = (
+  column: number,
+  TH: HTMLTableCellElement,
+  headerLevel: number
+) => {
+  // Dark mode
+  TH.classList.add(globals.handsontable.darkTableHeaderClass)
+}

client/src/app/shared/viewboxes/viewboxes.component.ts

@@ -434,7 +434,7 @@ export class ViewboxesComponent implements OnInit, AfterViewInit, OnDestroy {
 
                 if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
                 // Dark mode
-                th.classList.add('darkTH')
+                th.classList.add(globals.handsontable.darkTableHeaderClass)
               }
             })
             hotInstance?.render()

client/src/app/spreadsheet.worker.ts

@@ -0,0 +1,30 @@
+/// <reference lib="webworker" />
+
+/**
+ * We use normal version of the XLSX (SheetJS)
+ * Because at the moment "@sheet/crypto" can't work in the Web Worker environment
+ * Because of the missing "global" variable.
+ */
+import * as XLSX from 'xlsx'
+
+addEventListener('message', ({ data }) => {
+  const { data: xldata, opts: xlopts } = data as {
+    data: any
+    opts?: any
+  }
+
+  try {
+    const workbook = XLSX.read(xldata, xlopts)
+
+    postMessage({
+      event: 'reading_end',
+      workbook: workbook
+    })
+  } catch (err: any) {
+    if (err.message.toLowerCase().includes('password')) {
+      postMessage({
+        error: err
+      })
+    }
+  }
+})

client/src/app/stage/stage.component.ts

@@ -8,6 +8,7 @@ import { HotTableInterface } from '../models/HotTable.interface'
 import { LicenceService } from '../services/licence.service'
 import { globals } from '../_globals'
 import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-stage',
@@ -35,7 +36,7 @@ export class StageComponent implements OnInit {
     maxRows: this.licenceState.value.stage_rows_allowed || Infinity,
     afterGetColHeader: (column, th, headerLevel) => {
       // Dark mode
-      th.classList.add('darkTH')
+      th.classList.add(globals.handsontable.darkTableHeaderClass)
     }
   }
 
@@ -66,7 +67,7 @@ export class StageComponent implements OnInit {
     )
     if (xlmap) {
       const id = this.hotTable.data[0].XLMAP_ID
-      this.route.navigateByUrl('/home/files/' + id)
+      this.route.navigateByUrl('/home/excel-maps/' + id)
     } else {
       this.route.navigateByUrl('/editor/' + this.tableDetails.BASE_TABLE)
     }
@@ -180,10 +181,12 @@ export class StageComponent implements OnInit {
 
     this.sasService
       .request('editors/restore', data)
-      .then((res: EditorsRestoreServiceResponse) => {
-        if (res.restore_out) {
+      .then((res: RequestWrapperResponse<EditorsRestoreServiceResponse>) => {
+        if (res.adapterResponse.restore_out) {
           this.route.navigate([`/stage`]).then(() => {
-            this.route.navigate([`/stage/${res.restore_out[0].LOADREF}`])
+            this.route.navigate([
+              `/stage/${res.adapterResponse.restore_out[0].LOADREF}`
+            ])
           })
         }
       })

client/src/app/system/models/environment-info.model.ts

@@ -1,15 +1,16 @@
 export interface EnvironmentInfo {
-  SYSSITE: string
-  SYSSCPL: string
-  SYSTCPIPHOSTNAME: string
-  SYSVLONG: string
-  MEMSIZE: string
-  SYSPROCESSMODE: string
-  SYSHOSTNAME: string
-  SYSUSERID: string
-  SYSHOSTINFOLONG: string
-  SYSENCODING: string
-  AUTOEXEC: string
-  ISADMIN: number
-  DC_ADMIN_GROUP: string
+  SYSSITE?: string
+  SYSSCPL?: string
+  SYSTCPIPHOSTNAME?: string
+  SYSVLONG?: string
+  MEMSIZE?: string
+  SYSPROCESSMODE?: string
+  SYSHOSTNAME?: string
+  SYSUSERID?: string
+  SYSHOSTINFOLONG?: string
+  SYSENCODING?: string
+  AUTOEXEC?: string
+  ISADMIN?: number
+  DC_ADMIN_GROUP?: string
+  APP_LOC?: string
 }

client/src/app/system/system.component.html

@@ -77,6 +77,10 @@
               DC Admin Group:
               <span class="dark">{{ environmentInfo?.DC_ADMIN_GROUP }}</span>
             </p>
+            <p cds-text="label" class="m-0">
+              App Location:
+              <span class="dark">{{ environmentInfo?.APP_LOC }}</span>
+            </p>
           </div>
         </div>
 

client/src/app/system/system.component.ts

@@ -9,6 +9,7 @@ import { AppInfo } from './models/app-info.model'
 import { EnvironmentInfo } from './models/environment-info.model'
 import { AppSettingsService } from '../services/app-settings.service'
 import { AppSettings } from '../models/AppSettings'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-system',
@@ -51,7 +52,7 @@ export class SystemComponent implements OnInit {
     this.environmentInfo = this.appService.getEnvironmentInfo()
     this.settings = this.appSettingsService.settings.value
 
-    if (this.environmentInfo) {
+    if (this.environmentInfo.AUTOEXEC) {
       this.environmentInfo.AUTOEXEC = decodeURIComponent(
         this.environmentInfo.AUTOEXEC
       )
@@ -83,12 +84,12 @@ export class SystemComponent implements OnInit {
 
     this.sasService
       .request('admin/refreshcatalog', null)
-      .then((res: any) => {
-        this.response = this.parseResponse(res)
+      .then((res: RequestWrapperResponse) => {
+        this.response = this.parseResponse(res.adapterResponse)
         this.responseModal = true
       })
       .catch((err: any) => {
-        this.response = this.parseResponse(err)
+        this.response = this.parseResponse(err.adapterResponse)
         this.responseModal = true
       })
       .finally(() => {
@@ -101,12 +102,12 @@ export class SystemComponent implements OnInit {
 
     this.sasService
       .request('admin/refreshtablelineage', null)
-      .then((res: any) => {
-        this.response = this.parseResponse(res)
+      .then((res: RequestWrapperResponse) => {
+        this.response = this.parseResponse(res.adapterResponse)
         this.responseModal = true
       })
       .catch((err: any) => {
-        this.response = this.parseResponse(err)
+        this.response = this.parseResponse(err.adapterResponse)
         this.responseModal = true
       })
       .finally(() => {

client/src/app/user/user.component.ts

@@ -6,6 +6,7 @@ import { Location } from '@angular/common'
 import { SasService } from '../services/sas.service'
 import { SASjsConfig } from '@sasjs/adapter'
 import { ServerType } from '@sasjs/utils/types/serverType'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-user',
@@ -89,10 +90,10 @@ export class UserComponent implements OnInit {
         } else {
           this.sasService
             .request('usernav/usermembers', null)
-            .then((res: any) => {
+            .then((res: RequestWrapperResponse) => {
               this.loading = false
-              this.users = res.users
-              globals.usernav.userList = res.users
+              this.users = res.adapterResponse.users
+              globals.usernav.userList = res.adapterResponse.users
             })
         }
       } else {
@@ -141,18 +142,18 @@ export class UserComponent implements OnInit {
 
           this.sasService
             .request('usernav/usergroupsbymember', data)
-            .then((res: any) => {
+            .then((res: RequestWrapperResponse) => {
               this.loading = false
 
               switch (this.serverType) {
                 case ServerType.Sas9: {
-                  this.userInfo = res.info[0]
-                  this.userEmails = res.emails
-                  this.userEmailsCount = res.emails.length
-                  this.userRoles = res.roles
-                  this.userRolesCount = res.roles.length
-                  this.userLogins = res.logins
-                  this.userLoginsCount = res.logins.length
+                  this.userInfo = res.adapterResponse.info[0]
+                  this.userEmails = res.adapterResponse.emails
+                  this.userEmailsCount = res.adapterResponse.emails.length
+                  this.userRoles = res.adapterResponse.roles
+                  this.userRolesCount = res.adapterResponse.roles.length
+                  this.userLogins = res.adapterResponse.logins
+                  this.userLoginsCount = res.adapterResponse.logins.length
 
                   break
                 }
@@ -162,7 +163,7 @@ export class UserComponent implements OnInit {
                       (userAr) => userAr.URI === uri
                     )
                   } else {
-                    const group = res.groups[0]
+                    const group = res.adapterResponse.groups[0]
 
                     this.userInfo = {
                       URI: group.ID,
@@ -175,9 +176,9 @@ export class UserComponent implements OnInit {
                 }
               }
 
-              this.userData = res
-              this.userGroups = res.groups
-              this.userGroupsCount = res.groups.length
+              this.userData = res.adapterResponse
+              this.userGroups = res.adapterResponse.groups
+              this.userGroupsCount = res.adapterResponse.groups.length
             })
         }
       }
@@ -238,18 +239,18 @@ export class UserComponent implements OnInit {
       let data = { iwant: [{ uri: user.URI }] }
       this.sasService
         .request('usernav/usergroupsbymember', data)
-        .then((res: any) => {
+        .then((res: RequestWrapperResponse) => {
           this.loading = false
 
           switch (this.serverType) {
             case ServerType.Sas9: {
-              this.userInfo = res.info[0]
-              this.userEmails = res.emails
-              this.userEmailsCount = res.emails.length
-              this.userRoles = res.roles
-              this.userRolesCount = res.roles.length
-              this.userLogins = res.logins
-              this.userLoginsCount = res.logins.length
+              this.userInfo = res.adapterResponse.info[0]
+              this.userEmails = res.adapterResponse.emails
+              this.userEmailsCount = res.adapterResponse.emails.length
+              this.userRoles = res.adapterResponse.roles
+              this.userRolesCount = res.adapterResponse.roles.length
+              this.userLogins = res.adapterResponse.logins
+              this.userLoginsCount = res.adapterResponse.logins.length
 
               break
             }
@@ -259,7 +260,7 @@ export class UserComponent implements OnInit {
                   (userAr) => userAr.URI === user.URI
                 )
               } else {
-                const group = res.groups[0]
+                const group = res.adapterResponse.groups[0]
 
                 this.userInfo = {
                   URI: group.ID,
@@ -272,9 +273,9 @@ export class UserComponent implements OnInit {
             }
           }
 
-          this.userData = res
-          this.userGroups = res.groups
-          this.userGroupsCount = res.groups.length
+          this.userData = res.adapterResponse
+          this.userGroups = res.adapterResponse.groups
+          this.userGroupsCount = res.adapterResponse.groups.length
         })
     }
   }

client/src/app/viewer/viewer.component.ts

@@ -9,7 +9,7 @@ import {
 import { SasStoreService } from '../services/sas-store.service'
 import { Subscription } from 'rxjs'
 
-import { Params, Router } from '@angular/router'
+import { Router } from '@angular/router'
 import { ActivatedRoute } from '@angular/router'
 import { globals } from '../_globals'
 
@@ -18,7 +18,6 @@ import { HelperService } from '../services/helper.service'
 import { HotTableRegisterer } from '@handsontable/angular'
 import { SasService } from '../services/sas.service'
 import { SASjsConfig } from '@sasjs/adapter'
-import { AppService } from '../services/app.service'
 import { QueryComponent } from '../query/query.component'
 
 import { FilterGroup, FilterQuery } from '../models/FilterQuery'
@@ -38,6 +37,7 @@ import { DataFormat } from '../models/sas/common/DateFormat'
 import { Libinfo } from '../models/sas/common/Libinfo'
 import { LicenceService } from '../services/licence.service'
 import { Location } from '@angular/common'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-viewer',
@@ -118,7 +118,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
     },
     afterGetColHeader: (col: number, th: any, headerLevel: number) => {
       // Dark mode
-      th.classList.add('darkTH')
+      th.classList.add(globals.handsontable.darkTableHeaderClass)
     },
     rowHeaderWidth: 15,
     rowHeights: 20,
@@ -1076,7 +1076,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
               if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
 
               // Dark mode
-              th.classList.add('darkTH')
+              th.classList.add(globals.handsontable.darkTableHeaderClass)
             }
           })
         }

client/src/app/xlmap/xlmap.component.scss

@@ -39,8 +39,6 @@ clr-tree-node button {
 }
 
 .content-area {
-  padding: 0.5rem !important;
-
   display: flex;
   flex-direction: column;
 }

client/src/app/xlmap/xlmap.component.ts

@@ -22,6 +22,7 @@ import {
 } from '../services'
 import { getCellAddress, getFinishingCell } from './utils/xl.utils'
 import { blobToFile, byteArrayToBinaryString } from './utils/file.utils'
+import { UploadFileResponse } from '../models/UploadFile'
 
 interface XLMapRule {
   XLMAP_ID: string
@@ -145,7 +146,7 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
 
   public afterGetColHeader(column: number, th: any) {
     // Dark mode
-    th.classList.add('darkTH')
+    th.classList.add(globals.handsontable.darkTableHeaderClass)
   }
 
   public xlmapOnClick(xlmap: XLMapListItem) {
@@ -159,7 +160,7 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
       }
       this.selectedTab = Tabs.Rules
       this.viewXLMapRules()
-      this.router.navigateByUrl('/home/files/' + xlmap.id)
+      this.router.navigateByUrl('/home/excel-maps/' + xlmap.id)
     }
   }
 
@@ -340,9 +341,9 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
       .uploadFile(uploadUrl, filesToUpload, {
         table: this.selectedXLMap.targetDS
       })
-      .then((res: any) => {
-        if (res.sasjsAbort) {
-          const abortRes = res
+      .then((res: UploadFileResponse) => {
+        if (res.adapterResponse.sasjsAbort) {
+          const abortRes = res.adapterResponse
           const abortMsg = abortRes.sasjsAbort[0].MSG
           const macMsg = abortRes.sasjsAbort[0].MAC
 
@@ -351,14 +352,14 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
             SYSERRORTEXT: abortRes.SYSERRORTEXT,
             MAC: macMsg
           })
-        } else if (res.sasparams) {
-          const params = res.sasparams[0]
+        } else if (res.adapterResponse.sasparams) {
+          const params = res.adapterResponse.sasparams[0]
           const tableId = params.DSID
           this.router.navigateByUrl('/stage/' + tableId)
         }
       })
       .catch((err: any) => {
-        this.eventService.catchResponseError('file upload', err)
+        this.eventService.catchResponseError('file upload', err.response)
       })
       .finally(() => {
         this.status = Status.ReadyToSubmit

client/src/colors.scss

@@ -1,4 +1,6 @@
 $headerBackground: #314351;
 
 $trackColor: #3b5268;
-$thumbColor: #273849;
+$thumbColor: #273849;
+
+$codeBorder: #e2e2e2;

client/src/styles.scss

@@ -84,6 +84,31 @@ body[cds-theme="light"] {
   line-height: 1.8 !important;
 }
 
+[cds-text=caption_clean] {
+  font-size: var(--cds-global-typography-caption-font-size);
+  font-weight: var(--cds-global-typography-caption-font-weight);
+  line-height: var(--cds-global-typography-caption-line-height);
+  letter-spacing: var(--cds-global-typography-caption-letter-spacing);
+
+  &::after, &::before {
+    display: none;
+  }
+}
+
+.log-wrapper {
+  min-height: 50px;
+  padding: 10px;
+  margin-top: 10px;
+
+  white-space: pre-wrap;
+  border-radius: 3px;
+
+  border: 1px solid $codeBorder;
+
+  height: 48vh;
+  overflow: auto;
+}
+
 // Custom loading spinner
 .slider {
   position: absolute;
@@ -200,6 +225,10 @@ body[cds-theme="light"] {
   width: 100%;
 }
 
+.w-100-i {
+  width: 100% !important;
+}
+
 .w-40 {
   width: 40%;
 }
@@ -288,6 +317,10 @@ body[cds-theme="light"] {
   margin-left: 5px;
 }
 
+.ml-5-i {
+  margin-left: 5px !important;
+}
+
 .ml-10 {
   margin-left: 10px;
 }
@@ -458,6 +491,10 @@ body[cds-theme="light"] {
   pointer-events: none;
 }
 
+.whitespace-nowrap {
+  white-space: nowrap;
+}
+
 .text-center {
   text-align: center;
 }
@@ -784,6 +821,10 @@ clr-icon.is-info {
   box-shadow: none !important;
 }
 
+.htCustomHidden {
+  display: none;
+}
+
 body[cds-theme="dark"] {
   .htDark {
     background: #888;
@@ -1040,7 +1081,6 @@ clr-tree-node {
     padding: 0px 8px 0px 8px;
     width: auto;
     height: auto;
-    display: flex;
     align-items: center;
   }
 

client/tsconfig.worker.json

@@ -0,0 +1,15 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/worker",
+    "lib": [
+      "es2018",
+      "webworker"
+    ],
+    "types": []
+  },
+  "include": [
+    "src/**/*.worker.ts"
+  ]
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "6.10.0",
+  "version": "6.12.0",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/package.json

@@ -29,6 +29,6 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "^4.11.1",
-    "@sasjs/core": "^4.52.4"
+    "@sasjs/core": "^4.52.5"
   }
 }

sas/sasjs/macros/bitemporal_dataloader.sas

@@ -100,6 +100,7 @@ Areas for optimisation
   @li mf_getattrn.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
+  @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
   @li mf_getuser.sas
   @li mf_getvarlist.sas
@@ -621,7 +622,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
 %put DCNOTE: Extracting matching observations from &base_lib..&base_dsn;
 
 %if &engine_type=OLEDB %then %do;
-  %let temp_table=##BITEMP_&base_dsn;
+  %let temp_table=##%mf_getuniquefileref(prefix=BTMP)_&base_dsn;
   %if &loadtype=BITEMPORAL or &loadtype=TXTEMPORAL %then
         %let base_table=(select * from [dbo].&base_dsn
                         where convert(datetime,&SQLNOW) < &tech_to );
@@ -1049,7 +1050,7 @@ run;
   %let cat_string=catx('|' ,&bus_from,&bus_to);
 
   data work.bitemp5a_lkp (keep=&md5_col)
-    %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    %if "%substr(&sysver,1,1)" ne "4" & "%substr(&sysver,1,1)" ne "5" %then %do;
       /nonote2err
     %end;
     ;
@@ -1191,10 +1192,10 @@ run;
   /* if OLEDB then create a temp table for efficiency */
   %local innertable;
   %if &engine_type=OLEDB %then %do;
-    %let innertable=[##BITEMP_&base_dsn];
+    %let innertable=[&temp_table];
     %let top_table=[dbo].&base_dsn;
     %let flexinow=&SQLNOW;
-    create table &base_lib.."##BITEMP_&base_dsn"n as
+    create table &base_lib.."&temp_table"n as
       select * from work.bitemp5d_subquery;
     /* open up a connection for pass through SQL */
     %dc_assignlib(WRITE,&base_lib,passthru=myAlias)

sas/sasjs/services/approvers/gethistory.sas

@@ -105,8 +105,6 @@ create view work.submits as
     ,a.submitted_by_nm  as submitter
     ,put(a.REVIEWED_ON_DTTM,yymmddhhmmss.) as REVIEWED
     ,a.submit_status_cd as status
-    ,a.reviewed_on_dttm
-    ,a.reviewed_by_nm as approver
   from &mpelib..mpe_submit(where=(submit_status_cd ne 'SUBMITTED')) a
 
 %macro gethistory();
@@ -120,8 +118,18 @@ create view work.submits as
 %end;
 %mend gethistory;
 %gethistory()
+;
 
-  order by a.submitted_on_dttm desc;
+/* get latest reason text */
+create table work.reviews as 
+  select a.* 
+    ,b.reviewed_on_dttm
+    ,b.reviewed_by_nm as approver
+    ,b.review_reason_txt 
+  from work.submits a 
+  left join &mpelib..mpe_review b 
+  on a.table_id=b.table_id 
+  order by a.table_id desc, b.reviewed_on_dttm desc;
 
 %mp_abort(iftrue= (&syscc > 0)
   ,mac=&_program
@@ -129,7 +137,9 @@ create view work.submits as
 )
 
 data work.fromsas;
-  set work.submits;
+  set work.reviews;
+  by descending table_id descending reviewed_on_dttm;
+  if first.table_id;
   if _n_ ge &startrow;
   n+1;
   if n>&hist then stop;

sas/sasjs/targets/sas9/services_meta/admin/configurator.sas

@@ -188,6 +188,7 @@ data _null_;
     put "<li> METAPERSON: " metaperson "</li>";
     put "<li> METAUSER: " metauser "</li>";
     put "<li> SYSUSERID: " sysuserid "</li>";
+    put "<li> SYSSITE: &syssite</li>";
     url='/SASStoredProcess/do';
   end;
   else do;

sas/sasjs/targets/sas9/services_meta/lineage/fetchtablelineage.sas

@@ -178,6 +178,66 @@ gray,#808080
 black,#00000
 yellow,#ffe119
 white,#ffffff
+bazaar,#98777b
+beaver,#9f8170
+bisque,#ffe4c4
+blond,#faf0be
+blush,#de5d83
+bole,#79443b
+bone,#e3dac9
+brass,#b5a642
+bronze,#cd7f32
+bubbles,#e7feff
+buff,#f0dc82
+camel,#c19a6b
+canary,#ffff99
+capri,#00bfff
+cardinal,#c41e3a
+carmine,#ff0040
+celadon,#ace1af
+celeste,#b2ffff
+cerise,#de3163
+cerulean,#007ba7
+cherry,#de3163
+chestnut,#cd5c5c
+cinnabar,#e34234
+cinnamon,#d2691e
+citrine,#e4d00a
+cobalt,#0047ab
+coffee,#6f4e37
+copper,#b87333
+coral,#ff7f50
+corn,#fbec5d
+cornsilk,#fff8dc
+cream,#fffdd0
+crimson,#dc143c
+daffodil,#ffff31
+denim,#1560bd
+desert,#c19a6b
+drab,#967117
+ecru,#c2b280
+eggplant,#614051
+eggshell,#f0ead6
+emerald,#50c878
+fallow,#c19a6b
+fawn,#e5aa70
+fern,#71bc78
+flame,#e25822
+flax,#eedc82
+folly,#ff004f
+fuchsia,#ff00ff
+fulvous,#e48400
+gamboge,#e49b0f
+ginger,#b06500
+glaucous,#6082b6
+glitter,#e6e8fa
+gold,#ffd700
+grullo,#a99a86
+icterine,#fcf75e
+indigo,#4b0082
+iris,#5a4fcf
+ivory,#fffff0
+jade,#00a86b
 ;;;;
 run;
 

sas/sasjs/targets/server/services_server/admin/configurator.sas

@@ -222,6 +222,7 @@ data _null_;
   put "<li> SYSUSERID: " sysuserid "</li>";
   put "<li> SYSHOSTNAME: &syshostname</li>";
   put "<li> SYSVLONG: &sysvlong</li>";
+  put "<li> SYSSITE: &syssite</li>";
 run;
 
 data _null_;