chore(release): 6.12.0 [skip ci]

# [6.12.0](https://git.datacontroller.io/dc/dc/compare/v6.11.1...v6.12.0) (2024-09-02)

### Bug Fixes

* added appLoc to the system page ([dd2138a](dd2138ac5e))
* bumping core for passthrough fix, [#124](#124) ([caa9854](caa9854ff0))
* excel with password flow, introducing web worker for XLSX.read ([a3ce367](a3ce367950))
* implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height ([4218da9](4218da91cd))
* **multi load:** xlsx read file ahead of time, while user choose datasets ([6547461](6547461637))
* refactored adapter request wrapper function to return job log as well ([67436f4](67436f4ff9))
* using temporary names for temporary tables ([ce50365](ce503653cd)), closes [#124](#124)

### Features

* searching data in excel files using new algorithm (massive performance improvement) ([bbb725c](bbb725c64c))

CHANGELOG.md

@@ -1,3 +1,21 @@
+# [6.12.0](https://git.datacontroller.io/dc/dc/compare/v6.11.1...v6.12.0) (2024-09-02)
+
+
+### Bug Fixes
+
+* added appLoc to the system page ([dd2138a](https://git.datacontroller.io/dc/dc/commit/dd2138ac5e6067de310e83d16fccc9b9764ba3ff))
+* bumping core for passthrough fix, [#124](https://git.datacontroller.io/dc/dc/issues/124) ([caa9854](https://git.datacontroller.io/dc/dc/commit/caa9854ff0431ccbb6ff1d6d3509dc877362cceb))
+* excel with password flow, introducing web worker for XLSX.read ([a3ce367](https://git.datacontroller.io/dc/dc/commit/a3ce36795007a4e3b6ac3499ffd119dc3758f387))
+* implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height ([4218da9](https://git.datacontroller.io/dc/dc/commit/4218da91cd193aa45346ad7e34ccc00ca89df4fb))
+* **multi load:** xlsx read file ahead of time, while user choose datasets ([6547461](https://git.datacontroller.io/dc/dc/commit/65474616379e1dacc1329b3bdc5eb14f34428bb1))
+* refactored adapter request wrapper function to return job log as well ([67436f4](https://git.datacontroller.io/dc/dc/commit/67436f4ff9bb4d77d5f897f47a3e3d472981f275))
+* using temporary names for temporary tables ([ce50365](https://git.datacontroller.io/dc/dc/commit/ce503653cd9fc36f72fb172bd14816e07c792e14)), closes [#124](https://git.datacontroller.io/dc/dc/issues/124)
+
+
+### Features
+
+* searching data in excel files using new algorithm (massive performance improvement) ([bbb725c](https://git.datacontroller.io/dc/dc/commit/bbb725c64cc23ed701b189623992408c42fdde8f))
+
 ## [6.11.1](https://git.datacontroller.io/dc/dc/compare/v6.11.0...v6.11.1) (2024-07-02)
 
 

client/angular.json

@@ -69,7 +69,8 @@
             ],
             "scripts": [
               "node_modules/marked/marked.min.js"
-            ]
+            ],
+            "webWorkerTsConfig": "tsconfig.worker.json"
           },
           "configurations": {
             "production": {
@@ -148,7 +149,8 @@
               "src/styles.scss"
             ],
             "scripts": [],
-            "karmaConfig": "karma.conf.js"
+            "karmaConfig": "karma.conf.js",
+            "webWorkerTsConfig": "tsconfig.worker.json"
           }
         },
         "lint": {

client/cypress/e2e/excel-multi-load.cy.ts

@@ -192,7 +192,10 @@ const checkIfTreeHasTables = (tables: string[], clickOnNode?: string, callback?:
       }
     }
 
-    if (nodeToClick) cy.get(nodeToClick).trigger('click')
+    if (nodeToClick) {
+      cy.wait(1000)
+      cy.get(nodeToClick).trigger('click')
+    }
 
     if (callback) callback(nodesCorrect)
   })
@@ -201,6 +204,7 @@ const checkIfTreeHasTables = (tables: string[], clickOnNode?: string, callback?:
 const submitTables = () => {
   cy.get('#submit-all').trigger('click')
   cy.get('#submit-tables').trigger('click')
+  cy.wait(1000)
 }
 
 const hasSuccessSubmits = (expectedNoOfSubmits: number, callback: (valid: boolean) => void) => {

client/cypress/e2e/excel.cy.ts

@@ -109,13 +109,8 @@ context('excel tests: ', function () {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
 
     attachExcelFile('duplicate_column_excel.xlsx', () => {
-      cy.get('.abortMsg', { timeout: longerCommandTimeout })
-        .should('exist')
-        .then((elements: any) => {
-          if (elements[0]) {
-            if (elements[0].innerText.toLowerCase().includes('missing')) done()
-          }
-        })
+      submitExcel()
+      rejectExcel(done)
     })
   })
 

client/package-lock.json

@@ -52,6 +52,7 @@
         "tslib": "^2.3.0",
         "vm": "^0.1.0",
         "webpack": "^5.91.0",
+        "xlsx": "^0.18.5",
         "zone.js": "~0.14.4"
       },
       "devDependencies": {
@@ -7122,6 +7123,14 @@
         "node": ">=8.9.0"
       }
     },
+    "node_modules/adler-32": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
+      "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/agent-base": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
@@ -8259,6 +8268,18 @@
       "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
       "dev": true
     },
+    "node_modules/cfb": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
+      "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
+      "dependencies": {
+        "adler-32": "~1.3.0",
+        "crc-32": "~1.2.0"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -8579,6 +8600,14 @@
       "integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==",
       "dev": true
     },
+    "node_modules/codepage": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
+      "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/color-convert": {
       "version": "1.9.3",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@@ -8966,6 +8995,17 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/crc-32": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
+      "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
+      "bin": {
+        "crc32": "bin/crc32.njs"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/create-ecdh": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
@@ -11578,6 +11618,14 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/frac": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
+      "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/fraction.js": {
       "version": "4.3.7",
       "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz",
@@ -19340,6 +19388,17 @@
       "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
       "dev": true
     },
+    "node_modules/ssf": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
+      "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
+      "dependencies": {
+        "frac": "~1.1.2"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/sshpk": {
       "version": "1.18.0",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@@ -21323,6 +21382,22 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/wmf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
+      "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/word": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
+      "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/word-wrap": {
       "version": "1.2.5",
       "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@@ -21469,6 +21544,26 @@
         }
       }
     },
+    "node_modules/xlsx": {
+      "version": "0.18.5",
+      "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
+      "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
+      "dependencies": {
+        "adler-32": "~1.3.0",
+        "cfb": "~1.2.1",
+        "codepage": "~1.15.0",
+        "crc-32": "~1.2.1",
+        "ssf": "~0.11.2",
+        "wmf": "~1.0.1",
+        "word": "~0.3.0"
+      },
+      "bin": {
+        "xlsx": "bin/xlsx.njs"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/xmldoc": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz",

client/package.json

@@ -80,6 +80,7 @@
     "tslib": "^2.3.0",
     "vm": "^0.1.0",
     "webpack": "^5.91.0",
+    "xlsx": "^0.18.5",
     "zone.js": "~0.14.4"
   },
   "devDependencies": {

client/src/app/app.component.ts

@@ -18,6 +18,7 @@ import {
   ClarityIcons,
   exclamationTriangleIcon,
   moonIcon,
+  processOnVmIcon,
   sunIcon,
   tableIcon,
   trashIcon
@@ -28,7 +29,8 @@ ClarityIcons.addIcons(
   sunIcon,
   exclamationTriangleIcon,
   tableIcon,
-  trashIcon
+  trashIcon,
+  processOnVmIcon
 )
 
 @Component({

client/src/app/deploy/sections/manual/manual.component.ts

@@ -1,6 +1,7 @@
 import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core'
 import SASjs, { SASjsConfig } from '@sasjs/adapter'
 import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 import { DeployService } from 'src/app/services/deploy.service'
 import { EventService } from 'src/app/services/event.service'
 import { LoggerService } from 'src/app/services/logger.service'
@@ -303,10 +304,10 @@ export class ManualComponent implements OnInit {
 
     this.sasService
       .request('public/startupservice', null)
-      .then((res: any) => {
-        this.loggerService.log(res)
+      .then((res: RequestWrapperResponse) => {
+        this.loggerService.log(res.adapterResponse)
 
-        if (res.saslibs) {
+        if (res.adapterResponse.saslibs) {
           this.validationState = 'success'
         } else {
           this.validationState = 'error'

client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts

@@ -3,6 +3,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'
 import SASjs, { SASjsConfig } from '@sasjs/adapter'
 import { ServerType } from '@sasjs/utils/types/serverType'
 import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
+import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
 import { SASGroup } from 'src/app/models/sas/public-getgroups.model'
 import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model'
 import { SasService } from 'src/app/services/sas.service'
@@ -68,11 +69,11 @@ export class SasjsConfiguratorComponent implements OnInit {
     this.loading = true
 
     this.sasService.request('usernav/usergroupsbymember', null).then(
-      (res: any) => {
-        this.METAPERSON = res.MF_GETUSER
-        this.SYSUSERID = res.SYSUSERID
-        this.SYSHOSTNAME = res.SYSHOSTNAME
-        this.SYSVLONG = res.SYSVLONG
+      (res: RequestWrapperResponse) => {
+        this.METAPERSON = res.adapterResponse.MF_GETUSER
+        this.SYSUSERID = res.adapterResponse.SYSUSERID
+        this.SYSHOSTNAME = res.adapterResponse.SYSHOSTNAME
+        this.SYSVLONG = res.adapterResponse.SYSVLONG
 
         /*
           We would like to present a default DCPATH (deployment path) to the
@@ -88,12 +89,14 @@ export class SasjsConfiguratorComponent implements OnInit {
         */
         this.dcDirectory =
           this.tmpDirectories[
-            ['L', 'H', 'A', 'S'].includes(res.SYSSCPL.substring(0, 1))
+            ['L', 'H', 'A', 'S'].includes(
+              res.adapterResponse.SYSSCPL.substring(0, 1)
+            )
               ? 'linux'
               : 'windows'
           ]
 
-        this.dcAdminGroupList = res.groups
+        this.dcAdminGroupList = res.adapterResponse.groups
         this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME
 
         this.loading = false

client/src/app/editor/editor.component.ts

@@ -57,10 +57,10 @@ import { LicenceService } from '../services/licence.service'
 import * as numbro from 'numbro'
 import * as languages from 'numbro/dist/languages.min'
 import { FileUploadEncoding } from '../models/FileUploadEncoding'
-import {
-  ParseResult,
-  SpreadsheetService
-} from '../services/spreadsheet.service'
+import { SpreadsheetService } from '../services/spreadsheet.service'
+import { UploadFileResponse } from '../models/UploadFile'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { ParseResult } from '../models/ParseResult.interface'
 
 @Component({
   selector: 'app-editor',
@@ -571,20 +571,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
     this.sasService
       .uploadFile(this.uploadUrl, filesToUpload, { table: this.libds })
       .then(
-        (res: any) => {
-          if (typeof res.sasjsAbort === 'undefined') {
-            if (typeof res.sasparams === 'undefined') {
+        (res: UploadFileResponse) => {
+          if (typeof res.adapterResponse.sasjsAbort === 'undefined') {
+            if (typeof res.adapterResponse.sasparams === 'undefined') {
               return
             } else {
               this.uploadLoading = false
-              let params = res.sasparams[0]
+              let params = res.adapterResponse.sasparams[0]
               this.successEnable = true
               this.tableId = params.DSID
               this.router.navigateByUrl('/stage/' + this.tableId)
             }
           } else {
             // handle succesfull response
-            const abortRes = res
+            const abortRes = res.adapterResponse
             const abortMsg = abortRes.sasjsAbort[0].MSG
             const macMsg = abortRes.sasjsAbort[0].MAC
 
@@ -609,7 +609,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
             this.fileUploadInputCompList.first.nativeElement.value = ''
           }
           this.uploader.queue = []
-          this.eventService.catchResponseError('file upload', err)
+          this.eventService.catchResponseError(
+            'file upload',
+            err.adapterResponse
+          )
         }
       )
   }
@@ -1561,9 +1564,11 @@ export class EditorComponent implements OnInit, AfterViewInit {
             'editors/stagedata',
             this.$dataFormats
           )
-          .then((res: any) => {
-            if (typeof res.sasparams !== 'undefined') {
-              this.router.navigateByUrl('/stage/' + res.sasparams[0].DSID)
+          .then((res: RequestWrapperResponse) => {
+            if (typeof res.adapterResponse.sasparams !== 'undefined') {
+              this.router.navigateByUrl(
+                '/stage/' + res.adapterResponse.sasparams[0].DSID
+              )
 
               return true
             }
@@ -1591,7 +1596,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
             this.disableSubmit = true
             this.submit = false
 
-            let errorText = typeof err === 'string' ? err : JSON.stringify(err)
+            let errorText =
+              typeof err.adapterRespnse === 'string'
+                ? err.adapterRespnse
+                : JSON.stringify(err.adapterRespnse)
 
             this.eventService.showAbortModal(
               'editors/stagedata',
@@ -1983,8 +1991,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
             suppressSuccessAbortModal: true,
             suppressErrorAbortModal: true
           })
-          .then((res: any) => {
-            const colSource = res.dynamic_values.map(
+          .then((res: RequestWrapperResponse) => {
+            const colSource = res.adapterResponse.dynamic_values.map(
               (el: any) => el[this.cellValidationFields.RAW_VALUE]
             )
 
@@ -1998,8 +2006,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
                   ...this.cellValidationSource[validationSourceIndex],
                   row: row,
                   col: column,
-                  values: res.dynamic_values,
-                  extended_values: res.dynamic_extended_values
+                  values: res.adapterResponse.dynamic_values,
+                  extended_values: res.adapterResponse.dynamic_extended_values
                 }
               }
 

client/src/app/group/group.component.ts

@@ -6,6 +6,7 @@ import { ServerType } from '@sasjs/utils/types/serverType'
 import { HelperService } from '../services/helper.service'
 import { SasService } from '../services/sas.service'
 import { globals } from '../_globals'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 @Component({
   selector: 'app-group',
@@ -82,11 +83,13 @@ export class GroupComponent implements OnInit {
               globals.usernav.groupList = groups
             })
         } else {
-          this.sasService.request('public/getgroups', null).then((res: any) => {
-            this.loading = false
-            this.groups = res.groups
-            globals.usernav.groupList = res.groups
-          })
+          this.sasService
+            .request('public/getgroups', null)
+            .then((res: RequestWrapperResponse) => {
+              this.loading = false
+              this.groups = res.adapterResponse.groups
+              globals.usernav.groupList = res.adapterResponse.groups
+            })
         }
       } else {
         this.groups = globals.usernav.groupList
@@ -128,14 +131,15 @@ export class GroupComponent implements OnInit {
           let data = { iwant: [{ groupid: this.paramURI }] }
           this.sasService
             .request('usernav/usermembersbygroup', data)
-            .then((res: any) => {
-              this.groupMembers = res.sasmembers
-              this.groupMemberCount = res.sasmembers.length
-              if (res.sasmembers[0] !== undefined) {
+            .then((res: RequestWrapperResponse) => {
+              this.groupMembers = res.adapterResponse.sasmembers
+              this.groupMemberCount = res.adapterResponse.sasmembers.length
+              if (res.adapterResponse.sasmembers[0] !== undefined) {
                 this.loading = false
-                this.groupUri = res.sasmembers[0].URIMEM || this.paramURI
-                this.groupName = res.sasmembers[0].GROUPNAME
-                this.groupDesc = res.sasmembers[0].GROUPDESC
+                this.groupUri =
+                  res.adapterResponse.sasmembers[0].URIMEM || this.paramURI
+                this.groupName = res.adapterResponse.sasmembers[0].GROUPNAME
+                this.groupDesc = res.adapterResponse.sasmembers[0].GROUPDESC
 
                 if (!this.groupName) {
                   this.groupName = this.paramURI
@@ -202,13 +206,13 @@ export class GroupComponent implements OnInit {
 
       this.sasService
         .request('usernav/usermembersbygroup', data)
-        .then((res: any) => {
+        .then((res: RequestWrapperResponse) => {
           this.loading = false
           this.groupUri = group.GROUPURI
           this.groupName = group.GROUPNAME
           this.groupDesc = group.GROUPDESC
-          this.groupMembers = res.sasmembers
-          this.groupMemberCount = res.sasmembers.length
+          this.groupMembers = res.adapterResponse.sasmembers
+          this.groupMemberCount = res.adapterResponse.sasmembers.length
         })
     }
   }

client/src/app/licensing/licensing.component.ts

@@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'
 import { ActivatedRoute, Router } from '@angular/router'
 import { AppService, LicenceService, SasService } from '../services'
 import { LicenseKeyData } from '../models/LicenseKeyData'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 enum LicenseActions {
   key = 'key',
@@ -116,8 +117,12 @@ export class LicensingComponent implements OnInit {
 
     this.sasService
       .request('admin/registerkey', table)
-      .then((res: any) => {
-        if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
+      .then((res: RequestWrapperResponse) => {
+        if (
+          res.adapterResponse.return &&
+          res.adapterResponse.return[0] &&
+          res.adapterResponse.return[0].MSG === 'SUCCESS'
+        ) {
           location.replace(location.href.split('#')[0])
         }
       })

client/src/app/lineage/lineage.component.ts

@@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
 import * as saveSvg from 'save-svg-as-png'
 import { LoggerService } from '../services/logger.service'
 import { LicenceService } from '../services/licence.service'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 const moment = require('moment')
 
 @Component({
@@ -115,8 +116,8 @@ export class LineageComponent {
 
     await this.sasService
       .request('lineage/getmetacols', libTable)
-      .then((res: any) => {
-        this.columnsList = res.metacols
+      .then((res: RequestWrapperResponse) => {
+        this.columnsList = res.adapterResponse.metacols
         if (this.columnsList && this.columnsList.length > 0) {
           // this.column = this.columnsList[0]['COLURI']
 
@@ -174,8 +175,8 @@ export class LineageComponent {
     let libTable = { SASControlTable: [{ liburi: $event }] }
     await this.sasService
       .request('lineage/getmetatables', libTable)
-      .then((res: any) => {
-        this.tablesList = res.metatables
+      .then((res: RequestWrapperResponse) => {
+        this.tablesList = res.adapterResponse.metatables
 
         if (this.tablesList && this.tablesList.length > 0) {
           library['tables'] = this.tablesList
@@ -295,8 +296,8 @@ export class LineageComponent {
       } else {
         await this.sasService
           .request('public/viewlibs', null)
-          .then((res: any) => {
-            this.libraryList = res.saslibs
+          .then((res: RequestWrapperResponse) => {
+            this.libraryList = res.adapterResponse.saslibs
             this.helperService.displayLibraries(this.libraryList)
 
             if (this.libraryList) {
@@ -402,8 +403,8 @@ export class LineageComponent {
     return new Promise<void>((resolve, reject) => {
       this.sasService
         .request('lineage/fetchtablelineage', libTable)
-        .then(async (res: any) => {
-          if (res.flatdata.length > 0) {
+        .then(async (res: RequestWrapperResponse) => {
+          if (res.adapterResponse.flatdata.length > 0) {
             if (this.licenceService.checkLineageLimit()) {
               this.eventService.showInfoModal(
                 'Notice',
@@ -421,20 +422,22 @@ export class LineageComponent {
           }
 
           this.lineageTableName =
-            res.info[0].LIBREF + '.' + res.info[0].TABLENAME
+            res.adapterResponse.info[0].LIBREF +
+            '.' +
+            res.adapterResponse.info[0].TABLENAME
 
-          let dotArray = res.finalfinal
+          let dotArray = res.adapterResponse.finalfinal
           let vizTmp: string = ''
 
           for (let i = 0; i < dotArray.length; i++) {
             vizTmp += unescape(dotArray[i].LINE) + '\n'
           }
 
-          this.flatdata = res.flatdata
+          this.flatdata = res.adapterResponse.flatdata
 
           if (this.libraryList) {
             let libraryToSelect = this.libraryList.find((library: any) =>
-              res.info[0].LIBURI.toUpperCase().includes(
+              res.adapterResponse.info[0].LIBURI.toUpperCase().includes(
                 library.LIBRARYID.toUpperCase()
               )
             )
@@ -450,7 +453,7 @@ export class LineageComponent {
               if (libraryToSelect['tables']) {
                 tableToSelect = libraryToSelect['tables'].find((table: any) =>
                   table.TABLEURI.toUpperCase().includes(
-                    res.info[0].TABLEID.toUpperCase()
+                    res.adapterResponse.info[0].TABLEID.toUpperCase()
                   )
                 )
 
@@ -495,10 +498,10 @@ export class LineageComponent {
             .replace(/\sds:/g, '\nds:')
             .replace(/\s\n/g, '\n')
 
-          this.idlookup = res.idlookup
+          this.idlookup = res.adapterResponse.idlookup
 
-          if (res.finalfinal.length > this.largeDotFileLimit) {
-            this.largeDotFileLines = res.finalfinal.length
+          if (res.adapterResponse.finalfinal.length > this.largeDotFileLimit) {
+            this.largeDotFileLines = res.adapterResponse.finalfinal.length
           } else {
             this.buildGraph()
           }
@@ -619,8 +622,8 @@ export class LineageComponent {
     return new Promise<void>((resolve, reject) => {
       this.sasService
         .request('lineage/fetchcollineage', libTable)
-        .then(async (res: any) => {
-          if (res.flatdata.length > 0) {
+        .then(async (res: RequestWrapperResponse) => {
+          if (res.adapterResponse.flatdata.length > 0) {
             if (this.licenceService.checkLineageLimit()) {
               this.eventService.showInfoModal(
                 'Notice',
@@ -631,18 +634,21 @@ export class LineageComponent {
             }
           }
 
-          if (typeof res === 'string') {
+          if (typeof res.adapterResponse === 'string') {
             this.vizInput = 'digraph G {SAS Error}'
             this.buildGraph()
             return
           }
 
-          this.lineageTableName = res.info[0].LIBREF + '.' + res.info[0].TABNAME
-          this.lineageColumnName = res.info[0].COLNAME
+          this.lineageTableName =
+            res.adapterResponse.info[0].LIBREF +
+            '.' +
+            res.adapterResponse.info[0].TABNAME
+          this.lineageColumnName = res.adapterResponse.info[0].COLNAME
 
-          this.idlookup = res.idlookup
+          this.idlookup = res.adapterResponse.idlookup
 
-          let dotArray = res.fromsas
+          let dotArray = res.adapterResponse.fromsas
           let vizTmp: string = ''
           for (let i = 0; i < dotArray.length; i++) {
             vizTmp += unescape(dotArray[i].STRING) + '\n'
@@ -653,11 +659,11 @@ export class LineageComponent {
             .replace(/\sds:/g, '\nds:')
             .replace(/\s\n/g, '\n')
 
-          this.flatdata = res.flatdata
+          this.flatdata = res.adapterResponse.flatdata
 
           if (this.libraryList) {
             let libraryToSelect = this.libraryList.find((library: any) =>
-              res.info[0]?.LIBURI?.toUpperCase()?.includes(
+              res.adapterResponse.info[0]?.LIBURI?.toUpperCase()?.includes(
                 library?.LIBRARYID?.toUpperCase()
               )
             )
@@ -672,7 +678,8 @@ export class LineageComponent {
 
               if (libraryToSelect['tables']) {
                 tableToSelect = libraryToSelect['tables'].find(
-                  (table: any) => table.TABLEURI === res.info[0].TABURI
+                  (table: any) =>
+                    table.TABLEURI === res.adapterResponse.info[0].TABURI
                 )
 
                 if (tableToSelect) {
@@ -714,8 +721,8 @@ export class LineageComponent {
             }
           }
 
-          if (res.fromsas.length > this.largeDotFileLimit) {
-            this.largeDotFileLines = res.fromsas.length
+          if (res.adapterResponse.fromsas.length > this.largeDotFileLimit) {
+            this.largeDotFileLines = res.adapterResponse.fromsas.length
           } else {
             this.buildGraph()
           }

client/src/app/metadata/metadata.component.ts

@@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
 import { globals } from '../_globals'
 
 import { Injectable } from '@angular/core'
+import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 
 interface MetaData {
   NAME: any
@@ -109,44 +110,52 @@ export class MetadataComponent implements OnInit {
       this.metatypesLoading = false
       this.metaDataSearch = globals.metadata.metaDataSearch
     } else {
-      this.sasService.request('metanav/metatypes', null).then((res: any) => {
-        this.metaDataList = res.types
-        globals.metadata.metaDataList = this.metaDataList
-        this.loading = false
-        this.metatypesLoading = false
-      })
+      this.sasService
+        .request('metanav/metatypes', null)
+        .then((res: RequestWrapperResponse) => {
+          this.metaDataList = res.adapterResponse.types
+          globals.metadata.metaDataList = this.metaDataList
+          this.loading = false
+          this.metatypesLoading = false
+        })
 
-      this.sasService.request('metanav/metarepos', null).then((res: any) => {
-        let foundation = false
-        this.repositories = []
-        for (let index = 0; index < res.outrepos.length; index++) {
-          this.repositories.push(res.outrepos[index].NAME)
-          if (res.outrepos[index].NAME === 'Foundation') {
-            foundation = true
+      this.sasService
+        .request('metanav/metarepos', null)
+        .then((res: RequestWrapperResponse) => {
+          let foundation = false
+          this.repositories = []
+          for (
+            let index = 0;
+            index < res.adapterResponse.outrepos.length;
+            index++
+          ) {
+            this.repositories.push(res.adapterResponse.outrepos[index].NAME)
+            if (res.adapterResponse.outrepos[index].NAME === 'Foundation') {
+              foundation = true
+            }
           }
-        }
-        if (foundation) {
-          this.repository = 'Foundation'
-        } else {
-          this.repository = res.outrepos[0].NAME
-        }
-        globals.metadata.metaRepositories = this.repositories
-        globals.metadata.selectedRepository = this.repository
-        if (this.objectRoute) {
-          this.eventService.closeSidebar()
-          this.showData = true
-          let name = ''
-          let id = this.route.snapshot.params['objectID']
-          // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
-          this.metaObjectList = []
-          this.metaObjectList.push({ ID: id, NAME: name })
-          this.metaObjectShowList = this.metaObjectList
-          this.metaObjectOnClick(
-            this.metaObjectShowList[0].ID,
-            this.metaObjectShowList[0]
-          )
-        }
-      })
+          if (foundation) {
+            this.repository = 'Foundation'
+          } else {
+            this.repository = res.adapterResponse.outrepos[0].NAME
+          }
+          globals.metadata.metaRepositories = this.repositories
+          globals.metadata.selectedRepository = this.repository
+          if (this.objectRoute) {
+            this.eventService.closeSidebar()
+            this.showData = true
+            let name = ''
+            let id = this.route.snapshot.params['objectID']
+            // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
+            this.metaObjectList = []
+            this.metaObjectList.push({ ID: id, NAME: name })
+            this.metaObjectShowList = this.metaObjectList
+            this.metaObjectOnClick(
+              this.metaObjectShowList[0].ID,
+              this.metaObjectShowList[0]
+            )
+          }
+        })
     }
   }
 
@@ -183,56 +192,64 @@ export class MetadataComponent implements OnInit {
     const data: any = {
       SASControlTable: [{ metatype: $event, repo: this.repository }]
     }
-    this.sasService.request('metanav/metaobjects', data).then((res: any) => {
-      this.metaObjectList = res.objects
-      this.getMetaObjectAttributes(this.metaObjectSize)
-      this.loading = false
-      this.assoTypeSelected = $event
-      this.eventService.closeSidebar()
-      this.showData = true
-    })
+    this.sasService
+      .request('metanav/metaobjects', data)
+      .then((res: RequestWrapperResponse) => {
+        this.metaObjectList = res.adapterResponse.objects
+        this.getMetaObjectAttributes(this.metaObjectSize)
+        this.loading = false
+        this.assoTypeSelected = $event
+        this.eventService.closeSidebar()
+        this.showData = true
+      })
   }
 
   public async selectmetaObject($event: any, metaData?: any) {
     let data: any = {
       SASControlTable: [{ objecturi: $event }]
     }
-    this.sasService.request('metanav/metadetails', data).then((res: any) => {
-      this.metaObjectAssociations = res.associations
-      this.root$ = of(this.getAssosiationsCount(res.associations))
-      this.showAcc = true
-      this.showTable = true
-      let metaObjectName = res.attributes.find(
-        (x: any) => x.NAME === 'Name'
-      ).VALUE
-      this.assoObjectSelected = metaObjectName
-      metaData.NAME = metaObjectName
-      let url = this.router.url
-      if (this.objectRoute) {
-        // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
-        this.location.replaceState(
-          url.slice(0, url.lastIndexOf('object')) +
-            'object/' +
-            $event.slice(1 + $event.indexOf('\\'))
+    this.sasService
+      .request('metanav/metadetails', data)
+      .then((res: RequestWrapperResponse) => {
+        this.metaObjectAssociations = res.adapterResponse.associations
+        this.root$ = of(
+          this.getAssosiationsCount(res.adapterResponse.associations)
         )
-      } else {
-        // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
-        this.location.replaceState(
-          url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
-        )
-      }
-      this.metaObjectAttributes = res.attributes
-    })
+        this.showAcc = true
+        this.showTable = true
+        let metaObjectName = res.adapterResponse.attributes.find(
+          (x: any) => x.NAME === 'Name'
+        ).VALUE
+        this.assoObjectSelected = metaObjectName
+        metaData.NAME = metaObjectName
+        let url = this.router.url
+        if (this.objectRoute) {
+          // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+          this.location.replaceState(
+            url.slice(0, url.lastIndexOf('object')) +
+              'object/' +
+              $event.slice(1 + $event.indexOf('\\'))
+          )
+        } else {
+          // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
+          this.location.replaceState(
+            url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
+          )
+        }
+        this.metaObjectAttributes = res.adapterResponse.attributes
+      })
   }
 
   public async selectAssosiationsDetails($event: any, metaData?: any) {
     let data: any = {
       SASControlTable: [{ objecturi: $event }]
     }
-    this.sasService.request('metanav/metadetails', data).then((res: any) => {
-      this.metaObjectAttributes = res.attributes
-      this.showTable = true
-    })
+    this.sasService
+      .request('metanav/metadetails', data)
+      .then((res: RequestWrapperResponse) => {
+        this.metaObjectAttributes = res.adapterResponse.attributes
+        this.showTable = true
+      })
   }
 
   public getAssosiationsCount(assosiationList: Array<any>) {
@@ -244,7 +261,7 @@ export class MetadataComponent implements OnInit {
           details: []
         })
       }
-      let assocObj = assosiationsHash.get(assosiation.ASSOC)
+      let assocObj: any = assosiationsHash.get(assosiation.ASSOC)
       assocObj.count++
       assocObj.details.push({
         ASSOCURI: assosiation.ASSOCURI,
@@ -254,7 +271,7 @@ export class MetadataComponent implements OnInit {
       })
     }
     let assocGrouped: Array<any> = []
-    assosiationsHash.forEach(function (val, key) {
+    assosiationsHash.forEach(function (val: any, key) {
       assocGrouped.push({
         ASSOC: key,
         count: val.count,
@@ -294,9 +311,9 @@ export class MetadataComponent implements OnInit {
     }
     return this.sasService
       .request('metanav/metadetails', data)
-      .then((res: any) => {
+      .then((res: RequestWrapperResponse) => {
         this.showTable = true
-        this.metaObjectAttributes = res.attributes
+        this.metaObjectAttributes = res.adapterResponse.attributes
         this.assoObjectSelected = asso.NAME
         let url = this.router.url
         if (this.objectRoute) {
@@ -314,7 +331,7 @@ export class MetadataComponent implements OnInit {
               asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\'))
           )
         }
-        return this.getAssosiationsCount(res.associations)
+        return this.getAssosiationsCount(res.adapterResponse.associations)
       })
   }
 

client/src/app/models/ParseParams.interface.ts

@@ -0,0 +1,29 @@
+import { DcValidator } from '../shared/dc-validator/dc-validator'
+import { FileUploadEncoding } from './FileUploadEncoding'
+import { FileUploader } from './FileUploader.class'
+import { ExcelRule } from './TableData'
+import XLSX from 'xlsx'
+
+export interface ParseParams {
+  file: File
+  password?: string
+  dcValidator: DcValidator
+  /**
+   * If workbook is provided, parse function will not run a XLSX.read()
+   * it will use this property instead. So the client must do a file read beforehand
+   */
+  workbook?: XLSX.WorkBook
+  /**
+   * Parse function will manipulate and return the uploader array which can be provided with files already in the queue
+   * Otherwise new empty instance will be created.
+   */
+  uploader?: FileUploader
+  headerPks: string[]
+  headerArray: string[]
+  headerShow: string[]
+  timeHeaders: string[]
+  dateHeaders: string[]
+  dateTimeHeaders: string[]
+  xlRules: ExcelRule[]
+  encoding?: FileUploadEncoding
+}

client/src/app/models/ParseResult.interface.ts

@@ -0,0 +1,15 @@
+import { FileUploader } from './FileUploader.class'
+import FoundRangeInfo from './RangeInfo'
+
+export interface ParseResult {
+  /**
+   * In case of CSV file, won't be returned
+   */
+  data?: any[]
+  /**
+   * In case of CSV file, won't be returned
+   */
+  headerShow?: string[]
+  rangeSheetRes?: FoundRangeInfo
+  uploader: FileUploader
+}

client/src/app/models/RangeInfo.ts

@@ -0,0 +1,13 @@
+export default interface FoundRangeInfo {
+  found: boolean
+  sheetName: string
+  rangeStartAddress: string
+  rangeEndAddress: string
+  rangeAddress: string
+  missingHeaders: MissingHeaders[]
+}
+
+export interface MissingHeaders {
+  sheetName: string
+  missingHeaders: string[]
+}

client/src/app/models/SearchDataExcelResult.interface.ts

@@ -0,0 +1,13 @@
+import { MissingHeaders } from './RangeInfo'
+
+export interface SearchDataExcelResult {
+  missing?: MissingHeaders[]
+  found?: {
+    data: any
+    arrayData: any[]
+    sheetName: string
+    headers: string[]
+    startAddress?: string
+    endAddress?: string
+  }
+}

client/src/app/models/UploadFile.ts

@@ -0,0 +1,4 @@
+export interface UploadFileResponse {
+  adapterResponse: any
+  log?: string
+}

client/src/app/models/request-wrapper/RequestWrapperResponse.ts

@@ -0,0 +1,4 @@
+export interface RequestWrapperResponse<responseType = any> {
+  adapterResponse: responseType
+  log?: string
+}

client/src/app/multi-dataset/multi-dataset.component.html

@@ -22,8 +22,8 @@
     />
   </div>
 
-  <ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
-    <div class="text-center mb-10">
+  <ng-container *ngIf="parsedDatasets.length && !submittedCsvDatasets.length">
+    <div *ngIf="!excelsSubmitted" class="text-center mb-10">
       <button (click)="onDiscard()" class="btn btn-danger btn-sm mr-10">
         Discard
       </button>
@@ -41,30 +41,54 @@
       <clr-tree-node *ngFor="let dataset of parsedDatasets">
         <button
           (click)="onParsedDatasetClick(dataset)"
-          class="clr-treenode-link whitespace-nowrap"
+          class="clr-treenode-link whitespace-nowrap d-flex clr-align-items-center"
           [class.active]="dataset.active"
         >
-          <cds-icon
-            *ngIf="!(dataset.datasource && dataset.parseResult)"
-            status="danger"
-            shape="exclamation-circle"
-          ></cds-icon>
-          <cds-icon
-            *ngIf="dataset.datasource && dataset.parseResult"
-            shape="table"
-          ></cds-icon>
-          {{ dataset.libds }}
+          <ng-container *ngIf="dataset.submitResult">
+            <cds-icon
+              *ngIf="dataset.submitResult.error"
+              status="danger"
+              shape="exclamation-circle"
+            ></cds-icon>
+            <cds-icon
+              *ngIf="dataset.submitResult.success"
+              status="success"
+              shape="check-circle"
+            ></cds-icon>
+          </ng-container>
+
+          <ng-container *ngIf="!dataset.submitResult">
+            <ng-container *ngIf="dataset.datasource">
+              <cds-icon
+                *ngIf="!(dataset.datasource.length && dataset.parseResult)"
+                status="danger"
+                shape="exclamation-circle"
+              ></cds-icon>
+              <cds-icon
+                *ngIf="dataset.datasource.length && dataset.parseResult"
+                shape="table"
+              ></cds-icon>
+            </ng-container>
+
+            <ng-container *ngIf="!dataset.datasource">
+              <cds-icon *ngIf="!dataset.parsingTable" shape="table"></cds-icon>
+
+              <clr-spinner *ngIf="dataset.parsingTable" clrSmall></clr-spinner>
+            </ng-container>
+          </ng-container>
+
+          <span class="ml-5"> {{ dataset.libds }} </span>
         </button>
       </clr-tree-node>
     </clr-tree>
   </ng-container>
 
-  <ng-container *ngIf="submittedDatasets.length">
+  <ng-container *ngIf="submittedCsvDatasets.length">
     <p cds-text="caption" class="ml-10 mb-10 mt-10">Submitted tables:</p>
     <clr-tree