dc/dc
dc/dc
3
1
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases 35 Wiki Activity

Merge pull request 'XLSX Performance Improvements (multi load)' (#118) from multi-load-performance into main
Some checks failed
Release / Build-production-and-ng-test (push) Successful in 3m55s
Details
Release / Build-and-test-development (push) Failing after 8m2s
Details
Release / release (push) Has been skipped
Details

Browse Source 
Reviewed-on: #118
This commit is contained in:
mihajlo 2024-07-26 11:34:20 +00:00
commit 403d08c86a
42 changed files with 2081 additions and 1415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/angular.json
View File

@ -69,7 +69,8 @@
], ],
"scripts": [ "scripts": [
"node_modules/marked/marked.min.js" "node_modules/marked/marked.min.js"
] ],
"webWorkerTsConfig": "tsconfig.worker.json"
}, },
"configurations": { "configurations": {
"production": { "production": {
@ -148,7 +149,8 @@
"src/styles.scss" "src/styles.scss"
], ],
"scripts": [], "scripts": [],
"karmaConfig": "karma.conf.js" "karmaConfig": "karma.conf.js",
"webWorkerTsConfig": "tsconfig.worker.json"
} }
}, },
"lint": { "lint": {

95
client/package-lock.json generated
View File

@ -52,6 +52,7 @@
"tslib": "^2.3.0", "tslib": "^2.3.0",
"vm": "^0.1.0", "vm": "^0.1.0",
"webpack": "^5.91.0", "webpack": "^5.91.0",
"xlsx": "^0.18.5",
"zone.js": "~0.14.4" "zone.js": "~0.14.4"
}, },
"devDependencies": { "devDependencies": {
@ -7122,6 +7123,14 @@
"node": ">=8.9.0" "node": ">=8.9.0"
} }
}, },
"node_modules/adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/agent-base": { "node_modules/agent-base": {
"version": "7.1.1", "version": "7.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
@ -8259,6 +8268,18 @@
"integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
"dev": true "dev": true
}, },
"node_modules/cfb": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
"integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
"dependencies": {
"adler-32": "~1.3.0",
"crc-32": "~1.2.0"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/chalk": { "node_modules/chalk": {
"version": "2.4.2", "version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@ -8579,6 +8600,14 @@
"integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==", "integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==",
"dev": true "dev": true
}, },
"node_modules/codepage": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
"integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/color-convert": { "node_modules/color-convert": {
"version": "1.9.3", "version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@ -8966,6 +8995,17 @@
"js-yaml": "bin/js-yaml.js" "js-yaml": "bin/js-yaml.js"
} }
}, },
"node_modules/crc-32": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
"integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
"bin": {
"crc32": "bin/crc32.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/create-ecdh": { "node_modules/create-ecdh": {
"version": "4.0.4", "version": "4.0.4",
"resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
@ -11578,6 +11618,14 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/frac": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/fraction.js": { "node_modules/fraction.js": {
"version": "4.3.7", "version": "4.3.7",
"resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz",
@ -19340,6 +19388,17 @@
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
"dev": true "dev": true
}, },
"node_modules/ssf": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
"dependencies": {
"frac": "~1.1.2"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/sshpk": { "node_modules/sshpk": {
"version": "1.18.0", "version": "1.18.0",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@ -21323,6 +21382,22 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/wmf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word-wrap": { "node_modules/word-wrap": {
"version": "1.2.5", "version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@ -21469,6 +21544,26 @@
} }
} }
}, },
"node_modules/xlsx": {
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"dependencies": {
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
},
"bin": {
"xlsx": "bin/xlsx.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/xmldoc": { "node_modules/xmldoc": {
"version": "1.3.0", "version": "1.3.0",
"resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz", "resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz",

1
client/package.json
View File

@ -80,6 +80,7 @@
"tslib": "^2.3.0", "tslib": "^2.3.0",
"vm": "^0.1.0", "vm": "^0.1.0",
"webpack": "^5.91.0", "webpack": "^5.91.0",
"xlsx": "^0.18.5",
"zone.js": "~0.14.4" "zone.js": "~0.14.4"
}, },
"devDependencies": { "devDependencies": {

7
client/src/app/deploy/sections/manual/manual.component.ts
View File

@ -1,6 +1,7 @@
import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core' import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core'
import SASjs, { SASjsConfig } from '@sasjs/adapter' import SASjs, { SASjsConfig } from '@sasjs/adapter'
import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings' import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
import { DeployService } from 'src/app/services/deploy.service' import { DeployService } from 'src/app/services/deploy.service'
import { EventService } from 'src/app/services/event.service' import { EventService } from 'src/app/services/event.service'
import { LoggerService } from 'src/app/services/logger.service' import { LoggerService } from 'src/app/services/logger.service'
@ -303,10 +304,10 @@ export class ManualComponent implements OnInit {
this.sasService this.sasService
.request('public/startupservice', null) .request('public/startupservice', null)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loggerService.log(res) this.loggerService.log(res.adapterResponse)
if (res.saslibs) { if (res.adapterResponse.saslibs) {
this.validationState = 'success' this.validationState = 'success'
} else { } else {
this.validationState = 'error' this.validationState = 'error'

17
client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts
View File

@ -3,6 +3,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'
import SASjs, { SASjsConfig } from '@sasjs/adapter' import SASjs, { SASjsConfig } from '@sasjs/adapter'
import { ServerType } from '@sasjs/utils/types/serverType' import { ServerType } from '@sasjs/utils/types/serverType'
import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings' import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
import { SASGroup } from 'src/app/models/sas/public-getgroups.model' import { SASGroup } from 'src/app/models/sas/public-getgroups.model'
import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model' import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model'
import { SasService } from 'src/app/services/sas.service' import { SasService } from 'src/app/services/sas.service'
@ -68,11 +69,11 @@ export class SasjsConfiguratorComponent implements OnInit {
this.loading = true this.loading = true
this.sasService.request('usernav/usergroupsbymember', null).then( this.sasService.request('usernav/usergroupsbymember', null).then(
(res: any) => { (res: RequestWrapperResponse) => {
this.METAPERSON = res.MF_GETUSER this.METAPERSON = res.adapterResponse.MF_GETUSER
this.SYSUSERID = res.SYSUSERID this.SYSUSERID = res.adapterResponse.SYSUSERID
this.SYSHOSTNAME = res.SYSHOSTNAME this.SYSHOSTNAME = res.adapterResponse.SYSHOSTNAME
this.SYSVLONG = res.SYSVLONG this.SYSVLONG = res.adapterResponse.SYSVLONG
/* /*
We would like to present a default DCPATH (deployment path) to the We would like to present a default DCPATH (deployment path) to the
@ -88,12 +89,14 @@ export class SasjsConfiguratorComponent implements OnInit {
*/ */
this.dcDirectory = this.dcDirectory =
this.tmpDirectories[ this.tmpDirectories[
['L', 'H', 'A', 'S'].includes(res.SYSSCPL.substring(0, 1)) ['L', 'H', 'A', 'S'].includes(
res.adapterResponse.SYSSCPL.substring(0, 1)
)
? 'linux' ? 'linux'
: 'windows' : 'windows'
] ]
this.dcAdminGroupList = res.groups this.dcAdminGroupList = res.adapterResponse.groups
this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME
this.loading = false this.loading = false

44
client/src/app/editor/editor.component.ts
View File

@ -57,10 +57,10 @@ import { LicenceService } from '../services/licence.service'
import * as numbro from 'numbro' import * as numbro from 'numbro'
import * as languages from 'numbro/dist/languages.min' import * as languages from 'numbro/dist/languages.min'
import { FileUploadEncoding } from '../models/FileUploadEncoding' import { FileUploadEncoding } from '../models/FileUploadEncoding'
import { import { SpreadsheetService } from '../services/spreadsheet.service'
ParseResult, import { UploadFileResponse } from '../models/UploadFile'
SpreadsheetService import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
} from '../services/spreadsheet.service' import { ParseResult } from '../models/ParseResult.interface'
@Component({ @Component({
selector: 'app-editor', selector: 'app-editor',
@ -571,20 +571,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.sasService this.sasService
.uploadFile(this.uploadUrl, filesToUpload, { table: this.libds }) .uploadFile(this.uploadUrl, filesToUpload, { table: this.libds })
.then( .then(
(res: any) => { (res: UploadFileResponse) => {
if (typeof res.sasjsAbort === 'undefined') { if (typeof res.adapterResponse.sasjsAbort === 'undefined') {
if (typeof res.sasparams === 'undefined') { if (typeof res.adapterResponse.sasparams === 'undefined') {
return return
} else { } else {
this.uploadLoading = false this.uploadLoading = false
let params = res.sasparams[0] let params = res.adapterResponse.sasparams[0]
this.successEnable = true this.successEnable = true
this.tableId = params.DSID this.tableId = params.DSID
this.router.navigateByUrl('/stage/' + this.tableId) this.router.navigateByUrl('/stage/' + this.tableId)
} }
} else { } else {
// handle succesfull response // handle succesfull response
const abortRes = res const abortRes = res.adapterResponse
const abortMsg = abortRes.sasjsAbort[0].MSG const abortMsg = abortRes.sasjsAbort[0].MSG
const macMsg = abortRes.sasjsAbort[0].MAC const macMsg = abortRes.sasjsAbort[0].MAC
@ -609,7 +609,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.fileUploadInputCompList.first.nativeElement.value = '' this.fileUploadInputCompList.first.nativeElement.value = ''
} }
this.uploader.queue = [] this.uploader.queue = []
this.eventService.catchResponseError('file upload', err) this.eventService.catchResponseError(
'file upload',
err.adapterResponse
)
} }
) )
} }
@ -1561,9 +1564,11 @@ export class EditorComponent implements OnInit, AfterViewInit {
'editors/stagedata', 'editors/stagedata',
this.$dataFormats this.$dataFormats
) )
.then((res: any) => { .then((res: RequestWrapperResponse) => {
if (typeof res.sasparams !== 'undefined') { if (typeof res.adapterResponse.sasparams !== 'undefined') {
this.router.navigateByUrl('/stage/' + res.sasparams[0].DSID) this.router.navigateByUrl(
'/stage/' + res.adapterResponse.sasparams[0].DSID
)
return true return true
} }
@ -1591,7 +1596,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.disableSubmit = true this.disableSubmit = true
this.submit = false this.submit = false
let errorText = typeof err === 'string' ? err : JSON.stringify(err) let errorText =
typeof err.adapterRespnse === 'string'
? err.adapterRespnse
: JSON.stringify(err.adapterRespnse)
this.eventService.showAbortModal( this.eventService.showAbortModal(
'editors/stagedata', 'editors/stagedata',
@ -1983,8 +1991,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
suppressSuccessAbortModal: true, suppressSuccessAbortModal: true,
suppressErrorAbortModal: true suppressErrorAbortModal: true
}) })
.then((res: any) => { .then((res: RequestWrapperResponse) => {
const colSource = res.dynamic_values.map( const colSource = res.adapterResponse.dynamic_values.map(
(el: any) => el[this.cellValidationFields.RAW_VALUE] (el: any) => el[this.cellValidationFields.RAW_VALUE]
) )
@ -1998,8 +2006,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
...this.cellValidationSource[validationSourceIndex], ...this.cellValidationSource[validationSourceIndex],
row: row, row: row,
col: column, col: column,
values: res.dynamic_values, values: res.adapterResponse.dynamic_values,
extended_values: res.dynamic_extended_values extended_values: res.adapterResponse.dynamic_extended_values
} }
} }

34
client/src/app/group/group.component.ts
View File

@ -6,6 +6,7 @@ import { ServerType } from '@sasjs/utils/types/serverType'
import { HelperService } from '../services/helper.service' import { HelperService } from '../services/helper.service'
import { SasService } from '../services/sas.service' import { SasService } from '../services/sas.service'
import { globals } from '../_globals' import { globals } from '../_globals'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-group', selector: 'app-group',
@ -82,11 +83,13 @@ export class GroupComponent implements OnInit {
globals.usernav.groupList = groups globals.usernav.groupList = groups
}) })
} else { } else {
this.sasService.request('public/getgroups', null).then((res: any) => { this.sasService
this.loading = false .request('public/getgroups', null)
this.groups = res.groups .then((res: RequestWrapperResponse) => {
globals.usernav.groupList = res.groups this.loading = false
}) this.groups = res.adapterResponse.groups
globals.usernav.groupList = res.adapterResponse.groups
})
} }
} else { } else {
this.groups = globals.usernav.groupList this.groups = globals.usernav.groupList
@ -128,14 +131,15 @@ export class GroupComponent implements OnInit {
let data = { iwant: [{ groupid: this.paramURI }] } let data = { iwant: [{ groupid: this.paramURI }] }
this.sasService this.sasService
.request('usernav/usermembersbygroup', data) .request('usernav/usermembersbygroup', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.groupMembers = res.sasmembers this.groupMembers = res.adapterResponse.sasmembers
this.groupMemberCount = res.sasmembers.length this.groupMemberCount = res.adapterResponse.sasmembers.length
if (res.sasmembers[0] !== undefined) { if (res.adapterResponse.sasmembers[0] !== undefined) {
this.loading = false this.loading = false
this.groupUri = res.sasmembers[0].URIMEM || this.paramURI this.groupUri =
this.groupName = res.sasmembers[0].GROUPNAME res.adapterResponse.sasmembers[0].URIMEM || this.paramURI
this.groupDesc = res.sasmembers[0].GROUPDESC this.groupName = res.adapterResponse.sasmembers[0].GROUPNAME
this.groupDesc = res.adapterResponse.sasmembers[0].GROUPDESC
if (!this.groupName) { if (!this.groupName) {
this.groupName = this.paramURI this.groupName = this.paramURI
@ -202,13 +206,13 @@ export class GroupComponent implements OnInit {
this.sasService this.sasService
.request('usernav/usermembersbygroup', data) .request('usernav/usermembersbygroup', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loading = false this.loading = false
this.groupUri = group.GROUPURI this.groupUri = group.GROUPURI
this.groupName = group.GROUPNAME this.groupName = group.GROUPNAME
this.groupDesc = group.GROUPDESC this.groupDesc = group.GROUPDESC
this.groupMembers = res.sasmembers this.groupMembers = res.adapterResponse.sasmembers
this.groupMemberCount = res.sasmembers.length this.groupMemberCount = res.adapterResponse.sasmembers.length
}) })
} }
} }

9
client/src/app/licensing/licensing.component.ts
View File

@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'
import { ActivatedRoute, Router } from '@angular/router' import { ActivatedRoute, Router } from '@angular/router'
import { AppService, LicenceService, SasService } from '../services' import { AppService, LicenceService, SasService } from '../services'
import { LicenseKeyData } from '../models/LicenseKeyData' import { LicenseKeyData } from '../models/LicenseKeyData'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
enum LicenseActions { enum LicenseActions {
key = 'key', key = 'key',
@ -116,8 +117,12 @@ export class LicensingComponent implements OnInit {
this.sasService this.sasService
.request('admin/registerkey', table) .request('admin/registerkey', table)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') { if (
res.adapterResponse.return &&
res.adapterResponse.return[0] &&
res.adapterResponse.return[0].MSG === 'SUCCESS'
) {
location.replace(location.href.split('#')[0]) location.replace(location.href.split('#')[0])
} }
}) })

63
client/src/app/lineage/lineage.component.ts
View File

@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
import * as saveSvg from 'save-svg-as-png' import * as saveSvg from 'save-svg-as-png'
import { LoggerService } from '../services/logger.service' import { LoggerService } from '../services/logger.service'
import { LicenceService } from '../services/licence.service' import { LicenceService } from '../services/licence.service'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
const moment = require('moment') const moment = require('moment')
@Component({ @Component({
@ -115,8 +116,8 @@ export class LineageComponent {
await this.sasService await this.sasService
.request('lineage/getmetacols', libTable) .request('lineage/getmetacols', libTable)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.columnsList = res.metacols this.columnsList = res.adapterResponse.metacols
if (this.columnsList && this.columnsList.length > 0) { if (this.columnsList && this.columnsList.length > 0) {
// this.column = this.columnsList[0]['COLURI'] // this.column = this.columnsList[0]['COLURI']
@ -174,8 +175,8 @@ export class LineageComponent {
let libTable = { SASControlTable: [{ liburi: $event }] } let libTable = { SASControlTable: [{ liburi: $event }] }
await this.sasService await this.sasService
.request('lineage/getmetatables', libTable) .request('lineage/getmetatables', libTable)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.tablesList = res.metatables this.tablesList = res.adapterResponse.metatables
if (this.tablesList && this.tablesList.length > 0) { if (this.tablesList && this.tablesList.length > 0) {
library['tables'] = this.tablesList library['tables'] = this.tablesList
@ -295,8 +296,8 @@ export class LineageComponent {
} else { } else {
await this.sasService await this.sasService
.request('public/viewlibs', null) .request('public/viewlibs', null)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.libraryList = res.saslibs this.libraryList = res.adapterResponse.saslibs
this.helperService.displayLibraries(this.libraryList) this.helperService.displayLibraries(this.libraryList)
if (this.libraryList) { if (this.libraryList) {
@ -402,8 +403,8 @@ export class LineageComponent {
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
this.sasService this.sasService
.request('lineage/fetchtablelineage', libTable) .request('lineage/fetchtablelineage', libTable)
.then(async (res: any) => { .then(async (res: RequestWrapperResponse) => {
if (res.flatdata.length > 0) { if (res.adapterResponse.flatdata.length > 0) {
if (this.licenceService.checkLineageLimit()) { if (this.licenceService.checkLineageLimit()) {
this.eventService.showInfoModal( this.eventService.showInfoModal(
'Notice', 'Notice',
@ -421,20 +422,22 @@ export class LineageComponent {
} }
this.lineageTableName = this.lineageTableName =
res.info[0].LIBREF + '.' + res.info[0].TABLENAME res.adapterResponse.info[0].LIBREF +
'.' +
res.adapterResponse.info[0].TABLENAME
let dotArray = res.finalfinal let dotArray = res.adapterResponse.finalfinal
let vizTmp: string = '' let vizTmp: string = ''
for (let i = 0; i < dotArray.length; i++) { for (let i = 0; i < dotArray.length; i++) {
vizTmp += unescape(dotArray[i].LINE) + '\n' vizTmp += unescape(dotArray[i].LINE) + '\n'
} }
this.flatdata = res.flatdata this.flatdata = res.adapterResponse.flatdata
if (this.libraryList) { if (this.libraryList) {
let libraryToSelect = this.libraryList.find((library: any) => let libraryToSelect = this.libraryList.find((library: any) =>
res.info[0].LIBURI.toUpperCase().includes( res.adapterResponse.info[0].LIBURI.toUpperCase().includes(
library.LIBRARYID.toUpperCase() library.LIBRARYID.toUpperCase()
) )
) )
@ -450,7 +453,7 @@ export class LineageComponent {
if (libraryToSelect['tables']) { if (libraryToSelect['tables']) {
tableToSelect = libraryToSelect['tables'].find((table: any) => tableToSelect = libraryToSelect['tables'].find((table: any) =>
table.TABLEURI.toUpperCase().includes( table.TABLEURI.toUpperCase().includes(
res.info[0].TABLEID.toUpperCase() res.adapterResponse.info[0].TABLEID.toUpperCase()
) )
) )
@ -495,10 +498,10 @@ export class LineageComponent {
.replace(/\sds:/g, '\nds:') .replace(/\sds:/g, '\nds:')
.replace(/\s\n/g, '\n') .replace(/\s\n/g, '\n')
this.idlookup = res.idlookup this.idlookup = res.adapterResponse.idlookup
if (res.finalfinal.length > this.largeDotFileLimit) { if (res.adapterResponse.finalfinal.length > this.largeDotFileLimit) {
this.largeDotFileLines = res.finalfinal.length this.largeDotFileLines = res.adapterResponse.finalfinal.length
} else { } else {
this.buildGraph() this.buildGraph()
} }
@ -619,8 +622,8 @@ export class LineageComponent {
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
this.sasService this.sasService
.request('lineage/fetchcollineage', libTable) .request('lineage/fetchcollineage', libTable)
.then(async (res: any) => { .then(async (res: RequestWrapperResponse) => {
if (res.flatdata.length > 0) { if (res.adapterResponse.flatdata.length > 0) {
if (this.licenceService.checkLineageLimit()) { if (this.licenceService.checkLineageLimit()) {
this.eventService.showInfoModal( this.eventService.showInfoModal(
'Notice', 'Notice',
@ -631,18 +634,21 @@ export class LineageComponent {
} }
} }
if (typeof res === 'string') { if (typeof res.adapterResponse === 'string') {
this.vizInput = 'digraph G {SAS Error}' this.vizInput = 'digraph G {SAS Error}'
this.buildGraph() this.buildGraph()
return return
} }
this.lineageTableName = res.info[0].LIBREF + '.' + res.info[0].TABNAME this.lineageTableName =
this.lineageColumnName = res.info[0].COLNAME res.adapterResponse.info[0].LIBREF +
'.' +
res.adapterResponse.info[0].TABNAME
this.lineageColumnName = res.adapterResponse.info[0].COLNAME
this.idlookup = res.idlookup this.idlookup = res.adapterResponse.idlookup
let dotArray = res.fromsas let dotArray = res.adapterResponse.fromsas
let vizTmp: string = '' let vizTmp: string = ''
for (let i = 0; i < dotArray.length; i++) { for (let i = 0; i < dotArray.length; i++) {
vizTmp += unescape(dotArray[i].STRING) + '\n' vizTmp += unescape(dotArray[i].STRING) + '\n'
@ -653,11 +659,11 @@ export class LineageComponent {
.replace(/\sds:/g, '\nds:') .replace(/\sds:/g, '\nds:')
.replace(/\s\n/g, '\n') .replace(/\s\n/g, '\n')
this.flatdata = res.flatdata this.flatdata = res.adapterResponse.flatdata
if (this.libraryList) { if (this.libraryList) {
let libraryToSelect = this.libraryList.find((library: any) => let libraryToSelect = this.libraryList.find((library: any) =>
res.info[0]?.LIBURI?.toUpperCase()?.includes( res.adapterResponse.info[0]?.LIBURI?.toUpperCase()?.includes(
library?.LIBRARYID?.toUpperCase() library?.LIBRARYID?.toUpperCase()
) )
) )
@ -672,7 +678,8 @@ export class LineageComponent {
if (libraryToSelect['tables']) { if (libraryToSelect['tables']) {
tableToSelect = libraryToSelect['tables'].find( tableToSelect = libraryToSelect['tables'].find(
(table: any) => table.TABLEURI === res.info[0].TABURI (table: any) =>
table.TABLEURI === res.adapterResponse.info[0].TABURI
) )
if (tableToSelect) { if (tableToSelect) {
@ -714,8 +721,8 @@ export class LineageComponent {
} }
} }
if (res.fromsas.length > this.largeDotFileLimit) { if (res.adapterResponse.fromsas.length > this.largeDotFileLimit) {
this.largeDotFileLines = res.fromsas.length this.largeDotFileLines = res.adapterResponse.fromsas.length
} else { } else {
this.buildGraph() this.buildGraph()
} }

173
client/src/app/metadata/metadata.component.ts
View File

@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
import { globals } from '../_globals' import { globals } from '../_globals'
import { Injectable } from '@angular/core' import { Injectable } from '@angular/core'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
interface MetaData { interface MetaData {
NAME: any NAME: any
@ -109,44 +110,52 @@ export class MetadataComponent implements OnInit {
this.metatypesLoading = false this.metatypesLoading = false
this.metaDataSearch = globals.metadata.metaDataSearch this.metaDataSearch = globals.metadata.metaDataSearch
} else { } else {
this.sasService.request('metanav/metatypes', null).then((res: any) => { this.sasService
this.metaDataList = res.types .request('metanav/metatypes', null)
globals.metadata.metaDataList = this.metaDataList .then((res: RequestWrapperResponse) => {
this.loading = false this.metaDataList = res.adapterResponse.types
this.metatypesLoading = false globals.metadata.metaDataList = this.metaDataList
}) this.loading = false
this.metatypesLoading = false
})
this.sasService.request('metanav/metarepos', null).then((res: any) => { this.sasService
let foundation = false .request('metanav/metarepos', null)
this.repositories = [] .then((res: RequestWrapperResponse) => {
for (let index = 0; index < res.outrepos.length; index++) { let foundation = false
this.repositories.push(res.outrepos[index].NAME) this.repositories = []
if (res.outrepos[index].NAME === 'Foundation') { for (
foundation = true let index = 0;
index < res.adapterResponse.outrepos.length;
index++
) {
this.repositories.push(res.adapterResponse.outrepos[index].NAME)
if (res.adapterResponse.outrepos[index].NAME === 'Foundation') {
foundation = true
}
} }
} if (foundation) {
if (foundation) { this.repository = 'Foundation'
this.repository = 'Foundation' } else {
} else { this.repository = res.adapterResponse.outrepos[0].NAME
this.repository = res.outrepos[0].NAME }
} globals.metadata.metaRepositories = this.repositories
globals.metadata.metaRepositories = this.repositories globals.metadata.selectedRepository = this.repository
globals.metadata.selectedRepository = this.repository if (this.objectRoute) {
if (this.objectRoute) { this.eventService.closeSidebar()
this.eventService.closeSidebar() this.showData = true
this.showData = true let name = ''
let name = '' let id = this.route.snapshot.params['objectID']
let id = this.route.snapshot.params['objectID'] // let temp = this.router.url.split("%20").join(" ").split("/").reverse();
// let temp = this.router.url.split("%20").join(" ").split("/").reverse(); this.metaObjectList = []
this.metaObjectList = [] this.metaObjectList.push({ ID: id, NAME: name })
this.metaObjectList.push({ ID: id, NAME: name }) this.metaObjectShowList = this.metaObjectList
this.metaObjectShowList = this.metaObjectList this.metaObjectOnClick(
this.metaObjectOnClick( this.metaObjectShowList[0].ID,
this.metaObjectShowList[0].ID, this.metaObjectShowList[0]
this.metaObjectShowList[0] )
) }
} })
})
} }
} }
@ -183,56 +192,64 @@ export class MetadataComponent implements OnInit {
const data: any = { const data: any = {
SASControlTable: [{ metatype: $event, repo: this.repository }] SASControlTable: [{ metatype: $event, repo: this.repository }]
} }
this.sasService.request('metanav/metaobjects', data).then((res: any) => { this.sasService
this.metaObjectList = res.objects .request('metanav/metaobjects', data)
this.getMetaObjectAttributes(this.metaObjectSize) .then((res: RequestWrapperResponse) => {
this.loading = false this.metaObjectList = res.adapterResponse.objects
this.assoTypeSelected = $event this.getMetaObjectAttributes(this.metaObjectSize)
this.eventService.closeSidebar() this.loading = false
this.showData = true this.assoTypeSelected = $event
}) this.eventService.closeSidebar()
this.showData = true
})
} }
public async selectmetaObject($event: any, metaData?: any) { public async selectmetaObject($event: any, metaData?: any) {
let data: any = { let data: any = {
SASControlTable: [{ objecturi: $event }] SASControlTable: [{ objecturi: $event }]
} }
this.sasService.request('metanav/metadetails', data).then((res: any) => { this.sasService
this.metaObjectAssociations = res.associations .request('metanav/metadetails', data)
this.root$ = of(this.getAssosiationsCount(res.associations)) .then((res: RequestWrapperResponse) => {
this.showAcc = true this.metaObjectAssociations = res.adapterResponse.associations
this.showTable = true this.root$ = of(
let metaObjectName = res.attributes.find( this.getAssosiationsCount(res.adapterResponse.associations)
(x: any) => x.NAME === 'Name'
).VALUE
this.assoObjectSelected = metaObjectName
metaData.NAME = metaObjectName
let url = this.router.url
if (this.objectRoute) {
// this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
this.location.replaceState(
url.slice(0, url.lastIndexOf('object')) +
'object/' +
$event.slice(1 + $event.indexOf('\\'))
) )
} else { this.showAcc = true
// this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME)); this.showTable = true
this.location.replaceState( let metaObjectName = res.adapterResponse.attributes.find(
url + '/object/' + $event.slice(1 + $event.indexOf('\\')) (x: any) => x.NAME === 'Name'
) ).VALUE
} this.assoObjectSelected = metaObjectName
this.metaObjectAttributes = res.attributes metaData.NAME = metaObjectName
}) let url = this.router.url
if (this.objectRoute) {
// this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
this.location.replaceState(
url.slice(0, url.lastIndexOf('object')) +
'object/' +
$event.slice(1 + $event.indexOf('\\'))
)
} else {
// this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME));
this.location.replaceState(
url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
)
}
this.metaObjectAttributes = res.adapterResponse.attributes
})
} }
public async selectAssosiationsDetails($event: any, metaData?: any) { public async selectAssosiationsDetails($event: any, metaData?: any) {
let data: any = { let data: any = {
SASControlTable: [{ objecturi: $event }] SASControlTable: [{ objecturi: $event }]
} }
this.sasService.request('metanav/metadetails', data).then((res: any) => { this.sasService
this.metaObjectAttributes = res.attributes .request('metanav/metadetails', data)
this.showTable = true .then((res: RequestWrapperResponse) => {
}) this.metaObjectAttributes = res.adapterResponse.attributes
this.showTable = true
})
} }
public getAssosiationsCount(assosiationList: Array<any>) { public getAssosiationsCount(assosiationList: Array<any>) {
@ -244,7 +261,7 @@ export class MetadataComponent implements OnInit {
details: [] details: []
}) })
} }
let assocObj = assosiationsHash.get(assosiation.ASSOC) let assocObj: any = assosiationsHash.get(assosiation.ASSOC)
assocObj.count++ assocObj.count++
assocObj.details.push({ assocObj.details.push({
ASSOCURI: assosiation.ASSOCURI, ASSOCURI: assosiation.ASSOCURI,
@ -254,7 +271,7 @@ export class MetadataComponent implements OnInit {
}) })
} }
let assocGrouped: Array<any> = [] let assocGrouped: Array<any> = []
assosiationsHash.forEach(function (val, key) { assosiationsHash.forEach(function (val: any, key) {
assocGrouped.push({ assocGrouped.push({
ASSOC: key, ASSOC: key,
count: val.count, count: val.count,
@ -294,9 +311,9 @@ export class MetadataComponent implements OnInit {
} }
return this.sasService return this.sasService
.request('metanav/metadetails', data) .request('metanav/metadetails', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.showTable = true this.showTable = true
this.metaObjectAttributes = res.attributes this.metaObjectAttributes = res.adapterResponse.attributes
this.assoObjectSelected = asso.NAME this.assoObjectSelected = asso.NAME
let url = this.router.url let url = this.router.url
if (this.objectRoute) { if (this.objectRoute) {
@ -314,7 +331,7 @@ export class MetadataComponent implements OnInit {
asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\')) asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\'))
) )
} }
return this.getAssosiationsCount(res.associations) return this.getAssosiationsCount(res.adapterResponse.associations)
}) })
} }

23
client/src/app/models/ParseParams.interface.ts Normal file
View File

@ -0,0 +1,23 @@
import { DcValidator } from '../shared/dc-validator/dc-validator'
import { FileUploadEncoding } from './FileUploadEncoding'
import { FileUploader } from './FileUploader.class'
import { ExcelRule } from './TableData'
export interface ParseParams {
file: File
password?: string
dcValidator: DcValidator
/**
* Parse function will manipulate and return the uploader array which can be provided with files already in the queue
* Otherwise new empty instance will be created.
*/
uploader?: FileUploader
headerPks: string[]
headerArray: string[]
headerShow: string[]
timeHeaders: string[]
dateHeaders: string[]
dateTimeHeaders: string[]
xlRules: ExcelRule[]
encoding?: FileUploadEncoding
}

15
client/src/app/models/ParseResult.interface.ts Normal file
View File

@ -0,0 +1,15 @@
import { FileUploader } from './FileUploader.class'
import SheetInfo from './SheetInfo'
export interface ParseResult {
/**
* In case of CSV file, won't be returned
*/
data?: any[]
/**
* In case of CSV file, won't be returned
*/
headerShow?: string[]
rangeSheetRes?: SheetInfo
uploader: FileUploader
}

4
client/src/app/models/UploadFile.ts Normal file
View File

@ -0,0 +1,4 @@
export interface UploadFileResponse {
adapterResponse: any
log?: string
}

0
client/src/app/models/RequestWrapperOptions.ts → client/src/app/models/request-wrapper/RequestWrapperOptions.ts
View File

4
client/src/app/models/request-wrapper/RequestWrapperResponse.ts Normal file
View File

@ -0,0 +1,4 @@
export interface RequestWrapperResponse<responseType = any> {
adapterResponse: responseType
log?: string
}

191
client/src/app/multi-dataset/multi-dataset.component.html
View File

@ -22,8 +22,8 @@
/> />
</div> </div>
<ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length"> <ng-container *ngIf="parsedDatasets.length && !submittedCsvDatasets.length">
<div class="text-center mb-10"> <div *ngIf="!excelsSubmitted" class="text-center mb-10">
<button (click)="onDiscard()" class="btn btn-danger btn-sm mr-10"> <button (click)="onDiscard()" class="btn btn-danger btn-sm mr-10">
Discard Discard
</button> </button>
@ -41,30 +41,54 @@
<clr-tree-node *ngFor="let dataset of parsedDatasets"> <clr-tree-node *ngFor="let dataset of parsedDatasets">
<button <button
(click)="onParsedDatasetClick(dataset)" (click)="onParsedDatasetClick(dataset)"
class="clr-treenode-link whitespace-nowrap" class="clr-treenode-link whitespace-nowrap d-flex clr-align-items-center"
[class.active]="dataset.active" [class.active]="dataset.active"
> >
<cds-icon <ng-container *ngIf="dataset.submitResult">
*ngIf="!(dataset.datasource && dataset.parseResult)" <cds-icon
status="danger" *ngIf="dataset.submitResult.error"
shape="exclamation-circle" status="danger"
></cds-icon> shape="exclamation-circle"
<cds-icon ></cds-icon>
*ngIf="dataset.datasource && dataset.parseResult" <cds-icon
shape="table" *ngIf="dataset.submitResult.success"
></cds-icon> status="success"
{{ dataset.libds }} shape="check-circle"
></cds-icon>
</ng-container>
<ng-container *ngIf="!dataset.submitResult">
<ng-container *ngIf="dataset.datasource">
<cds-icon
*ngIf="!(dataset.datasource.length && dataset.parseResult)"
status="danger"
shape="exclamation-circle"
></cds-icon>
<cds-icon
*ngIf="dataset.datasource.length && dataset.parseResult"
shape="table"
></cds-icon>
</ng-container>
<ng-container *ngIf="!dataset.datasource">
<cds-icon *ngIf="!dataset.parsingTable" shape="table"></cds-icon>
<clr-spinner *ngIf="dataset.parsingTable" clrSmall></clr-spinner>
</ng-container>
</ng-container>
<span class="ml-5"> {{ dataset.libds }} </span>
</button> </button>
</clr-tree-node> </clr-tree-node>
</clr-tree> </clr-tree>
</ng-container> </ng-container>
<ng-container *ngIf="submittedDatasets.length"> <ng-container *ngIf="submittedCsvDatasets.length">
<p cds-text="caption" class="ml-10 mb-10 mt-10">Submitted tables:</p> <p cds-text="caption" class="ml-10 mb-10 mt-10">Submitted tables:</p>
<clr-tree> <clr-tree>
<clr-tree-node *ngFor="let dataset of submittedDatasets"> <clr-tree-node *ngFor="let dataset of submittedCsvDatasets">
<button <button
(click)="onSubmittedDatasetClick(dataset)" (click)="onSubmittedCsvDatasetClick(dataset)"
class="clr-treenode-link whitespace-nowrap" class="clr-treenode-link whitespace-nowrap"
[class.active]="dataset.active" [class.active]="dataset.active"
> >
@ -90,7 +114,7 @@
</div> --> </div> -->
</app-sidebar> </app-sidebar>
<div class="content-area"> <div #contentArea class="content-area">
<div class="card no-borders h-100 d-flex clr-flex-column"> <div class="card no-borders h-100 d-flex clr-flex-column">
<div <div
class="header-row clr-row justify-content-between clr-justify-content-center w-100 m-0" class="header-row clr-row justify-content-between clr-justify-content-center w-100 m-0"
@ -116,7 +140,7 @@
<ng-container *ngIf="!parsedDatasets.length && selectedFile !== null"> <ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
<div class="d-flex clr-justify-content-center mt-15"> <div class="d-flex clr-justify-content-center mt-15">
<div class="dataset-input-wrapper"> <div class="dataset-input-wrapper">
<p cds-text="secondary regular" class="mb-20"> <p cds-text="secondary regular" class="mb-5">
Selected file: <strong>{{ selectedFile.name }}</strong> Selected file: <strong>{{ selectedFile.name }}</strong>
<clr-tooltip> <clr-tooltip>
<cds-icon <cds-icon
@ -129,6 +153,9 @@
<clr-tooltip-content> Discard the file </clr-tooltip-content> <clr-tooltip-content> Discard the file </clr-tooltip-content>
</clr-tooltip> </clr-tooltip>
</p> </p>
<p cds-text="secondary regular" class="mb-20">
File size: <strong>{{ selectedFile.sizeMB }} MB</strong>
</p>
<p cds-text="secondary regular" class="mb-15"> <p cds-text="secondary regular" class="mb-15">
Paste or type the list of datasets to upload: Paste or type the list of datasets to upload:
</p> </p>
@ -151,7 +178,7 @@
<div class="dataset-selection-actions text-right mt-10"> <div class="dataset-selection-actions text-right mt-10">
<button <button
(click)="onUploadFile()" (click)="onStartParsingFile()"
id="continue-btn" id="continue-btn"
class="btn btn-primary btn-sm" class="btn btn-primary btn-sm"
[disabled]="!matchedDatasets.length" [disabled]="!matchedDatasets.length"
@ -164,7 +191,9 @@
</div> </div>
</ng-container> </ng-container>
<ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length"> <ng-container
*ngIf="parsedDatasets.length && !submittedCsvDatasets.length"
>
<div <div
*ngIf="!activeParsedDataset" *ngIf="!activeParsedDataset"
class="no-table-selected pointer-events-none" class="no-table-selected pointer-events-none"
@ -180,7 +209,68 @@
</div> </div>
<ng-container *ngIf="activeParsedDataset"> <ng-container *ngIf="activeParsedDataset">
<div class="d-flex clr-justify-content-between p-10"> <div
*ngIf="activeParsedDataset.submitResult"
class="d-flex clr-justify-content-between p-10 mt-15 submission-results"
>
<div>
<p cds-text="secondary regular" class="mb-10">
Submit Status:
<span
*ngIf="activeParsedDataset.submitResult?.success"
class="color-green"
><strong>SUCCESS</strong></span
>
<span
*ngIf="activeParsedDataset.submitResult?.error"
class="color-red"
><strong>ERROR</strong></span
>
</p>
<p
*ngIf="activeParsedDataset.submitResult?.error"
cds-text="secondary regular"
>
Error details:
</p>
</div>
<div>
<button
*ngIf="
!submittingCsv && activeParsedDataset.submitResult?.error
"
(click)="reSubmitTable(activeParsedDataset)"
class="btn btn-primary mt-10"
[clrLoading]="submitLoading"
>
Resubmit
</button>
<button
(click)="
downloadFile(
activeParsedDataset.submitResult.log ||
activeParsedDataset.submitResult.success ||
activeParsedDataset.submitResult.error
)
"
class="btn btn-primary-outline mt-10"
>
Download log
</button>
</div>
</div>
<div
*ngIf="activeParsedDataset.submitResult?.error"
class="error-field mt-15"
>
<div class="log-wrapper">
{{ activeParsedDataset.submitResult?.error | json }}
</div>
</div>
<div class="d-flex clr-justify-content-between p-10 mt-15">
<div> <div>
<p cds-text="secondary regular" class="mb-10"> <p cds-text="secondary regular" class="mb-10">
Found in range: Found in range:
@ -197,7 +287,17 @@
</ng-container> </ng-container>
<ng-container *ngIf="!activeParsedDataset.parseResult"> <ng-container *ngIf="!activeParsedDataset.parseResult">
<strong>No data found</strong> <strong *ngIf="!activeParsedDataset.parsingTable"
>No data found</strong
>
<span
*ngIf="activeParsedDataset.parsingTable"
class="d-flex clr-align-items-center"
>
<strong>Searching for the data...</strong>
<clr-spinner class="ml-5" clrSmall></clr-spinner>
</span>
</ng-container> </ng-container>
</p> </p>
<p cds-text="secondary regular"> <p cds-text="secondary regular">
@ -262,9 +362,9 @@
</ng-container> </ng-container>
</ng-container> </ng-container>
<ng-container *ngIf="submittedDatasets.length"> <ng-container *ngIf="submittedCsvDatasets.length">
<div <div
*ngIf="!activeSubmittedDataset" *ngIf="!activeSubmittedCsvDataset"
class="no-table-selected pointer-events-none" class="no-table-selected pointer-events-none"
> >
<clr-icon <clr-icon
@ -278,31 +378,17 @@
</div> </div>
</ng-container> </ng-container>
<ng-container *ngIf="activeSubmittedDataset"> <ng-container *ngIf="activeSubmittedCsvDataset">
<div class="d-flex clr-justify-content-between p-10"> <div class="d-flex clr-justify-content-between p-10">
<div> <div>
<p
*ngIf="activeSubmittedDataset.parseResult"
cds-text="secondary regular"
class="mb-10"
>
Found in range:
<strong
>"{{
activeSubmittedDataset.parseResult.rangeSheetRes?.sheetName
}}"!{{
activeSubmittedDataset.parseResult.rangeSheetRes?.rangeAddress
}}</strong
>
</p>
<p cds-text="secondary regular" class="mb-10"> <p cds-text="secondary regular" class="mb-10">
Matched with dataset: Matched with dataset:
<strong> <strong>
<clr-tooltip> <clr-tooltip>
<a <a
clrTooltipTrigger clrTooltipTrigger
[routerLink]="'/editor/' + activeSubmittedDataset.libds" [routerLink]="'/editor/' + activeSubmittedCsvDataset.libds"
>{{ activeSubmittedDataset.libds }}</a >{{ activeSubmittedCsvDataset.libds }}</a
> >
<clr-tooltip-content <clr-tooltip-content
[clrPosition]="'top-right'" [clrPosition]="'top-right'"
@ -315,15 +401,17 @@
</p> </p>
<p cds-text="secondary regular" class="mb-10"> <p cds-text="secondary regular" class="mb-10">
Status: Status:
<span *ngIf="activeSubmittedDataset.success" class="color-green" <span
*ngIf="activeSubmittedCsvDataset.success"
class="color-green"
><strong>SUCCESS</strong></span ><strong>SUCCESS</strong></span
> >
<span *ngIf="activeSubmittedDataset.error" class="color-red" <span *ngIf="activeSubmittedCsvDataset.error" class="color-red"
><strong>ERROR</strong></span ><strong>ERROR</strong></span
> >
</p> </p>
<p <p
*ngIf="activeSubmittedDataset.error" *ngIf="activeSubmittedCsvDataset.error"
cds-text="secondary regular" cds-text="secondary regular"
> >
Error details: Error details:
@ -331,18 +419,11 @@
</div> </div>
<div> <div>
<button
*ngIf="!submittingCsv && activeSubmittedDataset.error"
(click)="reSubmitTable(activeSubmittedDataset)"
class="btn btn-primary mt-10"
[clrLoading]="submitLoading"
>
Resubmit
</button>
<button <button
(click)=" (click)="
downloadFile( downloadFile(
activeSubmittedDataset.success || activeSubmittedDataset.error activeSubmittedCsvDataset.success ||
activeSubmittedCsvDataset.error
) )
" "
class="btn btn-primary-outline mt-10" class="btn btn-primary-outline mt-10"
@ -352,9 +433,9 @@
</div> </div>
</div> </div>
<div *ngIf="activeSubmittedDataset.error" class="error-field mt-15"> <div *ngIf="activeSubmittedCsvDataset.error" class="error-field mt-15">
<div class="log-wrapper"> <div class="log-wrapper">
{{ activeSubmittedDataset.error | json }} {{ activeSubmittedCsvDataset.error | json }}
</div> </div>
</div> </div>
</ng-container> </ng-container>

4
client/src/app/multi-dataset/multi-dataset.component.scss
View File

@ -47,4 +47,8 @@
.licence-limit-notice { .licence-limit-notice {
color: var(--cds-alias-status-warning-dark); color: var(--cds-alias-status-warning-dark);
}
.submission-results {
border-bottom: 1px solid #d3d3d3;
} }

242
client/src/app/multi-dataset/multi-dataset.component.ts
View File

@ -1,8 +1,10 @@
import { import {
ChangeDetectorRef, ChangeDetectorRef,
Component, Component,
ElementRef,
HostBinding, HostBinding,
OnInit OnInit,
ViewChild
} from '@angular/core' } from '@angular/core'
import { import {
EventService, EventService,
@ -11,17 +13,13 @@ import {
SasService, SasService,
SasStoreService SasStoreService
} from '../services' } from '../services'
import * as XLSX from '@sheet/crypto'
import { globals } from '../_globals' import { globals } from '../_globals'
import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model' import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
import { DcValidator } from '../shared/dc-validator/dc-validator' import { DcValidator } from '../shared/dc-validator/dc-validator'
import { ExcelRule } from '../models/TableData' import { ExcelRule } from '../models/TableData'
import { HotTableInterface } from '../models/HotTable.interface' import { HotTableInterface } from '../models/HotTable.interface'
import { Col } from '../shared/dc-validator/models/col.model' import { Col } from '../shared/dc-validator/models/col.model'
import { import { SpreadsheetService } from '../services/spreadsheet.service'
ParseResult,
SpreadsheetService
} from '../services/spreadsheet.service'
import Handsontable from 'handsontable' import Handsontable from 'handsontable'
import { HotTableRegisterer } from '@handsontable/angular' import { HotTableRegisterer } from '@handsontable/angular'
import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model' import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
@ -29,6 +27,9 @@ import { CellChange, ChangeSource } from 'handsontable/common'
import { baseAfterGetColHeader } from '../shared/utils/hot.utils' import { baseAfterGetColHeader } from '../shared/utils/hot.utils'
import { ColumnSettings } from 'handsontable/settings' import { ColumnSettings } from 'handsontable/settings'
import { UploadFile } from '@sasjs/adapter' import { UploadFile } from '@sasjs/adapter'
import { UploadFileResponse } from '../models/UploadFile'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
import { ParseResult } from '../models/ParseResult.interface'
@Component({ @Component({
selector: 'app-multi-dataset', selector: 'app-multi-dataset',
@ -37,6 +38,7 @@ import { UploadFile } from '@sasjs/adapter'
}) })
export class MultiDatasetComponent implements OnInit { export class MultiDatasetComponent implements OnInit {
@HostBinding('class.content-container') contentContainerClass = true @HostBinding('class.content-container') contentContainerClass = true
@ViewChild('contentArea', { static: true }) contentAreaRef!: ElementRef
public licenceState = this.licenceService.licenceState public licenceState = this.licenceService.licenceState
public Infinity = Infinity public Infinity = Infinity
@ -48,15 +50,19 @@ export class MultiDatasetComponent implements OnInit {
public csvFiles: UploadFile[] = [] public csvFiles: UploadFile[] = []
public csvSubmitting: boolean = false public csvSubmitting: boolean = false
public selectedFile: File | null = null public autoDetectingColumns: boolean = false
public selectedFile: SelectedFile | null = null
public parsedDatasets: ParsedDataset[] = [] public parsedDatasets: ParsedDataset[] = []
public submittedDatasets: SubmittedDatasetResult[] = [] public submittedCsvDatasets: SubmittedCsvDatasetResult[] = []
public datasetsLoading: boolean = false public datasetsLoading: boolean = false
public uploadLoading: boolean = false public uploadLoading: boolean = false
public submitLoading: boolean = false public submitLoading: boolean = false
public matchedDatasets: string[] = [] public matchedDatasets: string[] = []
public sheetNames: string[] = []
public userInputDatasets: string = '' public userInputDatasets: string = ''
public libsAndTables: { public libsAndTables: {
@ -194,6 +200,11 @@ export class MultiDatasetComponent implements OnInit {
// For EXCEL if multiple files, we only take one (the first one) // For EXCEL if multiple files, we only take one (the first one)
this.selectedFile = event.target.files[0] this.selectedFile = event.target.files[0]
if (this.selectedFile)
this.selectedFile.sizeMB = this.spreadsheetService.bytesToMB(
this.selectedFile.size
)
this.initUserInputHot() this.initUserInputHot()
this.onAutoDetectColumns() this.onAutoDetectColumns()
} else if (matchedExtension === 'csv') { } else if (matchedExtension === 'csv') {
@ -246,7 +257,7 @@ export class MultiDatasetComponent implements OnInit {
this.userInputDatasets = '' this.userInputDatasets = ''
} }
async onUploadFile() { async onStartParsingFile() {
this.uploadLoading = true this.uploadLoading = true
const datasetFetchingPromises: Promise< const datasetFetchingPromises: Promise<
@ -271,18 +282,28 @@ export class MultiDatasetComponent implements OnInit {
const datasetObjects = this.buildDatasetsObjects(datasets) const datasetObjects = this.buildDatasetsObjects(datasets)
for (let datasetObject of datasetObjects) { datasetObjects.forEach((datasetObject) => {
this.parsedDatasets.push({
libds: datasetObject.libds,
includeInSubmission: true,
datasetInfo: datasetObject,
parsingTable: true
})
})
for (let parsedDataset of this.parsedDatasets) {
this.spreadsheetService this.spreadsheetService
.parseExcelFile({ .parseExcelFile({
file: this.selectedFile!, file: this.selectedFile!,
dcValidator: datasetObject.dcValidator!, password: this.selectedFile!.password || undefined,
headerPks: datasetObject.headerPks, dcValidator: parsedDataset.datasetInfo.dcValidator!,
headerArray: datasetObject.headerArray, headerPks: parsedDataset.datasetInfo.headerPks,
headerArray: parsedDataset.datasetInfo.headerArray,
headerShow: [], headerShow: [],
timeHeaders: datasetObject.timeHeaders, timeHeaders: parsedDataset.datasetInfo.timeHeaders,
dateHeaders: datasetObject.dateHeaders, dateHeaders: parsedDataset.datasetInfo.dateHeaders,
dateTimeHeaders: datasetObject.dateTimeHeaders, dateTimeHeaders: parsedDataset.datasetInfo.dateTimeHeaders,
xlRules: datasetObject.xlRules xlRules: parsedDataset.datasetInfo.xlRules
}) })
.then((parseResult: ParseResult | undefined) => { .then((parseResult: ParseResult | undefined) => {
console.log('parseResult', parseResult) console.log('parseResult', parseResult)
@ -304,23 +325,17 @@ export class MultiDatasetComponent implements OnInit {
datasource.push(itemObject) datasource.push(itemObject)
}) })
this.parsedDatasets.push({ parsedDataset.datasource = datasource
libds: datasetObject.libds, parsedDataset.parseResult = parseResult
parseResult: parseResult, parsedDataset.parsingTable = false
includeInSubmission: true,
datasetInfo: datasetObject,
datasource: datasource
})
} }
}) })
.catch((error: string) => { .catch((error: string) => {
console.warn('Parsing excel file error.', error) console.warn('Parsing excel file error.', error)
this.parsedDatasets.push({ parsedDataset.datasource = []
libds: datasetObject.libds, parsedDataset.includeInSubmission = false
includeInSubmission: false, parsedDataset.parsingTable = false
datasetInfo: datasetObject
})
}) })
} }
}) })
@ -349,13 +364,17 @@ export class MultiDatasetComponent implements OnInit {
setTimeout(() => { setTimeout(() => {
this.hotInstance = this.hotRegisterer.getInstance('hotInstance') this.hotInstance = this.hotRegisterer.getInstance('hotInstance')
// Set height of parsed data to full height of the page content area
const contentAreaHeight = this.contentAreaRef.nativeElement.clientHeight
const hotHeight = `${contentAreaHeight - 160}px`
if (this.activeParsedDataset) { if (this.activeParsedDataset) {
this.hotInstance.updateSettings({ this.hotInstance.updateSettings({
data: this.activeParsedDataset.datasource || [], data: this.activeParsedDataset.datasource || [],
colHeaders: this.activeParsedDataset.datasetInfo.headerColumns, colHeaders: this.activeParsedDataset.datasetInfo.headerColumns,
columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(), columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(),
readOnly: true, readOnly: true,
height: '300px', height: hotHeight || '300px',
className: 'htDark' className: 'htDark'
}) })
} }
@ -507,13 +526,32 @@ export class MultiDatasetComponent implements OnInit {
* convention. {@link isValidDatasetFormat} * convention. {@link isValidDatasetFormat}
*/ */
async onAutoDetectColumns() { async onAutoDetectColumns() {
const sheetNames = await this.parseExcelSheetNames() let passwordError = false
if (sheetNames) { await this.parseExcelSheetNames()
.then((sheetNames) => {
this.sheetNames = sheetNames
})
.catch((err) => {
if (err.includes('password')) {
passwordError = true
}
})
if (passwordError) {
this.onDiscardFile()
this.eventService.showInfoModal(
'Locked file',
'We failed to unlock the file.'
)
return
}
if (this.sheetNames) {
this.matchedDatasets = [] this.matchedDatasets = []
this.userInputDatasets = '' this.userInputDatasets = ''
sheetNames.forEach((sheetName: string, index: number) => { this.sheetNames.forEach((sheetName: string, index: number) => {
const trimmedSheetname = sheetName.trim() const trimmedSheetname = sheetName.trim()
if ( if (
@ -561,8 +599,8 @@ export class MultiDatasetComponent implements OnInit {
this.initHot() this.initHot()
} }
onSubmittedDatasetClick(submittedDataset: SubmittedDatasetResult) { onSubmittedCsvDatasetClick(submittedDataset: SubmittedCsvDatasetResult) {
this.deselectAllSubmittedDatasets() this.deselectAllSubmittedCsvDatasets()
submittedDataset.active = true submittedDataset.active = true
} }
@ -571,8 +609,10 @@ export class MultiDatasetComponent implements OnInit {
return this.parsedDatasets.find((dataset) => dataset.active) return this.parsedDatasets.find((dataset) => dataset.active)
} }
public get activeSubmittedDataset(): SubmittedDatasetResult | undefined { public get activeSubmittedCsvDataset():
return this.submittedDatasets.find((dataset) => dataset.active) | SubmittedCsvDatasetResult
| undefined {
return this.submittedCsvDatasets.find((dataset) => dataset.active)
} }
public get notFoundDatasets(): string[] { public get notFoundDatasets(): string[] {
@ -606,8 +646,12 @@ export class MultiDatasetComponent implements OnInit {
return this.csvFiles.length > 0 return this.csvFiles.length > 0
} }
public get excelsSubmitted(): boolean {
return !!this.parsedDatasets.filter((ds) => ds.submitResult).length
}
public downloadFile(response: any) { public downloadFile(response: any) {
const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log` const filename = `stagedata-${this.activeSubmittedCsvDataset?.libds}-log`
this.helperService.downloadTextFile(filename, JSON.stringify(response)) this.helperService.downloadTextFile(filename, JSON.stringify(response))
} }
@ -615,7 +659,7 @@ export class MultiDatasetComponent implements OnInit {
* Submits attached CSVs which are matched with existing datasets * Submits attached CSVs which are matched with existing datasets
*/ */
async submitCsvFiles() { async submitCsvFiles() {
let requestsResults: SubmittedDatasetResult[] = [] let requestsResults: SubmittedCsvDatasetResult[] = []
for (let file of this.csvFiles) { for (let file of this.csvFiles) {
const libds = this.parseDatasetFromCsvName(file.fileName) const libds = this.parseDatasetFromCsvName(file.fileName)
@ -626,17 +670,17 @@ export class MultiDatasetComponent implements OnInit {
await this.sasService await this.sasService
.uploadFile('services/editors/loadfile', [file], { table: libds }) .uploadFile('services/editors/loadfile', [file], { table: libds })
.then( .then(
(res: any) => { (res: UploadFileResponse) => {
if (typeof res.sasjsAbort !== 'undefined') { if (typeof res.adapterResponse.sasjsAbort !== 'undefined') {
error = res.sasjsAbort error = res.adapterResponse.sasjsAbort
} else { } else {
success = res success = res.adapterResponse
} }
}, },
(err: any) => { (err: any) => {
console.error('err', err) console.error('err', err)
error = err error = err.adapterResponse
} }
) )
@ -647,7 +691,7 @@ export class MultiDatasetComponent implements OnInit {
}) })
} }
this.submittedDatasets = requestsResults this.submittedCsvDatasets = requestsResults
} }
/** /**
@ -678,7 +722,7 @@ export class MultiDatasetComponent implements OnInit {
} }
/** /**
* Sends tables to the SAS sequentially * Sends tables found in an excel to the SAS sequentially
* *
* @param explicitDatasets if empty all datasets will be sent, otherwise only datasets * @param explicitDatasets if empty all datasets will be sent, otherwise only datasets
* in the array will be sent. eg. ['lib1.table_1', 'lib1.table_2'] * in the array will be sent. eg. ['lib1.table_1', 'lib1.table_2']
@ -688,10 +732,6 @@ export class MultiDatasetComponent implements OnInit {
this.submitLoading = true this.submitLoading = true
let requestsResults: SubmittedDatasetResult[] = explicitDatasets
? this.submittedDatasets
: []
for (let table of this.parsedDatasets) { for (let table of this.parsedDatasets) {
// Skip the table if no data inside // Skip the table if no data inside
if (!table.parseResult || !table.datasource) continue if (!table.parseResult || !table.datasource) continue
@ -733,6 +773,7 @@ export class MultiDatasetComponent implements OnInit {
let error let error
let success let success
let log
await this.sasStoreService await this.sasStoreService
.updateTable( .updateTable(
@ -741,56 +782,47 @@ export class MultiDatasetComponent implements OnInit {
'SASControlTable', 'SASControlTable',
'editors/stagedata', 'editors/stagedata',
table.datasetInfo.data.$sasdata, table.datasetInfo.data.$sasdata,
true true,
{
debug: true
}
) )
.then((res: EditorsStageDataSASResponse) => { .then((res: RequestWrapperResponse<EditorsStageDataSASResponse>) => {
success = res success = res.adapterResponse
log = res.log
}) })
.catch((err: any) => { .catch((err: any) => {
console.error('err', err) console.error('err', err)
error = err error = err.adapterResponse
log = err.log
}) })
const requestResult = { const requestResult = {
success, success,
error, error,
log,
parseResult: table.parseResult, parseResult: table.parseResult,
libds: table.libds libds: table.libds
} }
// If explicit datasets are set don't just push to th array table.submitResult = requestResult
// instead replace if result already exist from before (this might be re-submit)
if (explicitDatasets) {
const existingResultIndex = requestsResults.findIndex(
(result) => result.libds === table.libds
)
if (existingResultIndex > -1) {
requestsResults[existingResultIndex] = requestResult
} else {
requestsResults.push(requestResult)
}
} else {
requestsResults.push(requestResult)
}
} }
} }
this.submittedDatasets = requestsResults
this.showSubmitReasonModal = false this.showSubmitReasonModal = false
this.submitLoading = false this.submitLoading = false
this.deselectAllParsedDatasets() this.deselectAllParsedDatasets()
} }
async reSubmitTable(activeSubmittedDataset: SubmittedDatasetResult) { async reSubmitTable(activeSubmittedDataset: ParsedDataset) {
// Submit only particular table // Submit only particular table
await this.submitTables([activeSubmittedDataset.libds]) await this.submitTables([activeSubmittedDataset.libds])
// Activate new resubmitted table // Activate new resubmitted table
const newSubmittedDataset = this.submittedDatasets.find( const newSubmittedDataset = this.parsedDatasets.find(
(sd) => sd.libds === activeSubmittedDataset.libds (sd) => sd.libds === activeSubmittedDataset.libds
) )
if (newSubmittedDataset) newSubmittedDataset.active = true if (newSubmittedDataset) newSubmittedDataset.active = true
} }
@ -815,40 +847,26 @@ export class MultiDatasetComponent implements OnInit {
.map((row) => (row ? `${row[0]}.${row[1]}` : '')) .map((row) => (row ? `${row[0]}.${row[1]}` : ''))
} }
private parseExcelSheetNames(): Promise<string[]> { /**
* Read the file minimally just to get the sheet names, not reading full file
* to help boost the performance
*
* @returns sheet names in string array
*/
private async parseExcelSheetNames(): Promise<string[]> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const reader = new FileReader() if (!this.selectedFile) return resolve([])
if (!this.selectedFile) { this.spreadsheetService
console.warn('selectedFile is missing') .parseExcelSheetNames(this.selectedFile)
return resolve([]) .then((parsed) => {
} if (parsed.password) this.selectedFile!.password = parsed.password
reader.onload = (event: ProgressEvent<FileReader>) => { return resolve(parsed.sheetNames)
if (!event?.target) { })
console.warn('File reader event.target is missing') .catch((err) => {
return return reject(err)
}
const data = event.target.result
const workbook = XLSX.read(data, {
type: 'binary'
}) })
try {
const sheet_names_list = workbook.SheetNames
return resolve(sheet_names_list)
} catch (e) {
console.error(e)
}
}
reader.onerror = function (ex) {
console.log(ex)
}
reader.readAsBinaryString(this.selectedFile)
}) })
} }
@ -977,8 +995,8 @@ export class MultiDatasetComponent implements OnInit {
} }
} }
private deselectAllSubmittedDatasets() { private deselectAllSubmittedCsvDatasets() {
for (let submittedDataset of this.submittedDatasets) { for (let submittedDataset of this.submittedCsvDatasets) {
submittedDataset.active = false submittedDataset.active = false
} }
} }
@ -1002,16 +1020,28 @@ export interface ParsedDataset {
libds: string libds: string
parseResult?: ParseResult parseResult?: ParseResult
datasetInfo: DatasetsObject datasetInfo: DatasetsObject
submitResult?: SubmittedDatasetResult
datasource?: any[] datasource?: any[]
includeInSubmission: boolean includeInSubmission: boolean
status?: 'success' | 'error' status?: 'success' | 'error'
active?: boolean active?: boolean
parsingTable?: boolean
} }
export interface SubmittedDatasetResult { export interface SubmittedDatasetResult {
success: EditorsStageDataSASResponse | undefined
error: any
log?: string
}
export interface SubmittedCsvDatasetResult {
libds: string libds: string
success: EditorsStageDataSASResponse | undefined success: EditorsStageDataSASResponse | undefined
error: any error: any
parseResult?: ParseResult
active?: boolean active?: boolean
} }
export interface SelectedFile extends File {
sizeMB?: number
password?: string
}

113
client/src/app/role/role.component.ts
View File

@ -4,6 +4,7 @@ import { HelperService } from '../services/helper.service'
import { Location } from '@angular/common' import { Location } from '@angular/common'
import { Router, ActivatedRoute } from '@angular/router' import { Router, ActivatedRoute } from '@angular/router'
import { SasService } from '../services/sas.service' import { SasService } from '../services/sas.service'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-role', selector: 'app-role',
@ -47,61 +48,69 @@ export class RoleComponent implements OnInit {
} else { } else {
if (globals.usernav.roleList === undefined) { if (globals.usernav.roleList === undefined) {
this.loading = true this.loading = true
this.sasService.request('usernav/userroles', null).then((res: any) => { this.sasService
this.loading = false .request('usernav/userroles', null)
this.roles = res.roles .then((res: RequestWrapperResponse) => {
globals.usernav.roleList = res.roles this.loading = false
if (this.paramPresent) { this.roles = res.adapterResponse.roles
if (this.roles !== undefined) { globals.usernav.roleList = res.adapterResponse.roles
let validRole = this.findRole(this.roles, this.paramURI) if (this.paramPresent) {
if (validRole !== false) { if (this.roles !== undefined) {
this.loading = true let validRole = this.findRole(this.roles, this.paramURI)
let data = { iwant: [{ roleid: this.paramURI }] } if (validRole !== false) {
this.sasService this.loading = true
.request('usernav/usermembersbyrole', data) let data = { iwant: [{ roleid: this.paramURI }] }
.then((res: any) => { this.sasService
this.loading = false .request('usernav/usermembersbyrole', data)
this.roleMembers = res.sasmembers .then((res: RequestWrapperResponse) => {
this.roleMembersCount = res.sasmembers.length this.loading = false
this.roleGroups = res.sasgroups this.roleMembers = res.adapterResponse.sasmembers
this.roleGroupsCount = res.sasgroups.length this.roleMembersCount =
this.roleUri = validRole.ROLEURI res.adapterResponse.sasmembers.length
this.roleName = validRole.ROLENAME this.roleGroups = res.adapterResponse.sasgroups
this.roleDesc = validRole.ROLEDESC this.roleGroupsCount =
}) res.adapterResponse.sasgroups.length
this.roleUri = validRole.ROLEURI
this.roleName = validRole.ROLENAME
this.roleDesc = validRole.ROLEDESC
})
}
} }
} }
} })
})
} else { } else {
this.roles = globals.usernav.roleList this.roles = globals.usernav.roleList
this.roleSearch = globals.usernav.roleSearch this.roleSearch = globals.usernav.roleSearch
this.sasService.request('usernav/userroles', null).then((res: any) => { this.sasService
this.roles = res.roles .request('usernav/userroles', null)
globals.usernav.roleList = res.roles .then((res: RequestWrapperResponse) => {
this.roles = res.adapterResponse.roles
globals.usernav.roleList = res.adapterResponse.roles
if (this.paramPresent) { if (this.paramPresent) {
if (this.roles !== undefined) { if (this.roles !== undefined) {
let validRole = this.findRole(this.roles, this.paramURI) let validRole = this.findRole(this.roles, this.paramURI)
if (validRole !== false) { if (validRole !== false) {
this.loading = true this.loading = true
let data = { iwant: [{ roleid: this.paramURI }] } let data = { iwant: [{ roleid: this.paramURI }] }
this.sasService this.sasService
.request('usernav/usermembersbyrole', data) .request('usernav/usermembersbyrole', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loading = false this.loading = false
this.roleMembers = res.sasmembers this.roleMembers = res.adapterResponse.sasmembers
this.roleMembersCount = res.sasmembers.length this.roleMembersCount =
this.roleGroups = res.sasgroups res.adapterResponse.sasmembers.length
this.roleGroupsCount = res.sasgroups.length this.roleGroups = res.adapterResponse.sasgroups
this.roleUri = validRole.ROLEURI this.roleGroupsCount =
this.roleName = validRole.ROLENAME res.adapterResponse.sasgroups.length
this.roleDesc = validRole.ROLEDESC this.roleUri = validRole.ROLEURI
}) this.roleName = validRole.ROLENAME
this.roleDesc = validRole.ROLEDESC
})
}
} }
} }
} })
})
} }
} }
} }
@ -125,12 +134,12 @@ export class RoleComponent implements OnInit {
let data = { iwant: [{ roleid: role.ROLEURI }] } let data = { iwant: [{ roleid: role.ROLEURI }] }
this.sasService this.sasService
.request('usernav/usermembersbyrole', data) .request('usernav/usermembersbyrole', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loading = false this.loading = false
this.roleMembers = res.sasmembers this.roleMembers = res.adapterResponse.sasmembers
this.roleMembersCount = res.sasmembers.length this.roleMembersCount = res.adapterResponse.sasmembers.length
this.roleGroups = res.sasgroups this.roleGroups = res.adapterResponse.sasgroups
this.roleGroupsCount = res.sasgroups.length this.roleGroupsCount = res.adapterResponse.sasgroups.length
this.roleUri = role.ROLEURI this.roleUri = role.ROLEURI
this.roleName = role.ROLENAME this.roleName = role.ROLENAME
this.roleDesc = role.ROLEDESC this.roleDesc = role.ROLEDESC

59
client/src/app/services/app.service.ts
View File

@ -9,11 +9,13 @@ import { EnvironmentInfo } from '../system/models/environment-info.model'
import { LicenceService } from './licence.service' import { LicenceService } from './licence.service'
import { AppSettingsService } from './app-settings.service' import { AppSettingsService } from './app-settings.service'
import { AppThemes } from '../models/AppSettings' import { AppThemes } from '../models/AppSettings'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
import { AppStoreService } from './app-store.service'
@Injectable() @Injectable()
export class AppService { export class AppService {
public syssite = new BehaviorSubject<string[] | null>(null) public syssite = new BehaviorSubject<string[] | null>(null)
private environmentInfo: EnvironmentInfo | null = null private environmentInfo: EnvironmentInfo = {}
constructor( constructor(
private licenceService: LicenceService, private licenceService: LicenceService,
@ -21,7 +23,8 @@ export class AppService {
private sasService: SasService, private sasService: SasService,
private loggerService: LoggerService, private loggerService: LoggerService,
private appSettingsService: AppSettingsService, private appSettingsService: AppSettingsService,
private router: Router private router: Router,
private appStoreService: AppStoreService
) { ) {
this.subscribe() this.subscribe()
@ -81,16 +84,19 @@ export class AppService {
await this.sasService await this.sasService
.request('public/startupservice', null) .request('public/startupservice', null)
.then(async (res: any) => { .then(async (res: RequestWrapperResponse) => {
this.syssite.next([res.SYSSITE]) this.syssite.next([res.adapterResponse.SYSSITE])
let missingProps: string[] = [] let missingProps: string[] = []
if (!res.globvars || (res.globvars && !res.globvars[0])) if (
!res.adapterResponse.globvars ||
(res.adapterResponse.globvars && !res.adapterResponse.globvars[0])
)
missingProps.push('Globvars') missingProps.push('Globvars')
if (!res.sasdatasets) missingProps.push('Sasdatasets') if (!res.adapterResponse.sasdatasets) missingProps.push('Sasdatasets')
if (!res.saslibs) missingProps.push('Saslibs') if (!res.adapterResponse.saslibs) missingProps.push('Saslibs')
if (!res.xlmaps) missingProps.push('XLMaps') if (!res.adapterResponse.xlmaps) missingProps.push('XLMaps')
if (missingProps.length > 0) { if (missingProps.length > 0) {
startupServiceError = true startupServiceError = true
@ -103,23 +109,26 @@ export class AppService {
return return
} }
const dcAdapterSettings = this.appStoreService.getDcAdapterSettings()
this.environmentInfo = { this.environmentInfo = {
SYSSITE: res.SYSSITE, SYSSITE: res.adapterResponse.SYSSITE,
SYSSCPL: res.SYSSCPL, SYSSCPL: res.adapterResponse.SYSSCPL,
SYSTCPIPHOSTNAME: res.SYSTCPIPHOSTNAME, SYSTCPIPHOSTNAME: res.adapterResponse.SYSTCPIPHOSTNAME,
SYSVLONG: res.SYSVLONG, SYSVLONG: res.adapterResponse.SYSVLONG,
MEMSIZE: res.MEMSIZE, MEMSIZE: res.adapterResponse.MEMSIZE,
SYSPROCESSMODE: res.SYSPROCESSMODE, SYSPROCESSMODE: res.adapterResponse.SYSPROCESSMODE,
SYSHOSTNAME: res.SYSHOSTNAME, SYSHOSTNAME: res.adapterResponse.SYSHOSTNAME,
SYSUSERID: res.SYSUSERID, SYSUSERID: res.adapterResponse.SYSUSERID,
SYSHOSTINFOLONG: res.SYSHOSTINFOLONG, SYSHOSTINFOLONG: res.adapterResponse.SYSHOSTINFOLONG,
SYSENCODING: res.SYSENCODING, SYSENCODING: res.adapterResponse.SYSENCODING,
AUTOEXEC: res.AUTOEXEC, AUTOEXEC: res.adapterResponse.AUTOEXEC,
ISADMIN: res.globvars[0].ISADMIN, ISADMIN: res.adapterResponse.globvars[0].ISADMIN,
DC_ADMIN_GROUP: res.globvars[0].DC_ADMIN_GROUP DC_ADMIN_GROUP: res.adapterResponse.globvars[0].DC_ADMIN_GROUP,
APP_LOC: dcAdapterSettings?.appLoc
} }
let libs = res.sasdatasets let libs = res.adapterResponse.sasdatasets
let libGroup: any = {} let libGroup: any = {}
let libsAndTables let libsAndTables
let libraries let libraries
@ -153,7 +162,7 @@ export class AppService {
globals.editor.libsAndTables = libsAndTables globals.editor.libsAndTables = libsAndTables
} }
globals.xlmaps = res.xlmaps.map((xlmap: any) => ({ globals.xlmaps = res.adapterResponse.xlmaps.map((xlmap: any) => ({
id: xlmap[0], id: xlmap[0],
description: xlmap[1], description: xlmap[1],
targetDS: xlmap[2] targetDS: xlmap[2]
@ -162,9 +171,9 @@ export class AppService {
globals.editor.libraries = libraries globals.editor.libraries = libraries
globals.editor.startupSet = true globals.editor.startupSet = true
globals.dcLib = res.globvars[0].DCLIB globals.dcLib = res.adapterResponse.globvars[0].DCLIB
await this.licenceService.activation(res) await this.licenceService.activation(res.adapterResponse)
}) })
.catch((err: any) => { .catch((err: any) => {
startupServiceError = true startupServiceError = true

121
client/src/app/services/sas-store.service.ts
View File

@ -18,6 +18,7 @@ import { isSpecialMissing } from '@sasjs/utils/input/validators'
import { Col } from '../shared/dc-validator/models/col.model' import { Col } from '../shared/dc-validator/models/col.model'
import { get } from 'lodash-es' import { get } from 'lodash-es'
import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model' import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Injectable() @Injectable()
export class SasStoreService { export class SasStoreService {
@ -58,12 +59,10 @@ export class SasStoreService {
) { ) {
const tables: any = {} const tables: any = {}
tables[tableName] = [tableData] tables[tableName] = [tableData]
const res: EditorsGetDataSASResponse = await this.sasService.request( const res: RequestWrapperResponse<EditorsGetDataSASResponse> =
program, await this.sasService.request(program, tables)
tables
)
const response: EditorsGetDataServiceResponse = { const response: EditorsGetDataServiceResponse = {
data: res, data: res.adapterResponse,
libds: libds libds: libds
} }
return response return response
@ -84,8 +83,9 @@ export class SasStoreService {
tableName: string, tableName: string,
program: string, program: string,
$dataFormats: $DataFormats | null, $dataFormats: $DataFormats | null,
suppressErrorSuccessMessages?: boolean suppressErrorSuccessMessages?: boolean,
): Promise<EditorsStageDataSASResponse> { adapterConfig?: any
): Promise<RequestWrapperResponse<EditorsStageDataSASResponse>> {
// add sp as third argument of createData call // add sp as third argument of createData call
let tables: any = { let tables: any = {
@ -103,7 +103,7 @@ export class SasStoreService {
let res = await this.sasService.request<EditorsStageDataSASResponse>( let res = await this.sasService.request<EditorsStageDataSASResponse>(
program, program,
tables, tables,
null, adapterConfig,
{ {
suppressErrorAbortModal: suppressErrorSuccessMessages, suppressErrorAbortModal: suppressErrorSuccessMessages,
suppressSuccessAbortModal: suppressErrorSuccessMessages suppressSuccessAbortModal: suppressErrorSuccessMessages
@ -127,8 +127,7 @@ export class SasStoreService {
) { ) {
let tables: any = {} let tables: any = {}
tables[tableName] = [tableData] tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables) return (await this.sasService.request(program, tables)).adapterResponse
return res
} }
/** /**
@ -152,8 +151,8 @@ export class SasStoreService {
* @returns All submits * @returns All submits
*/ */
public async getSubmitts() { public async getSubmitts() {
let res: any = await this.sasService.request('editors/getsubmits', null) return (await this.sasService.request('editors/getsubmits', null))
return res .adapterResponse
} }
/** /**
@ -161,7 +160,8 @@ export class SasStoreService {
* @returns All libraries * @returns All libraries
*/ */
public async viewLibs() { public async viewLibs() {
return this.sasService.request('public/viewlibs', null) return (await this.sasService.request('public/viewlibs', null))
.adapterResponse
} }
public async refreshLibInfo(libref: string) { public async refreshLibInfo(libref: string) {
@ -169,30 +169,22 @@ export class SasStoreService {
lib2refresh: [{ libref }] lib2refresh: [{ libref }]
} }
return this.sasService.request('public/refreshlibinfo', data) return (await this.sasService.request('public/refreshlibinfo', data))
} .adapterResponse
public async versionHistory(libDataset: any) {
const data = { iwant: [{ LIBDS: libDataset }] }
let res: any = await this.sasService.request(
'public/getversionhistory',
data
)
return res
} }
public async viewTables(lib: any) { public async viewTables(lib: any) {
let tables = { SASControlTable: [{ MPLIB: lib }] } let tables = { SASControlTable: [{ MPLIB: lib }] }
let res: any = await this.sasService.request('public/viewtables', tables) return (await this.sasService.request('public/viewtables', tables))
return res .adapterResponse
} }
public async viewData(libDataset: any, filter_pk: any) { public async viewData(libDataset: any, filter_pk: any) {
let tables = { let tables = {
SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }] SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }]
} }
let res: any = await this.sasService.request('public/viewdata', tables) return (await this.sasService.request('public/viewdata', tables))
return res .adapterResponse
} }
public async viewDataSearch( public async viewDataSearch(
@ -213,41 +205,36 @@ export class SasStoreService {
} }
] ]
} }
let res: any = await this.sasService.request('public/viewdata', tables)
return res return (await this.sasService.request('public/viewdata', tables))
.adapterResponse
} }
public async getXLMapRules(id: string) { public async getXLMapRules(id: string) {
const tables = { const tables = {
getxlmaps_in: [{ XLMAP_ID: id }] getxlmaps_in: [{ XLMAP_ID: id }]
} }
const res: any = await this.sasService.request('editors/getxlmaps', tables) return (await this.sasService.request('editors/getxlmaps', tables))
return res .adapterResponse
}
public async getDetails(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
} }
public async showDiffs(tableData: any, tableName: string, program: string) { public async showDiffs(tableData: any, tableName: string, program: string) {
let tables: any = {} let tables: any = {}
tables[tableName] = [tableData] tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables, { return (
useComputeApi: false await this.sasService.request(program, tables, {
}) useComputeApi: false
return res })
).adapterResponse
} }
public async rejecting(tableData: any, tableName: string, program: string) { public async rejecting(tableData: any, tableName: string, program: string) {
let tables: any = {} let tables: any = {}
tables[tableName] = [tableData] tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables, { return (
useComputeApi: false await this.sasService.request(program, tables, {
}) useComputeApi: false
return res })
).adapterResponse
} }
public async approveTable( public async approveTable(
@ -258,15 +245,13 @@ export class SasStoreService {
let tables: any = {} let tables: any = {}
tables[tableName] = [tableData] tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables) return (await this.sasService.request(program, tables)).adapterResponse
return res
} }
public async getHistory(tableData: any, tableName: string, program: string) { public async getHistory(tableData: any, tableName: string, program: string) {
let tables: any = {} let tables: any = {}
tables[tableName] = [tableData] tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables) return (await this.sasService.request(program, tables)).adapterResponse
return res
} }
setQueryVariables(dataset: string, cols: any) { setQueryVariables(dataset: string, cols: any) {
@ -278,8 +263,8 @@ export class SasStoreService {
public async getChangeInfo(tableId: any) { public async getChangeInfo(tableId: any) {
let obj = { TABLE: tableId } let obj = { TABLE: tableId }
let table = { SASControlTable: [obj] } let table = { SASControlTable: [obj] }
let res: any = await this.sasService.request('public/getchangeinfo', table) return (await this.sasService.request('public/getchangeinfo', table))
return res .adapterResponse
} }
public async getQueryValues( public async getQueryValues(
@ -304,13 +289,13 @@ export class SasStoreService {
tables.FILTERQUERY = filterQuery tables.FILTERQUERY = filterQuery
} }
let res: any = await this.sasService return (
.request('public/getcolvals', tables) await this.sasService
.catch((er: any) => { .request('public/getcolvals', tables)
throw er .catch((er: any) => {
}) throw er
})
return res ).adapterResponse
} }
public async saveQuery(libds: string, filterQuery: QueryClause[]) { public async saveQuery(libds: string, filterQuery: QueryClause[]) {
@ -319,21 +304,17 @@ export class SasStoreService {
filterquery: filterQuery filterquery: filterQuery
} }
let res: any = await this.sasService.request( const res = await this.sasService.request('public/validatefilter', tables)
'public/validatefilter',
tables
)
this.filter.next(res) this.filter.next(res)
return res
return res.adapterResponse
} }
public async openTable(tableId: string) { public async openTable(tableId: string) {
let tables = { iwant: [{ table_id: tableId }] } let tables = { iwant: [{ table_id: tableId }] }
let res: any = await this.sasService.request( return (await this.sasService.request('auditors/getstagetable', tables))
'auditors/getstagetable', .adapterResponse
tables
)
return res
} }
public checkOperator(operator: any, value: any, type: string) { public checkOperator(operator: any, value: any, type: string) {

92
client/src/app/services/sas.service.ts
View File

@ -12,8 +12,10 @@ import { ServerType } from '@sasjs/utils/types/serverType'
import { DcAdapterSettings } from '../models/DcAdapterSettings' import { DcAdapterSettings } from '../models/DcAdapterSettings'
import { AppStoreService } from './app-store.service' import { AppStoreService } from './app-store.service'
import { LoggerService } from './logger.service' import { LoggerService } from './logger.service'
import { RequestWrapperOptions } from '../models/RequestWrapperOptions' import { RequestWrapperOptions } from '../models/request-wrapper/RequestWrapperOptions'
import { ErrorBody } from '../models/ErrorBody' import { ErrorBody } from '../models/ErrorBody'
import { UploadFileResponse } from '../models/UploadFile'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Injectable({ @Injectable({
providedIn: 'root' providedIn: 'root'
@ -95,14 +97,16 @@ export class SasService {
* @param config additional parameters to force eg. { debug: false } * @param config additional parameters to force eg. { debug: false }
* @param wrapperOptions used to provide options to the request wrapper function * @param wrapperOptions used to provide options to the request wrapper function
* for example to suppress error or success abort modals after request is finished * for example to suppress error or success abort modals after request is finished
* @returns * @returns adapter response or an error. It will return the `log` as well.
* The log could be potentially be wrong if multiple requests happen because the log this
* function return is the last request in the Adapter Array for the given URL.
*/ */
public request<responseType = any>( public request<responseType = any>(
url: string, url: string,
data: any, data: any,
config?: any, config?: any,
wrapperOptions?: RequestWrapperOptions wrapperOptions?: RequestWrapperOptions
): Promise<responseType> { ): Promise<RequestWrapperResponse<responseType>> {
url = 'services/' + url url = 'services/' + url
if (!wrapperOptions) wrapperOptions = {} if (!wrapperOptions) wrapperOptions = {}
@ -117,9 +121,16 @@ export class SasService {
}) })
.then( .then(
(res: any) => { (res: any) => {
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === url)
if (res.login === false) { if (res.login === false) {
this.shouldLogin.next(true) this.shouldLogin.next(true)
reject(false) reject({
adapterResponse: false,
log: sasRequest?.logFile
})
} }
if (!this.userService.user && res.MF_GETUSER) { if (!this.userService.user && res.MF_GETUSER) {
@ -141,7 +152,12 @@ export class SasService {
} }
if (res.status === 404) { if (res.status === 404) {
reject({ MESSAGE: res.body || 'SAS Responded with error' }) reject({
adapterResponse: {
MESSAGE: res.body || 'SAS Responded with error'
},
log: sasRequest?.logFile
})
} }
if (typeof res.sasjsAbort !== 'undefined') { if (typeof res.sasjsAbort !== 'undefined') {
@ -157,7 +173,12 @@ export class SasService {
this.eventService.startupDataLoaded() this.eventService.startupDataLoaded()
this.router.navigateByUrl('/deploy') this.router.navigateByUrl('/deploy')
reject({ error: abortMsg }) reject({
adapterResponse: {
error: abortMsg
},
log: sasRequest?.logFile
})
return return
} }
@ -174,14 +195,26 @@ export class SasService {
) )
} }
reject({ error: abortMsg }) reject({
adapterResponse: {
error: abortMsg
},
log: sasRequest?.logFile
})
} }
resolve(res) resolve({
adapterResponse: res,
log: sasRequest?.logFile
})
}, },
(err: { error: ErrorBody | undefined }) => { (err: { error: ErrorBody | undefined }) => {
console.error(err) console.error(err)
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === url)
if (err.error) { if (err.error) {
let errorMessage: string | undefined = err.error.message let errorMessage: string | undefined = err.error.message
let log: string | undefined let log: string | undefined
@ -205,10 +238,18 @@ export class SasService {
'Request error' 'Request error'
) )
} }
reject({ error: errorMessage }) reject({
adapterResponse: {
error: errorMessage
},
log: sasRequest?.logFile
})
} }
reject(err) reject({
adapterResponse: err,
log: sasRequest?.logFile
})
} }
) )
}) })
@ -222,8 +263,35 @@ export class SasService {
* @param params Aditional parameters eg. { debug: false } * @param params Aditional parameters eg. { debug: false }
* @returns HTTP Response * @returns HTTP Response
*/ */
public uploadFile(sasService: string, files: UploadFile[], params: any) { public uploadFile(
return this.sasjsAdapter.uploadFile(sasService, files, params) sasService: string,
files: UploadFile[],
params: any
): Promise<UploadFileResponse> {
return new Promise((resolve, reject) => {
this.sasjsAdapter.uploadFile(sasService, files, params).then(
(res) => {
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === 'services/editors/loadfile')
resolve({
adapterResponse: res,
log: sasRequest?.logFile
})
},
(err) => {
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === 'services/editors/loadfile')
reject({
response: err,
log: sasRequest?.logFile
})
}
)
})
} }
public async login(username: string, password: string) { public async login(username: string, password: string) {

947
client/src/app/services/spreadsheet.service.ts
View File

@ -1,70 +1,13 @@
import { Injectable } from '@angular/core' import { Injectable } from '@angular/core'
import * as XLSX from '@sheet/crypto' import { ExcelPasswordModalService } from '../shared/excel-password-modal/excel-password-modal.service'
import {
ExcelPasswordModalService,
Result
} from '../shared/excel-password-modal/excel-password-modal.service'
import { EventService } from './event.service' import { EventService } from './event.service'
import { isSpecialMissing } from '@sasjs/utils/input/validators'
import {
dateFormat,
dateToUtcTime,
dateToTime
} from '../editor/utils/date.utils'
import {
excelDateToJSDate,
getMissingHeaders
} from '../editor/utils/grid.utils'
import { isStringNumber, isStringDecimal } from '../editor/utils/types.utils'
import SheetInfo from '../models/SheetInfo'
import { blobToFile } from '../xlmap/utils/file.utils'
import { ExcelRule } from '../models/TableData'
import { DcValidator } from '../shared/dc-validator/dc-validator'
import { LicenceService } from './licence.service' import { LicenceService } from './licence.service'
import { FileUploadEncoding } from '../models/FileUploadEncoding' import { SpreadsheetUtil } from '../shared/spreadsheet-util/spreadsheet-util'
import { FileUploader } from '../models/FileUploader.class' import { ParseParams } from '../models/ParseParams.interface'
import { ParseResult } from '../models/ParseResult.interface'
/** import { OpenOptions } from '../shared/excel-password-modal/models/options.interface'
* Used in combination with buffer import { Result } from '../shared/excel-password-modal/models/result.interface'
*/ import * as XLSX from '@sheet/crypto'
const iconv = require('iconv-lite')
/**
* In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
* Which will be send to backend
*/
const Buffer = require('buffer/').Buffer
type AOA = any[][]
export interface ParseParams {
file: File
dcValidator: DcValidator
/**
* Parse function will manipulate and return the uploader array which can be provided with files already in the queue
* Otherwise new empty instance will be created.
*/
uploader?: FileUploader
headerPks: string[]
headerArray: string[]
headerShow: string[]
timeHeaders: string[]
dateHeaders: string[]
dateTimeHeaders: string[]
xlRules: ExcelRule[]
encoding?: FileUploadEncoding
}
export interface ParseResult {
/**
* In case of CSV file, won't be returned
*/
data?: any[]
/**
* In case of CSV file, won't be returned
*/
headerShow?: string[]
rangeSheetRes?: SheetInfo
uploader: FileUploader
}
@Injectable({ @Injectable({
providedIn: 'root' providedIn: 'root'
@ -78,829 +21,139 @@ export class SpreadsheetService {
private licenceService: LicenceService private licenceService: LicenceService
) {} ) {}
/**
* Parses attached file and searches fo the matching data
*
* @param parseParams params required for parsing the file
* @param onParseStateChange callback used to inform about parsing state
* so the user of the function can update the UI with latest info
* @param onTableFoundEvent callback fired when table range is found in the file
*
* @returns parsed list of files to upload and JSON data ready for HOT usage
*/
public parseExcelFile( public parseExcelFile(
parseParams: ParseParams, parseParams: ParseParams,
onParseStateChange?: (uploadState: string) => void, onParseStateChange?: (uploadState: string) => void,
onTableFoundEvent?: (info: string) => void onTableFoundEvent?: (info: string) => void
): Promise<ParseResult | undefined> { ): Promise<ParseResult | undefined> {
return new Promise((resolve, reject) => { const spreadSheetUtil = new SpreadsheetUtil({
let data: any[] = [] licenceState: this.licenceState
const uploader: FileUploader = parseParams.uploader || new FileUploader()
const file: File = parseParams.file
const filename = file.name
if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
if (onParseStateChange)
onParseStateChange(`Loading ${filename} into the browser`)
let foundData = {
sheet: ''
}
let fileType = filename.slice(
filename.lastIndexOf('.') + 1,
filename.lastIndexOf('.') + 4
)
if (fileType.toLowerCase() === 'xls') {
let reader: FileReader = new FileReader()
const self = this
reader.onload = async (theFile: any) => {
/* read workbook */
const bstr = this.toBstr(theFile.target.result)
let wb: XLSX.WorkBook | undefined = undefined
let fileUnlocking: boolean = false
const xlsxOptions: XLSX.ParsingOptions = {
type: 'binary',
cellDates: false,
cellFormula: true,
cellStyles: true,
cellNF: false,
cellText: false
}
try {
wb = XLSX.read(bstr, {
...xlsxOptions
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
fileUnlocking = true
while (fileUnlocking) {
const password = await this.promptExcelPassword()
if (password) {
try {
wb = XLSX.read(bstr, {
...xlsxOptions,
password: password
})
fileUnlocking = false
} catch (err: any) {
this.excelPasswordModalService.open({
error: true
})
if (!err.message.toLowerCase().includes('password')) {
fileUnlocking = false
}
}
} else {
fileUnlocking = false
}
}
} else {
return reject('Error reading the file')
}
}
if (!wb) {
return reject('No workbook found.')
}
/* save data */
let isComplete: boolean = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
let csvArrayData: any[] = []
const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
wb,
parseParams
)
missingHeaders = rangeSheetRes.missingHeaders
if (rangeSheetRes.foundData) {
isComplete = true
csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
if (onParseStateChange)
onParseStateChange(
`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
)
let startAddress = ''
let endAddress = ''
for (
let row = rangeSheetRes.startRow;
row < rangeSheetRes.endRow;
++row
) {
const arr: any[] = []
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
const addr = XLSX.utils.encode_cell({
r: rangeSheetRes.rangeStartRow + row,
c: rangeSheetRes.rangeStartCol + col
})
let cell
if (!ws[addr]) {
cell = { v: '' }
} else {
cell = ws[addr]
}
if (startAddress === '' && ws[addr]) startAddress = addr
endAddress = addr
arr.push(cell)
})
// If we found at least one non empty value it means it is not empty row
// othervise, it is empty row
let arrNonEmptyValue = arr.find((x) => x.v !== '')
if (arrNonEmptyValue) csvArrayData.push(arr)
}
rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
if (onTableFoundEvent)
onTableFoundEvent(
`Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
)
} else {
missingHeaders = rangeSheetRes.missingHeaders
}
if (missingHeaders.length > 0) {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
let abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
uploader.queue.pop()
return reject(abortMsg)
}
// If first row is empty, that means no data has been found
if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
let abortMsg = 'No relevant data found in File !'
uploader.queue.pop()
return reject(abortMsg)
}
if (
parseParams.dateTimeHeaders.length > 0 ||
parseParams.dateHeaders.length > 0 ||
parseParams.timeHeaders.length > 0
) {
csvArrayData = this.updateDateTimeCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (parseParams.xlRules.length > 0) {
csvArrayData = this.updateXLRuleCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (!isComplete) {
let abortMsg = ''
if (missingHeaders.length === 0) {
abortMsg = 'No relevant data found in File !'
} else {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
}
// abort message is fired, return undefined
uploader.queue.pop()
return reject(abortMsg)
} else {
parseParams.headerShow = csvArrayHeaders
csvArrayData = csvArrayData.map((row: any) =>
row.map((col: any) => (col.t === 'n' ? col.v : col.w))
)
csvArrayData = csvArrayData.map((row: any) => {
return row.map((col: any, index: number) => {
if (!col && col !== 0) col = ''
/**
* Keeping this for the reference
* Code below used to convert JSON to CSV
* now the XLSX is converting to CSV
*/
// if (isNaN(col)) {
// // Match and replace the double quotes, ignore the first and last char
// // in case they are double quotes already
// col = col.replace(/(?<!^)"(?!$)/g, '""')
// if (col.search(/,/g) > -1 ||
// col.search(/\r|\n/g) > -1
// ) {
// // Missing quotes at the end
// if (col.search(/"$/g) < 0) {
// col = col + '"' // So we add them
// }
// // Missing quotes at the start
// if (col.search(/^"/g) < 0) {
// col = '"' + col // So we add them
// }
// }
// }
const colName = parseParams.headerShow[index]
const colRule = parseParams.dcValidator?.getRule(colName)
if (colRule?.type === 'numeric') {
if (isSpecialMissing(col) && !col.includes('.'))
col = '.' + col
}
return col
})
})
data = csvArrayData
// Apply licence rows limitation if exists, it is only affecting data
// which will be send to SAS
const strippedCsvArrayData = csvArrayData.slice(
0,
this.licenceState.value.submit_rows_limit
)
// To submit to sas service, we need clean version of CSV of file
// attached. XLSX will do the parsing and heavy lifting
// First we create worksheet of json (data we extracted)
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
skipHeader: true
})
// create CSV to be uploaded from worksheet
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
// Prepend headers
csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
// Blob from which CSV file will be created depending of the selected
// encoding
let blob: Blob
if (parseParams.encoding === 'WLATIN1') {
// WLATIN1
let encoded = iconv.decode(
Buffer.from(csvContentClean),
'CP-1252'
)
blob = new Blob([encoded], { type: 'application/csv' })
} else {
// UTF-8
blob = new Blob([csvContentClean], { type: 'application/csv' })
}
let newCSVFile: File = blobToFile(blob, filename + '.csv')
uploader.addToQueue([newCSVFile])
}
if (data.length === 0) {
return reject(
`Table in the file is empty. Data found on sheet: ${foundData.sheet}`
)
}
return resolve({
uploader,
data,
rangeSheetRes,
headerShow: parseParams.headerShow
})
}
reader.readAsArrayBuffer(file)
} else if (fileType.toLowerCase() === 'csv') {
if (this.licenceState.value.submit_rows_limit !== Infinity) {
uploader.queue.pop()
return reject(
'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
)
}
if (parseParams.encoding === 'WLATIN1') {
let reader = new FileReader()
const self = this
// Closure to capture the file information.
reader.onload = (theFile: any) => {
let encoded = iconv.decode(
Buffer.from(theFile.target.result),
'CP-1252'
)
let blob = new Blob([encoded], { type: fileType })
let encodedFile: File = blobToFile(blob, filename)
uploader.queue.pop()
uploader.addToQueue([encodedFile])
resolve({
uploader
})
}
reader.readAsArrayBuffer(file)
} else {
return resolve({
uploader
})
}
} else {
let abortMsg =
'Invalid file type "<b>' +
filename +
'</b>". Please upload csv or excel file.'
uploader.queue.pop()
return reject(abortMsg)
}
}) })
return spreadSheetUtil.parseExcelFile(
parseParams,
this.promptExcelPassword,
onParseStateChange,
onTableFoundEvent
)
} }
/** /**
* Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers * Read the file minimally just to get the sheet names, not reading full file
* @param wb Excel workbook * to help boost the performance
* @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range *
* @returns sheet names in string array
*/ */
private getRangeAndSheet( public async parseExcelSheetNames(file: File): Promise<{
wb: XLSX.WorkBook, sheetNames: string[]
parseParams: ParseParams password?: string
): SheetInfo { }> {
let data = [] return new Promise((resolve, reject) => {
const reader = new FileReader()
let rangeStartRow: number = 0 if (!file) {
let rangeStartCol: number = 0 console.warn('file is missing')
let startRow: number = -1 return resolve({ sheetNames: [] })
let endRow: number = -1
let sheetName: string = ''
let isComplete = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
wb.SheetNames.forEach((element: string) => {
// Checking for required data in each sheet in workbook/
if (isComplete) {
return
} }
missingHeaders = [] reader.onload = async (event: ProgressEvent<FileReader>) => {
sheetName = element if (!event?.target) {
const ws: XLSX.WorkSheet = wb.Sheets[sheetName] console.warn('File reader event.target is missing')
data = <AOA>XLSX.utils.sheet_to_json(ws, {
header: 1,
blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
defval: ''
})
if (data.length <= 1) {
return
}
let tempArr: string[] = []
parseParams.headerArray.forEach(() => tempArr.push(''))
data.push(tempArr)
let foundHeaders = false
data.forEach((row: any, index: number) => {
if (isComplete) {
return return
} }
if (foundHeaders) { let wb: XLSX.WorkBook | undefined = undefined
let isDataEnd = true let fileUnlocking: boolean = false
let isPkNull = false let password: string | undefined
const data = event.target.result
csvArrayHeadersLower.forEach((x) => { try {
const col = csvArrayHeadersMap[x] wb = XLSX.read(data, {
// Load file minimally to parse sheets
if (row[col] !== '' && row[col] !== undefined) { bookSheets: true,
isDataEnd = false type: 'binary'
} else {
if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
isPkNull = true
}
}
}) })
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
fileUnlocking = true
if (isDataEnd || isPkNull) { let passwordError = false
endRow = index
isComplete = true
} else {
if (startRow === -1) {
startRow = index
}
}
} else {
const rowLowerCase: string[] = row.map((x: any) =>
x.toString().toLowerCase()
)
// If in file there is no delete column, remove it from search of missing. while (fileUnlocking) {
// This way delete column will be optional to provide in file password = await this.promptExcelPassword({
if (!rowLowerCase.includes('_____delete__this__record_____')) { error: passwordError
const deleteIndex = csvArrayHeadersLower.indexOf(
'_____delete__this__record_____'
)
if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
}
foundHeaders = true
csvArrayHeadersLower.forEach((x) => {
if (rowLowerCase.indexOf(x) === -1) {
foundHeaders = false
}
})
let result = []
result = this.findValidHeaders(
rowLowerCase,
csvArrayHeadersLower,
index,
sheetName,
parseParams
)
if (result[0] === false) {
foundHeaders = false
if (result[1].length > 0) {
result[1].forEach((data: string) => {
missingHeaders.push(data)
}) })
}
} else {
csvArrayHeadersMap = result[1]
}
}
})
if (isComplete) { if (password) {
this.update_sheet_range(ws) try {
const worksheetSel = ws['!ref'] wb = XLSX.read(data, {
// Load file minimally to parse sheets
bookSheets: true,
type: 'binary',
password: password
})
if (worksheetSel) { fileUnlocking = false
const range = XLSX.utils.decode_range(ws['!ref'] || '') passwordError = false
rangeStartRow = range.s.r } catch (err: any) {
rangeStartCol = range.s.c passwordError = true
}
}
})
// If start row is still -1 that means first row of found range is empty if (!err.message.toLowerCase().includes('password')) {
if (startRow === -1) isComplete = false fileUnlocking = false
}
}
const returnObj: SheetInfo = { if (!password)
foundData: isComplete, return reject('Invalid password, failed to decrypt the file')
sheetName, } else {
startRow, fileUnlocking = false
endRow, return reject('No password provided')
csvArrayHeadersMap,
missingHeaders,
rangeStartRow,
rangeStartCol
}
return returnObj
}
private findValidHeaders(
row: string[],
headers: string[],
rowNumber: number,
tabName: string,
parseParams: ParseParams
): Array<any> {
let headersFound = false
const missingErrorArray = []
let j = 0
while (j < row.length) {
if (headersFound) {
// return;
} else {
if (headers.indexOf(row[j]) !== -1) {
let breakIndex
let rowStart = 0
let rowEnd = 0
let arrStart = 0
let foundHeadersArray: string[] = []
let spaceBreak = false
for (let i = j; i < row.length; i++) {
if (
row[i] === '' ||
(foundHeadersArray.indexOf(row[i]) !== -1 &&
this.isColHeader(row[i], parseParams.headerArray))
) {
if (row[i] === '') {
spaceBreak = true
} }
breakIndex = i
break
} else {
foundHeadersArray.push(row[i])
}
}
let tempArray: string[] = []
if (breakIndex !== undefined) {
tempArray = row.slice(j, breakIndex)
arrStart = j
rowEnd = breakIndex
if (spaceBreak) {
rowStart = j
j = breakIndex
} else {
rowStart = j
j = breakIndex - 1
} }
} else { } else {
tempArray = row.slice(j) return reject('Error reading the file')
rowStart = j
arrStart = j
rowEnd = row.length
j = row.length
}
let foundHeaders = true
//We check if there are missing headers
headers.forEach((x) => {
if (tempArray.indexOf(x) === -1) {
foundHeaders = false
}
})
if (foundHeaders) {
headersFound = true
let mapHeaders: any[] = headers
let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
map[obj] = -1
return map
}, {})
let temp = row.slice(rowStart, rowEnd)
headers.forEach((x) => {
csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
})
return [true, csvArrayHeadersMap]
} else {
let missingHeaders = getMissingHeaders(tempArray, headers)
let missingMessage = '<b>TAB(' + tabName + ')</b>'
missingErrorArray.push([
missingMessage +
' - ' +
missingHeaders[1].join(',') +
' ( missing ' +
missingHeaders[0].join(',') +
' )',
missingHeaders[1].length
])
} }
} }
if (!wb) return reject('Error parsing the workbook')
try {
const sheetNames = wb.SheetNames
return resolve({
sheetNames: sheetNames,
password: password
})
} catch (e) {
console.error(e)
}
} }
j++
} reader.onerror = function (ex) {
return [false, missingErrorArray] console.log(ex)
}
reader.readAsBinaryString(file)
})
} }
private isColHeader(col: string, headerArray: string[]) { public bytesToMB(size: number): number {
return headerArray.indexOf(col.toUpperCase()) > -1 return parseFloat((size / (1024 * 1024)).toFixed(2))
}
/**
* Function that updates the !ref range value provided in official docs.
* @param ws worksheet to be updated
*/
private update_sheet_range(ws: XLSX.WorkSheet) {
const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
Object.keys(ws)
.filter(function (x) {
return x.charAt(0) != '!'
})
.map(XLSX.utils.decode_cell)
.forEach(function (x: any) {
range.s.c = Math.min(range.s.c, x.c)
range.s.r = Math.min(range.s.r, x.r)
range.e.c = Math.max(range.e.c, x.c)
range.e.r = Math.max(range.e.r, x.r)
})
ws['!ref'] = XLSX.utils.encode_range(range)
} }
/** /**
* When excel is password protected we will display the password prompt for user to type password in. * When excel is password protected we will display the password prompt for user to type password in.
* @returns Password user input or undefined if discarded by user * @returns Password user input or undefined if discarded by user
*/ */
private promptExcelPassword(): Promise<string | undefined> { private promptExcelPassword = (
options?: OpenOptions
): Promise<string | undefined> => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.excelPasswordModalService.open().subscribe((result: Result) => { this.excelPasswordModalService
resolve(result.password) .open(options)
}) .subscribe((result: Result) => {
resolve(result.password)
})
}) })
} }
private updateDateTimeCols(
headers: any,
data: any,
parseParams: ParseParams
) {
if (parseParams.dateHeaders.length > 0) {
const dateCols: number[] = []
parseParams.dateHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
dateCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
} else {
if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date)
}
}
}
row[element] = obj
})
})
}
if (parseParams.timeHeaders.length > 0) {
let timeCols: number[] = []
parseParams.timeHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
timeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
timeCols.forEach((element) => {
const obj = row[element]
if (
isStringNumber(obj.v) ||
isStringDecimal(obj.v) ||
obj.v.includes('E-')
) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateToUtcTime(date)
}
row[element] = obj
})
})
}
if (parseParams.dateTimeHeaders.length > 0) {
let dateTimeCols: number[] = []
parseParams.dateTimeHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
dateTimeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateTimeCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
} else {
if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
let str = obj.v.substring(0, obj.v.indexOf(':'))
str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
obj.v = str
}
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date) + ' ' + dateToTime(date)
}
}
row[element] = obj
})
})
}
return data
}
private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
if (parseParams.xlRules.length > 0) {
const xlRuleCols: any = []
parseParams.xlRules.forEach((element: any) => {
if (headers.indexOf(element.XL_COLUMN) !== -1) {
element['index'] = headers.indexOf(element.XL_COLUMN)
xlRuleCols.push(element)
}
})
data.forEach((row: any[]) => {
xlRuleCols.forEach((element: any) => {
const obj = row[element.index]
if (element.XL_RULE === 'FORMULA') {
if ('f' in obj) {
if (obj['t'] === 'n') {
obj['v'] = '=' + obj['f']
} else {
obj['w'] = '=' + obj['f']
}
}
}
row[element] = obj
})
})
}
return data
}
private toBstr(res: any) {
let bytes = new Uint8Array(res)
let binary = ''
let length = bytes.byteLength
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i])
}
return binary
}
} }

38
client/src/app/shared/excel-password-modal/excel-password-modal.component.html
View File

@ -1,4 +1,4 @@
<ng-container *ngIf="options$ | async as options"> <div *ngIf="options$ | async as options" class="excel-password-root">
<clr-modal <clr-modal
[clrModalOpen]="options.open" [clrModalOpen]="options.open"
[clrModalSize]="'md'" [clrModalSize]="'md'"
@ -11,6 +11,7 @@
<p class="m-0">Please enter password:</p> <p class="m-0">Please enter password:</p>
<input <input
#filePasswordInput #filePasswordInput
[(ngModel)]="passwordInput"
data-lpignore="true" data-lpignore="true"
autocomplete="off" autocomplete="off"
id="filePasswordInput" id="filePasswordInput"
@ -19,20 +20,25 @@
/> />
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<p *ngIf="options.error" class="m-0 color-red"> <div>
Sorry that didn't work, try again. <p *ngIf="options.error" class="m-0 color-red">
</p> Sorry that didn't work, try again.
<button type="button" class="btn btn-sm btn-outline" (click)="close()"> </p>
Cancel </div>
</button>
<button <div class="buttons">
type="button" <button type="button" class="btn btn-sm btn-outline" (click)="close()">
class="btn btn-sm btn-success-outline" Cancel
[disabled]="filePasswordInput.value.length < 1" </button>
(click)="close(filePasswordInput.value)" <button
> type="button"
Unlock class="btn btn-sm btn-success-outline"
</button> [disabled]="filePasswordInput.value.length < 1"
(click)="close(filePasswordInput.value)"
>
Unlock
</button>
</div>
</div> </div>
</clr-modal> </clr-modal>
</ng-container> </div>

18
client/src/app/shared/excel-password-modal/excel-password-modal.component.scss Normal file
View File

@ -0,0 +1,18 @@
.excel-password-root {
::ng-deep {
.modal {
z-index: 1060;
}
}
}
.modal-footer {
display: flex;
align-items: center;
justify-content: space-between;
.buttons {
display: flex;
gap: 5px;
}
}

16
client/src/app/shared/excel-password-modal/excel-password-modal.component.ts
View File

@ -1,24 +1,24 @@
import { Component } from '@angular/core' import { Component } from '@angular/core'
import { Observable } from 'rxjs' import { Observable } from 'rxjs'
import { import { ExcelPasswordModalService } from './excel-password-modal.service'
ExcelPasswordModalService, import { Options } from './models/options.interface'
Options
} from './excel-password-modal.service'
@Component({ @Component({
selector: 'app-excel-password-modal', selector: 'app-excel-password-modal',
styleUrls: ['./excel-password-modal.component.scss'],
templateUrl: './excel-password-modal.component.html' templateUrl: './excel-password-modal.component.html'
}) })
export class ExcelPasswordModalComponent { export class ExcelPasswordModalComponent {
options$: Observable<Options> options$: Observable<Options> = this.excelPasswordModalService.optionsSubject$
fileUnlockError: boolean = false fileUnlockError: boolean = false
constructor(private excelPasswordModalService: ExcelPasswordModalService) { passwordInput: string = ''
this.options$ = this.excelPasswordModalService.optionsSubject$
} constructor(private excelPasswordModalService: ExcelPasswordModalService) {}
close(password?: string) { close(password?: string) {
this.passwordInput = ''
this.excelPasswordModalService.close(password) this.excelPasswordModalService.close(password)
} }
} }

14
client/src/app/shared/excel-password-modal/excel-password-modal.service.ts
View File

@ -1,17 +1,7 @@
import { Injectable } from '@angular/core' import { Injectable } from '@angular/core'
import { Subject, Observable } from 'rxjs' import { Subject, Observable } from 'rxjs'
import { OpenOptions, Options } from './models/options.interface'
export interface Options extends OpenOptions { import { Result } from './models/result.interface'
open: boolean
}
export interface Result {
password: string | undefined
}
export interface OpenOptions {
error?: boolean
}
@Injectable({ @Injectable({
providedIn: 'root' providedIn: 'root'

7
client/src/app/shared/excel-password-modal/models/options.interface.ts Normal file
View File

@ -0,0 +1,7 @@
export interface OpenOptions {
error?: boolean
}
export interface Options extends OpenOptions {
open: boolean
}

3
client/src/app/shared/excel-password-modal/models/result.interface.ts Normal file
View File

@ -0,0 +1,3 @@
export interface Result {
password: string | undefined
}

1
client/src/app/shared/sidebar/sidebar.component.scss
View File

@ -2,7 +2,6 @@ $sidebarWidth: 272px;
.clr-vertical-nav .nav-link.active { .clr-vertical-nav .nav-link.active {
background-color: transparent; background-color: transparent;
padding-left: 5px;
} }
clr-vertical-nav { clr-vertical-nav {

3
client/src/app/shared/spreadsheet-util/random.class.ts Normal file
View File

@ -0,0 +1,3 @@
export class RandomClass {
constructor(jo: string) {}
}

954
client/src/app/shared/spreadsheet-util/spreadsheet-util.ts Normal file
View File

@ -0,0 +1,954 @@
import { isSpecialMissing } from '@sasjs/utils/input/validators'
import {
dateFormat,
dateToUtcTime,
dateToTime
} from 'src/app/editor/utils/date.utils'
import {
getMissingHeaders,
excelDateToJSDate
} from 'src/app/editor/utils/grid.utils'
import {
isStringNumber,
isStringDecimal
} from 'src/app/editor/utils/types.utils'
import { FileUploader } from 'src/app/models/FileUploader.class'
import SheetInfo from 'src/app/models/SheetInfo'
import { blobToFile } from 'src/app/xlmap/utils/file.utils'
import * as XLSX from '@sheet/crypto'
import { LicenceState } from 'src/app/models/LicenceState'
import { BehaviorSubject } from 'rxjs'
import { ParseParams } from 'src/app/models/ParseParams.interface'
import { ParseResult } from 'src/app/models/ParseResult.interface'
import { OpenOptions } from '../excel-password-modal/models/options.interface'
/**
* Used in combination with buffer
*/
import * as iconv from 'iconv-lite'
/**
* In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
* Which will be send to backend
*/
import { Buffer } from 'buffer'
type AOA = any[][]
export interface ConstructorParams {
licenceState: BehaviorSubject<LicenceState>
}
export class SpreadsheetUtil {
private licenceState: BehaviorSubject<LicenceState>
constructor(params: ConstructorParams) {
this.licenceState = params.licenceState
}
/**
* Parses attached file and searches fo the matching data
*
* @param parseParams params required for parsing the file
* @param onParseStateChange callback used to inform about parsing state
* so the user of the function can update the UI with latest info
* @param onTableFoundEvent callback fired when table range is found in the file
*
* @returns parsed list of files to upload and JSON data ready for HOT usage
*/
public parseExcelFile(
parseParams: ParseParams,
promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
onParseStateChange?: (uploadState: string) => void,
onTableFoundEvent?: (info: string) => void
): Promise<ParseResult | undefined> {
return new Promise((resolve, reject) => {
// If file size is bigger then 2 MB we need to load its bytes in chunks
const sizeInMB = this.bytesToMB(parseParams.file.size)
if (sizeInMB > 2) {
}
let data: any[] = []
const uploader: FileUploader = parseParams.uploader || new FileUploader()
const file: File = parseParams.file
const filename = file.name
if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
if (onParseStateChange)
onParseStateChange(`Loading ${filename} into the browser`)
let foundData = {
sheet: ''
}
let fileType = filename.slice(
filename.lastIndexOf('.') + 1,
filename.lastIndexOf('.') + 4
)
if (fileType.toLowerCase() === 'xls') {
let reader: FileReader = new FileReader()
const self = this
reader.onload = async (theFile: any) => {
/* read workbook */
const bstr = this.toBstr(theFile.target.result)
let wb: XLSX.WorkBook | undefined = undefined
let fileUnlocking: boolean = false
const xlsxOptions: XLSX.ParsingOptions = {
type: 'binary',
cellDates: false,
cellFormula: true,
cellStyles: true,
cellNF: false,
cellText: false,
password: parseParams.password
}
try {
wb = await this.xlsxRead(bstr, {
...xlsxOptions
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
fileUnlocking = true
let passwordError = false
while (fileUnlocking) {
const password = await promptExcelPassword({
error: passwordError
})
if (password) {
try {
wb = await this.xlsxRead(bstr, {
...xlsxOptions,
password: password
})
fileUnlocking = false
passwordError = false
} catch (err: any) {
passwordError = true
if (!err.message.toLowerCase().includes('password')) {
fileUnlocking = false
}
}
} else {
fileUnlocking = false
}
}
} else {
return reject('Error reading the file')
}
}
if (!wb) {
return reject('No workbook found.')
}
/* save data */
let isComplete: boolean = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
let csvArrayData: any[] = []
const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
wb,
parseParams
)
missingHeaders = rangeSheetRes.missingHeaders
if (rangeSheetRes.foundData) {
isComplete = true
csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
if (onParseStateChange)
onParseStateChange(
`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
)
let startAddress = ''
let endAddress = ''
for (
let row = rangeSheetRes.startRow;
row < rangeSheetRes.endRow;
++row
) {
const arr: any[] = []
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
const addr = XLSX.utils.encode_cell({
r: rangeSheetRes.rangeStartRow + row,
c: rangeSheetRes.rangeStartCol + col
})
let cell
if (!ws[addr]) {
cell = { v: '' }
} else {
cell = ws[addr]
}
if (startAddress === '' && ws[addr]) startAddress = addr
endAddress = addr
arr.push(cell)
})
// If we found at least one non empty value it means it is not empty row
// othervise, it is empty row
let arrNonEmptyValue = arr.find((x) => x.v !== '')
if (arrNonEmptyValue) csvArrayData.push(arr)
}
rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
if (onTableFoundEvent)
onTableFoundEvent(
`Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
)
} else {
missingHeaders = rangeSheetRes.missingHeaders
}
if (missingHeaders.length > 0) {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
let abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
uploader.queue.pop()
return reject(abortMsg)
}
// If first row is empty, that means no data has been found
if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
let abortMsg = 'No relevant data found in File !'
uploader.queue.pop()
return reject(abortMsg)
}
if (
parseParams.dateTimeHeaders.length > 0 ||
parseParams.dateHeaders.length > 0 ||
parseParams.timeHeaders.length > 0
) {
csvArrayData = this.updateDateTimeCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (parseParams.xlRules.length > 0) {
csvArrayData = this.updateXLRuleCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (!isComplete) {
let abortMsg = ''
if (missingHeaders.length === 0) {
abortMsg = 'No relevant data found in File !'
} else {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
}
// abort message is fired, return undefined
uploader.queue.pop()
return reject(abortMsg)
} else {
parseParams.headerShow = csvArrayHeaders
csvArrayData = csvArrayData.map((row: any) =>
row.map((col: any) => (col.t === 'n' ? col.v : col.w))
)
csvArrayData = csvArrayData.map((row: any) => {
return row.map((col: any, index: number) => {
if (!col && col !== 0) col = ''
/**
* Keeping this for the reference
* Code below used to convert JSON to CSV
* now the XLSX is converting to CSV
*/
// if (isNaN(col)) {
// // Match and replace the double quotes, ignore the first and last char
// // in case they are double quotes already
// col = col.replace(/(?<!^)"(?!$)/g, '""')
// if (col.search(/,/g) > -1 ||
// col.search(/\r|\n/g) > -1
// ) {
// // Missing quotes at the end
// if (col.search(/"$/g) < 0) {
// col = col + '"' // So we add them
// }
// // Missing quotes at the start
// if (col.search(/^"/g) < 0) {
// col = '"' + col // So we add them
// }
// }
// }
const colName = parseParams.headerShow[index]
const colRule = parseParams.dcValidator?.getRule(colName)
if (colRule?.type === 'numeric') {
if (isSpecialMissing(col) && !col.includes('.'))
col = '.' + col
}
return col
})
})
data = csvArrayData
// Apply licence rows limitation if exists, it is only affecting data
// which will be send to SAS
const strippedCsvArrayData = csvArrayData.slice(
0,
this.licenceState.value.submit_rows_limit
)
// To submit to sas service, we need clean version of CSV of file
// attached. XLSX will do the parsing and heavy lifting
// First we create worksheet of json (data we extracted)
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
skipHeader: true
})
// create CSV to be uploaded from worksheet
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
// Prepend headers
csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
// Blob from which CSV file will be created depending of the selected
// encoding
let blob: Blob
if (parseParams.encoding === 'WLATIN1') {
// WLATIN1
let encoded = iconv.decode(
Buffer.from(csvContentClean),
'CP-1252'
)
blob = new Blob([encoded], { type: 'application/csv' })
} else {
// UTF-8
blob = new Blob([csvContentClean], { type: 'application/csv' })
}
let newCSVFile: File = blobToFile(blob, filename + '.csv')
uploader.addToQueue([newCSVFile])
}
if (data.length === 0) {
return reject(
`Table in the file is empty. Data found on sheet: ${foundData.sheet}`
)
}
return resolve({
uploader,
data,
rangeSheetRes,
headerShow: parseParams.headerShow
})
}
reader.readAsArrayBuffer(file)
} else if (fileType.toLowerCase() === 'csv') {
if (this.licenceState.value.submit_rows_limit !== Infinity) {
uploader.queue.pop()
return reject(
'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
)
}
if (parseParams.encoding === 'WLATIN1') {
let reader = new FileReader()
const self = this
// Closure to capture the file information.
reader.onload = (theFile: any) => {
let encoded = iconv.decode(
Buffer.from(theFile.target.result),
'CP-1252'
)
let blob = new Blob([encoded], { type: fileType })
let encodedFile: File = blobToFile(blob, filename)
uploader.queue.pop()
uploader.addToQueue([encodedFile])
resolve({
uploader
})
}
reader.readAsArrayBuffer(file)
} else {
return resolve({
uploader
})
}
} else {
let abortMsg =
'Invalid file type "<b>' +
filename +
'</b>". Please upload csv or excel file.'
uploader.queue.pop()
return reject(abortMsg)
}
})
}
public bytesToMB(size: number): number {
return parseFloat((size / (1024 * 1024)).toFixed(2))
}
/**
* XLSX Read wrapper which uses Web Worker to read the file and not block
* the UI while reading. It will allow reading bigger files.
* If worker fails, fallback is regular file read.
* @param data
* @param opts
* @returns
*/
private xlsxRead(
data: any,
opts?: XLSX.ParsingOptions | undefined
): Promise<XLSX.WorkBook> {
return new Promise((resolve, reject) => {
if (opts && opts.password) {
console.info('Not using worker to parse the XLSX - has password')
// At the moment worker can't use crypto version of SheetJS because of
// 'global not defined' issue
return resolve(XLSX.read(data, opts))
}
if (typeof Worker === 'undefined') {
console.info(
'Not using worker to parse the XLSX - no Worker available in this environment'
)
// Web workers are not supported in this environment.
// You should add a fallback so that your program still executes correctly.
return resolve(XLSX.read(data, opts))
}
// Ultimately use Web Worker to parse the excel
console.info('Using worker to parse the XLSX')
const worker = new Worker(
new URL('../../spreadsheet.worker', import.meta.url)
)
worker.onmessage = ({ data }) => {
if (data.event === 'reading_end') {
resolve(data.workbook)
} else if (data.error) {
reject(data.error)
} else {
console.info(
'Worker failed to parse the XLSX - fallback to non worker parsing'
)
// Fallback to reading without Worker
resolve(XLSX.read(data, opts))
}
}
worker.postMessage({
data,
opts
})
// Big timeout (10 minutes) in case Worker fails and no response
// and read the file with fallback method without worker
setTimeout(() => {
return resolve(XLSX.read(data, opts))
}, 600 * 1000) // 10 minutes
})
}
/**
* Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
* @param wb Excel workbook
* @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
*/
private getRangeAndSheet(
wb: XLSX.WorkBook,
parseParams: ParseParams
): SheetInfo {
let data = []
let rangeStartRow: number = 0
let rangeStartCol: number = 0
let startRow: number = -1
let endRow: number = -1
let sheetName: string = ''
let isComplete = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
wb.SheetNames.forEach((element: string) => {
// Checking for required data in each sheet in workbook/
if (isComplete) {
return
}
missingHeaders = []
sheetName = element
const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
data = <AOA>XLSX.utils.sheet_to_json(ws, {
header: 1,
blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
defval: ''
})
if (data.length <= 1) {
return
}
let tempArr: string[] = []
parseParams.headerArray.forEach(() => tempArr.push(''))
data.push(tempArr)
let foundHeaders = false
data.forEach((row: any, index: number) => {
if (isComplete) {
return
}
if (foundHeaders) {
let isDataEnd = true
let isPkNull = false
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
if (row[col] !== '' && row[col] !== undefined) {
isDataEnd = false
} else {
if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
isPkNull = true
}
}
})
if (isDataEnd || isPkNull) {
endRow = index
isComplete = true
} else {
if (startRow === -1) {
startRow = index
}
}
} else {
const rowLowerCase: string[] = row.map((x: any) =>
x.toString().toLowerCase()
)
// If in file there is no delete column, remove it from search of missing.
// This way delete column will be optional to provide in file
if (!rowLowerCase.includes('_____delete__this__record_____')) {
const deleteIndex = csvArrayHeadersLower.indexOf(
'_____delete__this__record_____'
)
if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
}
foundHeaders = true
csvArrayHeadersLower.forEach((x) => {
if (rowLowerCase.indexOf(x) === -1) {
foundHeaders = false
}
})
let result = []
result = this.findValidHeaders(
rowLowerCase,
csvArrayHeadersLower,
index,
sheetName,
parseParams
)
if (result[0] === false) {
foundHeaders = false
if (result[1].length > 0) {
result[1].forEach((data: string) => {
missingHeaders.push(data)
})
}
} else {
csvArrayHeadersMap = result[1]
}
}
})
if (isComplete) {
this.update_sheet_range(ws)
const worksheetSel = ws['!ref']
if (worksheetSel) {
const range = XLSX.utils.decode_range(ws['!ref'] || '')
rangeStartRow = range.s.r
rangeStartCol = range.s.c
}
}
})
// If start row is still -1 that means first row of found range is empty
if (startRow === -1) isComplete = false
const returnObj: SheetInfo = {
foundData: isComplete,
sheetName,
startRow,
endRow,
csvArrayHeadersMap,
missingHeaders,
rangeStartRow,
rangeStartCol
}
return returnObj
}
private findValidHeaders(
row: string[],
headers: string[],
rowNumber: number,
tabName: string,
parseParams: ParseParams
): Array<any> {
let headersFound = false
const missingErrorArray = []
let j = 0
while (j < row.length) {
if (headersFound) {
// return;
} else {
if (headers.indexOf(row[j]) !== -1) {
let breakIndex
let rowStart = 0
let rowEnd = 0
let arrStart = 0
let foundHeadersArray: string[] = []
let spaceBreak = false
for (let i = j; i < row.length; i++) {
if (
row[i] === '' ||
(foundHeadersArray.indexOf(row[i]) !== -1 &&
this.isColHeader(row[i], parseParams.headerArray))
) {
if (row[i] === '') {
spaceBreak = true
}
breakIndex = i
break
} else {
foundHeadersArray.push(row[i])
}
}
let tempArray: string[] = []
if (breakIndex !== undefined) {
tempArray = row.slice(j, breakIndex)
arrStart = j
rowEnd = breakIndex
if (spaceBreak) {
rowStart = j
j = breakIndex
} else {
rowStart = j
j = breakIndex - 1
}
} else {
tempArray = row.slice(j)
rowStart = j
arrStart = j
rowEnd = row.length
j = row.length
}
let foundHeaders = true
//We check if there are missing headers
headers.forEach((x) => {
if (tempArray.indexOf(x) === -1) {
foundHeaders = false
}
})
if (foundHeaders) {
headersFound = true
let mapHeaders: any[] = headers
let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
map[obj] = -1
return map
}, {})
let temp = row.slice(rowStart, rowEnd)
headers.forEach((x) => {
csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
})
return [true, csvArrayHeadersMap]
} else {
let missingHeaders = getMissingHeaders(tempArray, headers)
let missingMessage = '<b>TAB(' + tabName + ')</b>'
missingErrorArray.push([
missingMessage +
' - ' +
missingHeaders[1].join(',') +
' ( missing ' +
missingHeaders[0].join(',') +
' )',
missingHeaders[1].length
])
}
}
}
j++
}
return [false, missingErrorArray]
}
private isColHeader(col: string, headerArray: string[]) {
return headerArray.indexOf(col.toUpperCase()) > -1
}
/**
* Function that updates the !ref range value provided in official docs.
* @param ws worksheet to be updated
*/
private update_sheet_range(ws: XLSX.WorkSheet) {
const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
Object.keys(ws)
.filter(function (x) {
return x.charAt(0) != '!'
})
.map(XLSX.utils.decode_cell)
.forEach(function (x: any) {
range.s.c = Math.min(range.s.c, x.c)
range.s.r = Math.min(range.s.r, x.r)
range.e.c = Math.max(range.e.c, x.c)
range.e.r = Math.max(range.e.r, x.r)
})
ws['!ref'] = XLSX.utils.encode_range(range)
}
/**
* When excel is password protected we will display the password prompt for user to type password in.
* @returns Password user input or undefined if discarded by user
*/
// private promptExcelPassword(): Promise<string | undefined> {
// return new Promise((resolve, reject) => {
// this.excelPasswordModalService.open().subscribe((result: Result) => {
// resolve(result.password)
// })
// })
// }
private updateDateTimeCols(
headers: any,
data: any,
parseParams: ParseParams
) {
if (parseParams.dateHeaders.length > 0) {
const dateCols: number[] = []
parseParams.dateHeaders.forEach((element: any) => {
if (headers.indexOf(element) !== -1) {
dateCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
} else {
if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date)
}
}
}
row[element] = obj
})
})
}
if (parseParams.timeHeaders.length > 0) {
let timeCols: number[] = []
parseParams.timeHeaders.forEach((element: any) => {
if (headers.indexOf(element) !== -1) {
timeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
timeCols.forEach((element) => {
const obj = row[element]
if (
isStringNumber(obj.v) ||
isStringDecimal(obj.v) ||
obj.v.includes('E-')
) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateToUtcTime(date)
}
row[element] = obj
})
})
}
if (parseParams.dateTimeHeaders.length > 0) {
let dateTimeCols: number[] = []
parseParams.dateTimeHeaders.forEach((element: any) => {
if (headers.indexOf(element) !== -1) {
dateTimeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateTimeCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
} else {
if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
let str = obj.v.substring(0, obj.v.indexOf(':'))
str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
obj.v = str
}
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date) + ' ' + dateToTime(date)
}
}
row[element] = obj
})
})
}
return data
}
private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
if (parseParams.xlRules.length > 0) {
const xlRuleCols: any = []
parseParams.xlRules.forEach((element: any) => {
if (headers.indexOf(element.XL_COLUMN) !== -1) {
element['index'] = headers.indexOf(element.XL_COLUMN)
xlRuleCols.push(element)
}
})
data.forEach((row: any[]) => {
xlRuleCols.forEach((element: any) => {
const obj = row[element.index]
if (element.XL_RULE === 'FORMULA') {
if ('f' in obj) {
if (obj['t'] === 'n') {
obj['v'] = '=' + obj['f']
} else {
obj['w'] = '=' + obj['f']
}
}
}
row[element] = obj
})
})
}
return data
}
private toBstr(res: any) {
let bytes = new Uint8Array(res)
let binary = ''
let length = bytes.byteLength
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i])
}
return binary
}
}

9
client/src/app/shared/terms/terms.component.ts
View File

@ -8,6 +8,7 @@ import {
import { SasService } from '../../services/sas.service' import { SasService } from '../../services/sas.service'
import * as marked from 'marked' import * as marked from 'marked'
import { EULA } from 'src/environments/_eula' import { EULA } from 'src/environments/_eula'
import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-terms', selector: 'app-terms',
@ -51,8 +52,12 @@ export class TermsComponent implements OnInit, AfterViewInit {
this.sasService this.sasService
.request(`public/registeruser`, table) .request(`public/registeruser`, table)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') { if (
res.adapterResponse.return &&
res.adapterResponse.return[0] &&
res.adapterResponse.return[0].MSG === 'SUCCESS'
) {
location.reload() location.reload()
} }
}) })

30
client/src/app/spreadsheet.worker.ts Normal file
View File

@ -0,0 +1,30 @@
/// <reference lib="webworker" />
/**
* We use normal version of the XLSX (SheetJS)
* Because at the moment "@sheet/crypto" can't work in the Web Worker environment
* Because of the missing "global" variable.
*/
import * as XLSX from 'xlsx'
addEventListener('message', ({ data }) => {
const { data: xldata, opts: xlopts } = data as {
data: any
opts?: any
}
try {
const workbook = XLSX.read(xldata, xlopts)
postMessage({
event: 'reading_end',
workbook: workbook
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
postMessage({
error: err
})
}
}
})

9
client/src/app/stage/stage.component.ts
View File

@ -8,6 +8,7 @@ import { HotTableInterface } from '../models/HotTable.interface'
import { LicenceService } from '../services/licence.service' import { LicenceService } from '../services/licence.service'
import { globals } from '../_globals' import { globals } from '../_globals'
import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model' import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-stage', selector: 'app-stage',
@ -180,10 +181,12 @@ export class StageComponent implements OnInit {
this.sasService this.sasService
.request('editors/restore', data) .request('editors/restore', data)
.then((res: EditorsRestoreServiceResponse) => { .then((res: RequestWrapperResponse<EditorsRestoreServiceResponse>) => {
if (res.restore_out) { if (res.adapterResponse.restore_out) {
this.route.navigate([`/stage`]).then(() => { this.route.navigate([`/stage`]).then(() => {
this.route.navigate([`/stage/${res.restore_out[0].LOADREF}`]) this.route.navigate([
`/stage/${res.adapterResponse.restore_out[0].LOADREF}`
])
}) })
} }
}) })

27
client/src/app/system/models/environment-info.model.ts
View File

@ -1,15 +1,16 @@
export interface EnvironmentInfo { export interface EnvironmentInfo {
SYSSITE: string SYSSITE?: string
SYSSCPL: string SYSSCPL?: string
SYSTCPIPHOSTNAME: string SYSTCPIPHOSTNAME?: string
SYSVLONG: string SYSVLONG?: string
MEMSIZE: string MEMSIZE?: string
SYSPROCESSMODE: string SYSPROCESSMODE?: string
SYSHOSTNAME: string SYSHOSTNAME?: string
SYSUSERID: string SYSUSERID?: string
SYSHOSTINFOLONG: string SYSHOSTINFOLONG?: string
SYSENCODING: string SYSENCODING?: string
AUTOEXEC: string AUTOEXEC?: string
ISADMIN: number ISADMIN?: number
DC_ADMIN_GROUP: string DC_ADMIN_GROUP?: string
APP_LOC?: string
} }

4
client/src/app/system/system.component.html
View File

@ -77,6 +77,10 @@
DC Admin Group: DC Admin Group:
<span class="dark">{{ environmentInfo?.DC_ADMIN_GROUP }}</span> <span class="dark">{{ environmentInfo?.DC_ADMIN_GROUP }}</span>
</p> </p>
<p cds-text="label" class="m-0">
App Location:
<span class="dark">{{ environmentInfo?.APP_LOC }}</span>
</p>
</div> </div>
</div> </div>

15
client/src/app/system/system.component.ts
View File

@ -9,6 +9,7 @@ import { AppInfo } from './models/app-info.model'
import { EnvironmentInfo } from './models/environment-info.model' import { EnvironmentInfo } from './models/environment-info.model'
import { AppSettingsService } from '../services/app-settings.service' import { AppSettingsService } from '../services/app-settings.service'
import { AppSettings } from '../models/AppSettings' import { AppSettings } from '../models/AppSettings'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-system', selector: 'app-system',
@ -51,7 +52,7 @@ export class SystemComponent implements OnInit {
this.environmentInfo = this.appService.getEnvironmentInfo() this.environmentInfo = this.appService.getEnvironmentInfo()
this.settings = this.appSettingsService.settings.value this.settings = this.appSettingsService.settings.value
if (this.environmentInfo) { if (this.environmentInfo.AUTOEXEC) {
this.environmentInfo.AUTOEXEC = decodeURIComponent( this.environmentInfo.AUTOEXEC = decodeURIComponent(
this.environmentInfo.AUTOEXEC this.environmentInfo.AUTOEXEC
) )
@ -83,12 +84,12 @@ export class SystemComponent implements OnInit {
this.sasService this.sasService
.request('admin/refreshcatalog', null) .request('admin/refreshcatalog', null)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.response = this.parseResponse(res) this.response = this.parseResponse(res.adapterResponse)
this.responseModal = true this.responseModal = true
}) })
.catch((err: any) => { .catch((err: any) => {
this.response = this.parseResponse(err) this.response = this.parseResponse(err.adapterResponse)
this.responseModal = true this.responseModal = true
}) })
.finally(() => { .finally(() => {
@ -101,12 +102,12 @@ export class SystemComponent implements OnInit {
this.sasService this.sasService
.request('admin/refreshtablelineage', null) .request('admin/refreshtablelineage', null)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.response = this.parseResponse(res) this.response = this.parseResponse(res.adapterResponse)
this.responseModal = true this.responseModal = true
}) })
.catch((err: any) => { .catch((err: any) => {
this.response = this.parseResponse(err) this.response = this.parseResponse(err.adapterResponse)
this.responseModal = true this.responseModal = true
}) })
.finally(() => { .finally(() => {

55
client/src/app/user/user.component.ts
View File

@ -6,6 +6,7 @@ import { Location } from '@angular/common'
import { SasService } from '../services/sas.service' import { SasService } from '../services/sas.service'
import { SASjsConfig } from '@sasjs/adapter' import { SASjsConfig } from '@sasjs/adapter'
import { ServerType } from '@sasjs/utils/types/serverType' import { ServerType } from '@sasjs/utils/types/serverType'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-user', selector: 'app-user',
@ -89,10 +90,10 @@ export class UserComponent implements OnInit {
} else { } else {
this.sasService this.sasService
.request('usernav/usermembers', null) .request('usernav/usermembers', null)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loading = false this.loading = false
this.users = res.users this.users = res.adapterResponse.users
globals.usernav.userList = res.users globals.usernav.userList = res.adapterResponse.users
}) })
} }
} else { } else {
@ -141,18 +142,18 @@ export class UserComponent implements OnInit {
this.sasService this.sasService
.request('usernav/usergroupsbymember', data) .request('usernav/usergroupsbymember', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loading = false this.loading = false
switch (this.serverType) { switch (this.serverType) {
case ServerType.Sas9: { case ServerType.Sas9: {
this.userInfo = res.info[0] this.userInfo = res.adapterResponse.info[0]
this.userEmails = res.emails this.userEmails = res.adapterResponse.emails
this.userEmailsCount = res.emails.length this.userEmailsCount = res.adapterResponse.emails.length
this.userRoles = res.roles this.userRoles = res.adapterResponse.roles
this.userRolesCount = res.roles.length this.userRolesCount = res.adapterResponse.roles.length
this.userLogins = res.logins this.userLogins = res.adapterResponse.logins
this.userLoginsCount = res.logins.length this.userLoginsCount = res.adapterResponse.logins.length
break break
} }
@ -162,7 +163,7 @@ export class UserComponent implements OnInit {
(userAr) => userAr.URI === uri (userAr) => userAr.URI === uri
) )
} else { } else {
const group = res.groups[0] const group = res.adapterResponse.groups[0]
this.userInfo = { this.userInfo = {
URI: group.ID, URI: group.ID,
@ -175,9 +176,9 @@ export class UserComponent implements OnInit {
} }
} }
this.userData = res this.userData = res.adapterResponse
this.userGroups = res.groups this.userGroups = res.adapterResponse.groups
this.userGroupsCount = res.groups.length this.userGroupsCount = res.adapterResponse.groups.length
}) })
} }
} }
@ -238,18 +239,18 @@ export class UserComponent implements OnInit {
let data = { iwant: [{ uri: user.URI }] } let data = { iwant: [{ uri: user.URI }] }
this.sasService this.sasService
.request('usernav/usergroupsbymember', data) .request('usernav/usergroupsbymember', data)
.then((res: any) => { .then((res: RequestWrapperResponse) => {
this.loading = false this.loading = false
switch (this.serverType) { switch (this.serverType) {
case ServerType.Sas9: { case ServerType.Sas9: {
this.userInfo = res.info[0] this.userInfo = res.adapterResponse.info[0]
this.userEmails = res.emails this.userEmails = res.adapterResponse.emails
this.userEmailsCount = res.emails.length this.userEmailsCount = res.adapterResponse.emails.length
this.userRoles = res.roles this.userRoles = res.adapterResponse.roles
this.userRolesCount = res.roles.length this.userRolesCount = res.adapterResponse.roles.length
this.userLogins = res.logins this.userLogins = res.adapterResponse.logins
this.userLoginsCount = res.logins.length this.userLoginsCount = res.adapterResponse.logins.length
break break
} }
@ -259,7 +260,7 @@ export class UserComponent implements OnInit {
(userAr) => userAr.URI === user.URI (userAr) => userAr.URI === user.URI
) )
} else { } else {
const group = res.groups[0] const group = res.adapterResponse.groups[0]
this.userInfo = { this.userInfo = {
URI: group.ID, URI: group.ID,
@ -272,9 +273,9 @@ export class UserComponent implements OnInit {
} }
} }
this.userData = res this.userData = res.adapterResponse
this.userGroups = res.groups this.userGroups = res.adapterResponse.groups
this.userGroupsCount = res.groups.length this.userGroupsCount = res.adapterResponse.groups.length
}) })
} }
} }

1
client/src/app/viewer/viewer.component.ts
View File

@ -37,6 +37,7 @@ import { DataFormat } from '../models/sas/common/DateFormat'
import { Libinfo } from '../models/sas/common/Libinfo' import { Libinfo } from '../models/sas/common/Libinfo'
import { LicenceService } from '../services/licence.service' import { LicenceService } from '../services/licence.service'
import { Location } from '@angular/common' import { Location } from '@angular/common'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({ @Component({
selector: 'app-viewer', selector: 'app-viewer',

13
client/src/app/xlmap/xlmap.component.ts
View File

@ -22,6 +22,7 @@ import {
} from '../services' } from '../services'
import { getCellAddress, getFinishingCell } from './utils/xl.utils' import { getCellAddress, getFinishingCell } from './utils/xl.utils'
import { blobToFile, byteArrayToBinaryString } from './utils/file.utils' import { blobToFile, byteArrayToBinaryString } from './utils/file.utils'
import { UploadFileResponse } from '../models/UploadFile'
interface XLMapRule { interface XLMapRule {
XLMAP_ID: string XLMAP_ID: string
@ -340,9 +341,9 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
.uploadFile(uploadUrl, filesToUpload, { .uploadFile(uploadUrl, filesToUpload, {
table: this.selectedXLMap.targetDS table: this.selectedXLMap.targetDS
}) })
.then((res: any) => { .then((res: UploadFileResponse) => {
if (res.sasjsAbort) { if (res.adapterResponse.sasjsAbort) {
const abortRes = res const abortRes = res.adapterResponse
const abortMsg = abortRes.sasjsAbort[0].MSG const abortMsg = abortRes.sasjsAbort[0].MSG
const macMsg = abortRes.sasjsAbort[0].MAC const macMsg = abortRes.sasjsAbort[0].MAC
@ -351,14 +352,14 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
SYSERRORTEXT: abortRes.SYSERRORTEXT, SYSERRORTEXT: abortRes.SYSERRORTEXT,
MAC: macMsg MAC: macMsg
}) })
} else if (res.sasparams) { } else if (res.adapterResponse.sasparams) {
const params = res.sasparams[0] const params = res.adapterResponse.sasparams[0]
const tableId = params.DSID const tableId = params.DSID
this.router.navigateByUrl('/stage/' + tableId) this.router.navigateByUrl('/stage/' + tableId)
} }
}) })
.catch((err: any) => { .catch((err: any) => {
this.eventService.catchResponseError('file upload', err) this.eventService.catchResponseError('file upload', err.response)
}) })
.finally(() => { .finally(() => {
this.status = Status.ReadyToSubmit this.status = Status.ReadyToSubmit

15
client/tsconfig.worker.json Normal file
View File

@ -0,0 +1,15 @@
/* To learn more about this file see: https://angular.io/config/tsconfig. */
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "./out-tsc/worker",
"lib": [
"es2018",
"webworker"
],
"types": []
},
"include": [
"src/**/*.worker.ts"
]
}