diff --git a/client/angular.json b/client/angular.json index cc51584..4bec685 100644 --- a/client/angular.json +++ b/client/angular.json @@ -69,7 +69,8 @@ ], "scripts": [ "node_modules/marked/marked.min.js" - ] + ], + "webWorkerTsConfig": "tsconfig.worker.json" }, "configurations": { "production": { @@ -148,7 +149,8 @@ "src/styles.scss" ], "scripts": [], - "karmaConfig": "karma.conf.js" + "karmaConfig": "karma.conf.js", + "webWorkerTsConfig": "tsconfig.worker.json" } }, "lint": { diff --git a/client/package-lock.json b/client/package-lock.json index d9815cb..acb8b16 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -52,6 +52,7 @@ "tslib": "^2.3.0", "vm": "^0.1.0", "webpack": "^5.91.0", + "xlsx": "^0.18.5", "zone.js": "~0.14.4" }, "devDependencies": { @@ -7122,6 +7123,14 @@ "node": ">=8.9.0" } }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/agent-base": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", @@ -8259,6 +8268,18 @@ "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "dev": true }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -8579,6 +8600,14 @@ "integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==", "dev": true }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -8966,6 +8995,17 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/create-ecdh": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", @@ -11578,6 +11618,14 @@ "node": ">= 0.6" } }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -19340,6 +19388,17 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/sshpk": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", @@ -21323,6 +21382,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -21469,6 +21544,26 @@ } } }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/xmldoc": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz", diff --git a/client/package.json b/client/package.json index 4f6c77e..964c897 100644 --- a/client/package.json +++ b/client/package.json @@ -80,6 +80,7 @@ "tslib": "^2.3.0", "vm": "^0.1.0", "webpack": "^5.91.0", + "xlsx": "^0.18.5", "zone.js": "~0.14.4" }, "devDependencies": { diff --git a/client/src/app/deploy/sections/manual/manual.component.ts b/client/src/app/deploy/sections/manual/manual.component.ts index 44aa5ea..aa614f0 100644 --- a/client/src/app/deploy/sections/manual/manual.component.ts +++ b/client/src/app/deploy/sections/manual/manual.component.ts @@ -1,6 +1,7 @@ import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core' import SASjs, { SASjsConfig } from '@sasjs/adapter' import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings' +import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse' import { DeployService } from 'src/app/services/deploy.service' import { EventService } from 'src/app/services/event.service' import { LoggerService } from 'src/app/services/logger.service' @@ -303,10 +304,10 @@ export class ManualComponent implements OnInit { this.sasService .request('public/startupservice', null) - .then((res: any) => { - this.loggerService.log(res) + .then((res: RequestWrapperResponse) => { + this.loggerService.log(res.adapterResponse) - if (res.saslibs) { + if (res.adapterResponse.saslibs) { this.validationState = 'success' } else { this.validationState = 'error' diff --git a/client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts b/client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts index ad0e2dc..aa568e3 100644 --- a/client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts +++ b/client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.ts @@ -3,6 +3,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core' import SASjs, { SASjsConfig } from '@sasjs/adapter' import { ServerType } from '@sasjs/utils/types/serverType' import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings' +import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse' import { SASGroup } from 'src/app/models/sas/public-getgroups.model' import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model' import { SasService } from 'src/app/services/sas.service' @@ -68,11 +69,11 @@ export class SasjsConfiguratorComponent implements OnInit { this.loading = true this.sasService.request('usernav/usergroupsbymember', null).then( - (res: any) => { - this.METAPERSON = res.MF_GETUSER - this.SYSUSERID = res.SYSUSERID - this.SYSHOSTNAME = res.SYSHOSTNAME - this.SYSVLONG = res.SYSVLONG + (res: RequestWrapperResponse) => { + this.METAPERSON = res.adapterResponse.MF_GETUSER + this.SYSUSERID = res.adapterResponse.SYSUSERID + this.SYSHOSTNAME = res.adapterResponse.SYSHOSTNAME + this.SYSVLONG = res.adapterResponse.SYSVLONG /* We would like to present a default DCPATH (deployment path) to the @@ -88,12 +89,14 @@ export class SasjsConfiguratorComponent implements OnInit { */ this.dcDirectory = this.tmpDirectories[ - ['L', 'H', 'A', 'S'].includes(res.SYSSCPL.substring(0, 1)) + ['L', 'H', 'A', 'S'].includes( + res.adapterResponse.SYSSCPL.substring(0, 1) + ) ? 'linux' : 'windows' ] - this.dcAdminGroupList = res.groups + this.dcAdminGroupList = res.adapterResponse.groups this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME this.loading = false diff --git a/client/src/app/editor/editor.component.ts b/client/src/app/editor/editor.component.ts index 75fe59f..9d4a7ea 100644 --- a/client/src/app/editor/editor.component.ts +++ b/client/src/app/editor/editor.component.ts @@ -57,10 +57,10 @@ import { LicenceService } from '../services/licence.service' import * as numbro from 'numbro' import * as languages from 'numbro/dist/languages.min' import { FileUploadEncoding } from '../models/FileUploadEncoding' -import { - ParseResult, - SpreadsheetService -} from '../services/spreadsheet.service' +import { SpreadsheetService } from '../services/spreadsheet.service' +import { UploadFileResponse } from '../models/UploadFile' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' +import { ParseResult } from '../models/ParseResult.interface' @Component({ selector: 'app-editor', @@ -571,20 +571,20 @@ export class EditorComponent implements OnInit, AfterViewInit { this.sasService .uploadFile(this.uploadUrl, filesToUpload, { table: this.libds }) .then( - (res: any) => { - if (typeof res.sasjsAbort === 'undefined') { - if (typeof res.sasparams === 'undefined') { + (res: UploadFileResponse) => { + if (typeof res.adapterResponse.sasjsAbort === 'undefined') { + if (typeof res.adapterResponse.sasparams === 'undefined') { return } else { this.uploadLoading = false - let params = res.sasparams[0] + let params = res.adapterResponse.sasparams[0] this.successEnable = true this.tableId = params.DSID this.router.navigateByUrl('/stage/' + this.tableId) } } else { // handle succesfull response - const abortRes = res + const abortRes = res.adapterResponse const abortMsg = abortRes.sasjsAbort[0].MSG const macMsg = abortRes.sasjsAbort[0].MAC @@ -609,7 +609,10 @@ export class EditorComponent implements OnInit, AfterViewInit { this.fileUploadInputCompList.first.nativeElement.value = '' } this.uploader.queue = [] - this.eventService.catchResponseError('file upload', err) + this.eventService.catchResponseError( + 'file upload', + err.adapterResponse + ) } ) } @@ -1561,9 +1564,11 @@ export class EditorComponent implements OnInit, AfterViewInit { 'editors/stagedata', this.$dataFormats ) - .then((res: any) => { - if (typeof res.sasparams !== 'undefined') { - this.router.navigateByUrl('/stage/' + res.sasparams[0].DSID) + .then((res: RequestWrapperResponse) => { + if (typeof res.adapterResponse.sasparams !== 'undefined') { + this.router.navigateByUrl( + '/stage/' + res.adapterResponse.sasparams[0].DSID + ) return true } @@ -1591,7 +1596,10 @@ export class EditorComponent implements OnInit, AfterViewInit { this.disableSubmit = true this.submit = false - let errorText = typeof err === 'string' ? err : JSON.stringify(err) + let errorText = + typeof err.adapterRespnse === 'string' + ? err.adapterRespnse + : JSON.stringify(err.adapterRespnse) this.eventService.showAbortModal( 'editors/stagedata', @@ -1983,8 +1991,8 @@ export class EditorComponent implements OnInit, AfterViewInit { suppressSuccessAbortModal: true, suppressErrorAbortModal: true }) - .then((res: any) => { - const colSource = res.dynamic_values.map( + .then((res: RequestWrapperResponse) => { + const colSource = res.adapterResponse.dynamic_values.map( (el: any) => el[this.cellValidationFields.RAW_VALUE] ) @@ -1998,8 +2006,8 @@ export class EditorComponent implements OnInit, AfterViewInit { ...this.cellValidationSource[validationSourceIndex], row: row, col: column, - values: res.dynamic_values, - extended_values: res.dynamic_extended_values + values: res.adapterResponse.dynamic_values, + extended_values: res.adapterResponse.dynamic_extended_values } } diff --git a/client/src/app/group/group.component.ts b/client/src/app/group/group.component.ts index 6c5be2f..32d8734 100644 --- a/client/src/app/group/group.component.ts +++ b/client/src/app/group/group.component.ts @@ -6,6 +6,7 @@ import { ServerType } from '@sasjs/utils/types/serverType' import { HelperService } from '../services/helper.service' import { SasService } from '../services/sas.service' import { globals } from '../_globals' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-group', @@ -82,11 +83,13 @@ export class GroupComponent implements OnInit { globals.usernav.groupList = groups }) } else { - this.sasService.request('public/getgroups', null).then((res: any) => { - this.loading = false - this.groups = res.groups - globals.usernav.groupList = res.groups - }) + this.sasService + .request('public/getgroups', null) + .then((res: RequestWrapperResponse) => { + this.loading = false + this.groups = res.adapterResponse.groups + globals.usernav.groupList = res.adapterResponse.groups + }) } } else { this.groups = globals.usernav.groupList @@ -128,14 +131,15 @@ export class GroupComponent implements OnInit { let data = { iwant: [{ groupid: this.paramURI }] } this.sasService .request('usernav/usermembersbygroup', data) - .then((res: any) => { - this.groupMembers = res.sasmembers - this.groupMemberCount = res.sasmembers.length - if (res.sasmembers[0] !== undefined) { + .then((res: RequestWrapperResponse) => { + this.groupMembers = res.adapterResponse.sasmembers + this.groupMemberCount = res.adapterResponse.sasmembers.length + if (res.adapterResponse.sasmembers[0] !== undefined) { this.loading = false - this.groupUri = res.sasmembers[0].URIMEM || this.paramURI - this.groupName = res.sasmembers[0].GROUPNAME - this.groupDesc = res.sasmembers[0].GROUPDESC + this.groupUri = + res.adapterResponse.sasmembers[0].URIMEM || this.paramURI + this.groupName = res.adapterResponse.sasmembers[0].GROUPNAME + this.groupDesc = res.adapterResponse.sasmembers[0].GROUPDESC if (!this.groupName) { this.groupName = this.paramURI @@ -202,13 +206,13 @@ export class GroupComponent implements OnInit { this.sasService .request('usernav/usermembersbygroup', data) - .then((res: any) => { + .then((res: RequestWrapperResponse) => { this.loading = false this.groupUri = group.GROUPURI this.groupName = group.GROUPNAME this.groupDesc = group.GROUPDESC - this.groupMembers = res.sasmembers - this.groupMemberCount = res.sasmembers.length + this.groupMembers = res.adapterResponse.sasmembers + this.groupMemberCount = res.adapterResponse.sasmembers.length }) } } diff --git a/client/src/app/licensing/licensing.component.ts b/client/src/app/licensing/licensing.component.ts index 61e7ad7..e570927 100644 --- a/client/src/app/licensing/licensing.component.ts +++ b/client/src/app/licensing/licensing.component.ts @@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core' import { ActivatedRoute, Router } from '@angular/router' import { AppService, LicenceService, SasService } from '../services' import { LicenseKeyData } from '../models/LicenseKeyData' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' enum LicenseActions { key = 'key', @@ -116,8 +117,12 @@ export class LicensingComponent implements OnInit { this.sasService .request('admin/registerkey', table) - .then((res: any) => { - if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') { + .then((res: RequestWrapperResponse) => { + if ( + res.adapterResponse.return && + res.adapterResponse.return[0] && + res.adapterResponse.return[0].MSG === 'SUCCESS' + ) { location.replace(location.href.split('#')[0]) } }) diff --git a/client/src/app/lineage/lineage.component.ts b/client/src/app/lineage/lineage.component.ts index ee1a740..8e9c704 100644 --- a/client/src/app/lineage/lineage.component.ts +++ b/client/src/app/lineage/lineage.component.ts @@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service' import * as saveSvg from 'save-svg-as-png' import { LoggerService } from '../services/logger.service' import { LicenceService } from '../services/licence.service' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' const moment = require('moment') @Component({ @@ -115,8 +116,8 @@ export class LineageComponent { await this.sasService .request('lineage/getmetacols', libTable) - .then((res: any) => { - this.columnsList = res.metacols + .then((res: RequestWrapperResponse) => { + this.columnsList = res.adapterResponse.metacols if (this.columnsList && this.columnsList.length > 0) { // this.column = this.columnsList[0]['COLURI'] @@ -174,8 +175,8 @@ export class LineageComponent { let libTable = { SASControlTable: [{ liburi: $event }] } await this.sasService .request('lineage/getmetatables', libTable) - .then((res: any) => { - this.tablesList = res.metatables + .then((res: RequestWrapperResponse) => { + this.tablesList = res.adapterResponse.metatables if (this.tablesList && this.tablesList.length > 0) { library['tables'] = this.tablesList @@ -295,8 +296,8 @@ export class LineageComponent { } else { await this.sasService .request('public/viewlibs', null) - .then((res: any) => { - this.libraryList = res.saslibs + .then((res: RequestWrapperResponse) => { + this.libraryList = res.adapterResponse.saslibs this.helperService.displayLibraries(this.libraryList) if (this.libraryList) { @@ -402,8 +403,8 @@ export class LineageComponent { return new Promise((resolve, reject) => { this.sasService .request('lineage/fetchtablelineage', libTable) - .then(async (res: any) => { - if (res.flatdata.length > 0) { + .then(async (res: RequestWrapperResponse) => { + if (res.adapterResponse.flatdata.length > 0) { if (this.licenceService.checkLineageLimit()) { this.eventService.showInfoModal( 'Notice', @@ -421,20 +422,22 @@ export class LineageComponent { } this.lineageTableName = - res.info[0].LIBREF + '.' + res.info[0].TABLENAME + res.adapterResponse.info[0].LIBREF + + '.' + + res.adapterResponse.info[0].TABLENAME - let dotArray = res.finalfinal + let dotArray = res.adapterResponse.finalfinal let vizTmp: string = '' for (let i = 0; i < dotArray.length; i++) { vizTmp += unescape(dotArray[i].LINE) + '\n' } - this.flatdata = res.flatdata + this.flatdata = res.adapterResponse.flatdata if (this.libraryList) { let libraryToSelect = this.libraryList.find((library: any) => - res.info[0].LIBURI.toUpperCase().includes( + res.adapterResponse.info[0].LIBURI.toUpperCase().includes( library.LIBRARYID.toUpperCase() ) ) @@ -450,7 +453,7 @@ export class LineageComponent { if (libraryToSelect['tables']) { tableToSelect = libraryToSelect['tables'].find((table: any) => table.TABLEURI.toUpperCase().includes( - res.info[0].TABLEID.toUpperCase() + res.adapterResponse.info[0].TABLEID.toUpperCase() ) ) @@ -495,10 +498,10 @@ export class LineageComponent { .replace(/\sds:/g, '\nds:') .replace(/\s\n/g, '\n') - this.idlookup = res.idlookup + this.idlookup = res.adapterResponse.idlookup - if (res.finalfinal.length > this.largeDotFileLimit) { - this.largeDotFileLines = res.finalfinal.length + if (res.adapterResponse.finalfinal.length > this.largeDotFileLimit) { + this.largeDotFileLines = res.adapterResponse.finalfinal.length } else { this.buildGraph() } @@ -619,8 +622,8 @@ export class LineageComponent { return new Promise((resolve, reject) => { this.sasService .request('lineage/fetchcollineage', libTable) - .then(async (res: any) => { - if (res.flatdata.length > 0) { + .then(async (res: RequestWrapperResponse) => { + if (res.adapterResponse.flatdata.length > 0) { if (this.licenceService.checkLineageLimit()) { this.eventService.showInfoModal( 'Notice', @@ -631,18 +634,21 @@ export class LineageComponent { } } - if (typeof res === 'string') { + if (typeof res.adapterResponse === 'string') { this.vizInput = 'digraph G {SAS Error}' this.buildGraph() return } - this.lineageTableName = res.info[0].LIBREF + '.' + res.info[0].TABNAME - this.lineageColumnName = res.info[0].COLNAME + this.lineageTableName = + res.adapterResponse.info[0].LIBREF + + '.' + + res.adapterResponse.info[0].TABNAME + this.lineageColumnName = res.adapterResponse.info[0].COLNAME - this.idlookup = res.idlookup + this.idlookup = res.adapterResponse.idlookup - let dotArray = res.fromsas + let dotArray = res.adapterResponse.fromsas let vizTmp: string = '' for (let i = 0; i < dotArray.length; i++) { vizTmp += unescape(dotArray[i].STRING) + '\n' @@ -653,11 +659,11 @@ export class LineageComponent { .replace(/\sds:/g, '\nds:') .replace(/\s\n/g, '\n') - this.flatdata = res.flatdata + this.flatdata = res.adapterResponse.flatdata if (this.libraryList) { let libraryToSelect = this.libraryList.find((library: any) => - res.info[0]?.LIBURI?.toUpperCase()?.includes( + res.adapterResponse.info[0]?.LIBURI?.toUpperCase()?.includes( library?.LIBRARYID?.toUpperCase() ) ) @@ -672,7 +678,8 @@ export class LineageComponent { if (libraryToSelect['tables']) { tableToSelect = libraryToSelect['tables'].find( - (table: any) => table.TABLEURI === res.info[0].TABURI + (table: any) => + table.TABLEURI === res.adapterResponse.info[0].TABURI ) if (tableToSelect) { @@ -714,8 +721,8 @@ export class LineageComponent { } } - if (res.fromsas.length > this.largeDotFileLimit) { - this.largeDotFileLines = res.fromsas.length + if (res.adapterResponse.fromsas.length > this.largeDotFileLimit) { + this.largeDotFileLines = res.adapterResponse.fromsas.length } else { this.buildGraph() } diff --git a/client/src/app/metadata/metadata.component.ts b/client/src/app/metadata/metadata.component.ts index 0964de9..32a10cc 100644 --- a/client/src/app/metadata/metadata.component.ts +++ b/client/src/app/metadata/metadata.component.ts @@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service' import { globals } from '../_globals' import { Injectable } from '@angular/core' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' interface MetaData { NAME: any @@ -109,44 +110,52 @@ export class MetadataComponent implements OnInit { this.metatypesLoading = false this.metaDataSearch = globals.metadata.metaDataSearch } else { - this.sasService.request('metanav/metatypes', null).then((res: any) => { - this.metaDataList = res.types - globals.metadata.metaDataList = this.metaDataList - this.loading = false - this.metatypesLoading = false - }) + this.sasService + .request('metanav/metatypes', null) + .then((res: RequestWrapperResponse) => { + this.metaDataList = res.adapterResponse.types + globals.metadata.metaDataList = this.metaDataList + this.loading = false + this.metatypesLoading = false + }) - this.sasService.request('metanav/metarepos', null).then((res: any) => { - let foundation = false - this.repositories = [] - for (let index = 0; index < res.outrepos.length; index++) { - this.repositories.push(res.outrepos[index].NAME) - if (res.outrepos[index].NAME === 'Foundation') { - foundation = true + this.sasService + .request('metanav/metarepos', null) + .then((res: RequestWrapperResponse) => { + let foundation = false + this.repositories = [] + for ( + let index = 0; + index < res.adapterResponse.outrepos.length; + index++ + ) { + this.repositories.push(res.adapterResponse.outrepos[index].NAME) + if (res.adapterResponse.outrepos[index].NAME === 'Foundation') { + foundation = true + } } - } - if (foundation) { - this.repository = 'Foundation' - } else { - this.repository = res.outrepos[0].NAME - } - globals.metadata.metaRepositories = this.repositories - globals.metadata.selectedRepository = this.repository - if (this.objectRoute) { - this.eventService.closeSidebar() - this.showData = true - let name = '' - let id = this.route.snapshot.params['objectID'] - // let temp = this.router.url.split("%20").join(" ").split("/").reverse(); - this.metaObjectList = [] - this.metaObjectList.push({ ID: id, NAME: name }) - this.metaObjectShowList = this.metaObjectList - this.metaObjectOnClick( - this.metaObjectShowList[0].ID, - this.metaObjectShowList[0] - ) - } - }) + if (foundation) { + this.repository = 'Foundation' + } else { + this.repository = res.adapterResponse.outrepos[0].NAME + } + globals.metadata.metaRepositories = this.repositories + globals.metadata.selectedRepository = this.repository + if (this.objectRoute) { + this.eventService.closeSidebar() + this.showData = true + let name = '' + let id = this.route.snapshot.params['objectID'] + // let temp = this.router.url.split("%20").join(" ").split("/").reverse(); + this.metaObjectList = [] + this.metaObjectList.push({ ID: id, NAME: name }) + this.metaObjectShowList = this.metaObjectList + this.metaObjectOnClick( + this.metaObjectShowList[0].ID, + this.metaObjectShowList[0] + ) + } + }) } } @@ -183,56 +192,64 @@ export class MetadataComponent implements OnInit { const data: any = { SASControlTable: [{ metatype: $event, repo: this.repository }] } - this.sasService.request('metanav/metaobjects', data).then((res: any) => { - this.metaObjectList = res.objects - this.getMetaObjectAttributes(this.metaObjectSize) - this.loading = false - this.assoTypeSelected = $event - this.eventService.closeSidebar() - this.showData = true - }) + this.sasService + .request('metanav/metaobjects', data) + .then((res: RequestWrapperResponse) => { + this.metaObjectList = res.adapterResponse.objects + this.getMetaObjectAttributes(this.metaObjectSize) + this.loading = false + this.assoTypeSelected = $event + this.eventService.closeSidebar() + this.showData = true + }) } public async selectmetaObject($event: any, metaData?: any) { let data: any = { SASControlTable: [{ objecturi: $event }] } - this.sasService.request('metanav/metadetails', data).then((res: any) => { - this.metaObjectAssociations = res.associations - this.root$ = of(this.getAssosiationsCount(res.associations)) - this.showAcc = true - this.showTable = true - let metaObjectName = res.attributes.find( - (x: any) => x.NAME === 'Name' - ).VALUE - this.assoObjectSelected = metaObjectName - metaData.NAME = metaObjectName - let url = this.router.url - if (this.objectRoute) { - // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME)); - this.location.replaceState( - url.slice(0, url.lastIndexOf('object')) + - 'object/' + - $event.slice(1 + $event.indexOf('\\')) + this.sasService + .request('metanav/metadetails', data) + .then((res: RequestWrapperResponse) => { + this.metaObjectAssociations = res.adapterResponse.associations + this.root$ = of( + this.getAssosiationsCount(res.adapterResponse.associations) ) - } else { - // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME)); - this.location.replaceState( - url + '/object/' + $event.slice(1 + $event.indexOf('\\')) - ) - } - this.metaObjectAttributes = res.attributes - }) + this.showAcc = true + this.showTable = true + let metaObjectName = res.adapterResponse.attributes.find( + (x: any) => x.NAME === 'Name' + ).VALUE + this.assoObjectSelected = metaObjectName + metaData.NAME = metaObjectName + let url = this.router.url + if (this.objectRoute) { + // this.location.replaceState(url.slice(0, url.lastIndexOf("object")) + "object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME)); + this.location.replaceState( + url.slice(0, url.lastIndexOf('object')) + + 'object/' + + $event.slice(1 + $event.indexOf('\\')) + ) + } else { + // this.location.replaceState(url + "/object/" + $event.slice(1 + $event.indexOf("\\")) + "/" + escape(metaData.NAME)); + this.location.replaceState( + url + '/object/' + $event.slice(1 + $event.indexOf('\\')) + ) + } + this.metaObjectAttributes = res.adapterResponse.attributes + }) } public async selectAssosiationsDetails($event: any, metaData?: any) { let data: any = { SASControlTable: [{ objecturi: $event }] } - this.sasService.request('metanav/metadetails', data).then((res: any) => { - this.metaObjectAttributes = res.attributes - this.showTable = true - }) + this.sasService + .request('metanav/metadetails', data) + .then((res: RequestWrapperResponse) => { + this.metaObjectAttributes = res.adapterResponse.attributes + this.showTable = true + }) } public getAssosiationsCount(assosiationList: Array) { @@ -244,7 +261,7 @@ export class MetadataComponent implements OnInit { details: [] }) } - let assocObj = assosiationsHash.get(assosiation.ASSOC) + let assocObj: any = assosiationsHash.get(assosiation.ASSOC) assocObj.count++ assocObj.details.push({ ASSOCURI: assosiation.ASSOCURI, @@ -254,7 +271,7 @@ export class MetadataComponent implements OnInit { }) } let assocGrouped: Array = [] - assosiationsHash.forEach(function (val, key) { + assosiationsHash.forEach(function (val: any, key) { assocGrouped.push({ ASSOC: key, count: val.count, @@ -294,9 +311,9 @@ export class MetadataComponent implements OnInit { } return this.sasService .request('metanav/metadetails', data) - .then((res: any) => { + .then((res: RequestWrapperResponse) => { this.showTable = true - this.metaObjectAttributes = res.attributes + this.metaObjectAttributes = res.adapterResponse.attributes this.assoObjectSelected = asso.NAME let url = this.router.url if (this.objectRoute) { @@ -314,7 +331,7 @@ export class MetadataComponent implements OnInit { asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\')) ) } - return this.getAssosiationsCount(res.associations) + return this.getAssosiationsCount(res.adapterResponse.associations) }) } diff --git a/client/src/app/models/ParseParams.interface.ts b/client/src/app/models/ParseParams.interface.ts new file mode 100644 index 0000000..f5d9e8e --- /dev/null +++ b/client/src/app/models/ParseParams.interface.ts @@ -0,0 +1,23 @@ +import { DcValidator } from '../shared/dc-validator/dc-validator' +import { FileUploadEncoding } from './FileUploadEncoding' +import { FileUploader } from './FileUploader.class' +import { ExcelRule } from './TableData' + +export interface ParseParams { + file: File + password?: string + dcValidator: DcValidator + /** + * Parse function will manipulate and return the uploader array which can be provided with files already in the queue + * Otherwise new empty instance will be created. + */ + uploader?: FileUploader + headerPks: string[] + headerArray: string[] + headerShow: string[] + timeHeaders: string[] + dateHeaders: string[] + dateTimeHeaders: string[] + xlRules: ExcelRule[] + encoding?: FileUploadEncoding +} diff --git a/client/src/app/models/ParseResult.interface.ts b/client/src/app/models/ParseResult.interface.ts new file mode 100644 index 0000000..299307f --- /dev/null +++ b/client/src/app/models/ParseResult.interface.ts @@ -0,0 +1,15 @@ +import { FileUploader } from './FileUploader.class' +import SheetInfo from './SheetInfo' + +export interface ParseResult { + /** + * In case of CSV file, won't be returned + */ + data?: any[] + /** + * In case of CSV file, won't be returned + */ + headerShow?: string[] + rangeSheetRes?: SheetInfo + uploader: FileUploader +} diff --git a/client/src/app/models/UploadFile.ts b/client/src/app/models/UploadFile.ts new file mode 100644 index 0000000..dff0969 --- /dev/null +++ b/client/src/app/models/UploadFile.ts @@ -0,0 +1,4 @@ +export interface UploadFileResponse { + adapterResponse: any + log?: string +} diff --git a/client/src/app/models/RequestWrapperOptions.ts b/client/src/app/models/request-wrapper/RequestWrapperOptions.ts similarity index 100% rename from client/src/app/models/RequestWrapperOptions.ts rename to client/src/app/models/request-wrapper/RequestWrapperOptions.ts diff --git a/client/src/app/models/request-wrapper/RequestWrapperResponse.ts b/client/src/app/models/request-wrapper/RequestWrapperResponse.ts new file mode 100644 index 0000000..11ec566 --- /dev/null +++ b/client/src/app/models/request-wrapper/RequestWrapperResponse.ts @@ -0,0 +1,4 @@ +export interface RequestWrapperResponse { + adapterResponse: responseType + log?: string +} diff --git a/client/src/app/multi-dataset/multi-dataset.component.html b/client/src/app/multi-dataset/multi-dataset.component.html index ed4efa9..182c2c4 100644 --- a/client/src/app/multi-dataset/multi-dataset.component.html +++ b/client/src/app/multi-dataset/multi-dataset.component.html @@ -22,8 +22,8 @@ /> - -
+ +
@@ -41,30 +41,54 @@ - +

Submitted tables:

- +
--> -
+
-

+

Selected file: {{ selectedFile.name }} Discard the file

+

+ File size: {{ selectedFile.sizeMB }} MB +

Paste or type the list of datasets to upload:

@@ -151,7 +178,7 @@
+ +
+
+ +
+
+ {{ activeParsedDataset.submitResult?.error | json }} +
+
+ +

Found in range: @@ -197,7 +287,17 @@ - No data found + No data found + + + Searching for the data... + +

@@ -262,9 +362,9 @@ - +

- +
-

- Found in range: - "{{ - activeSubmittedDataset.parseResult.rangeSheetRes?.sheetName - }}"!{{ - activeSubmittedDataset.parseResult.rangeSheetRes?.rangeAddress - }} -

Matched with dataset: {{ activeSubmittedDataset.libds }}{{ activeSubmittedCsvDataset.libds }}

Status: - SUCCESS - ERROR

Error details: @@ -331,18 +419,11 @@

-
-
+
- {{ activeSubmittedDataset.error | json }} + {{ activeSubmittedCsvDataset.error | json }}
diff --git a/client/src/app/multi-dataset/multi-dataset.component.scss b/client/src/app/multi-dataset/multi-dataset.component.scss index 97d7fe3..8dbcfc0 100644 --- a/client/src/app/multi-dataset/multi-dataset.component.scss +++ b/client/src/app/multi-dataset/multi-dataset.component.scss @@ -47,4 +47,8 @@ .licence-limit-notice { color: var(--cds-alias-status-warning-dark); +} + +.submission-results { + border-bottom: 1px solid #d3d3d3; } \ No newline at end of file diff --git a/client/src/app/multi-dataset/multi-dataset.component.ts b/client/src/app/multi-dataset/multi-dataset.component.ts index d276f81..793372c 100644 --- a/client/src/app/multi-dataset/multi-dataset.component.ts +++ b/client/src/app/multi-dataset/multi-dataset.component.ts @@ -1,8 +1,10 @@ import { ChangeDetectorRef, Component, + ElementRef, HostBinding, - OnInit + OnInit, + ViewChild } from '@angular/core' import { EventService, @@ -11,17 +13,13 @@ import { SasService, SasStoreService } from '../services' -import * as XLSX from '@sheet/crypto' import { globals } from '../_globals' import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model' import { DcValidator } from '../shared/dc-validator/dc-validator' import { ExcelRule } from '../models/TableData' import { HotTableInterface } from '../models/HotTable.interface' import { Col } from '../shared/dc-validator/models/col.model' -import { - ParseResult, - SpreadsheetService -} from '../services/spreadsheet.service' +import { SpreadsheetService } from '../services/spreadsheet.service' import Handsontable from 'handsontable' import { HotTableRegisterer } from '@handsontable/angular' import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model' @@ -29,6 +27,9 @@ import { CellChange, ChangeSource } from 'handsontable/common' import { baseAfterGetColHeader } from '../shared/utils/hot.utils' import { ColumnSettings } from 'handsontable/settings' import { UploadFile } from '@sasjs/adapter' +import { UploadFileResponse } from '../models/UploadFile' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' +import { ParseResult } from '../models/ParseResult.interface' @Component({ selector: 'app-multi-dataset', @@ -37,6 +38,7 @@ import { UploadFile } from '@sasjs/adapter' }) export class MultiDatasetComponent implements OnInit { @HostBinding('class.content-container') contentContainerClass = true + @ViewChild('contentArea', { static: true }) contentAreaRef!: ElementRef public licenceState = this.licenceService.licenceState public Infinity = Infinity @@ -48,15 +50,19 @@ export class MultiDatasetComponent implements OnInit { public csvFiles: UploadFile[] = [] public csvSubmitting: boolean = false - public selectedFile: File | null = null + public autoDetectingColumns: boolean = false + + public selectedFile: SelectedFile | null = null public parsedDatasets: ParsedDataset[] = [] - public submittedDatasets: SubmittedDatasetResult[] = [] + public submittedCsvDatasets: SubmittedCsvDatasetResult[] = [] public datasetsLoading: boolean = false public uploadLoading: boolean = false public submitLoading: boolean = false public matchedDatasets: string[] = [] + public sheetNames: string[] = [] + public userInputDatasets: string = '' public libsAndTables: { @@ -194,6 +200,11 @@ export class MultiDatasetComponent implements OnInit { // For EXCEL if multiple files, we only take one (the first one) this.selectedFile = event.target.files[0] + if (this.selectedFile) + this.selectedFile.sizeMB = this.spreadsheetService.bytesToMB( + this.selectedFile.size + ) + this.initUserInputHot() this.onAutoDetectColumns() } else if (matchedExtension === 'csv') { @@ -246,7 +257,7 @@ export class MultiDatasetComponent implements OnInit { this.userInputDatasets = '' } - async onUploadFile() { + async onStartParsingFile() { this.uploadLoading = true const datasetFetchingPromises: Promise< @@ -271,18 +282,28 @@ export class MultiDatasetComponent implements OnInit { const datasetObjects = this.buildDatasetsObjects(datasets) - for (let datasetObject of datasetObjects) { + datasetObjects.forEach((datasetObject) => { + this.parsedDatasets.push({ + libds: datasetObject.libds, + includeInSubmission: true, + datasetInfo: datasetObject, + parsingTable: true + }) + }) + + for (let parsedDataset of this.parsedDatasets) { this.spreadsheetService .parseExcelFile({ file: this.selectedFile!, - dcValidator: datasetObject.dcValidator!, - headerPks: datasetObject.headerPks, - headerArray: datasetObject.headerArray, + password: this.selectedFile!.password || undefined, + dcValidator: parsedDataset.datasetInfo.dcValidator!, + headerPks: parsedDataset.datasetInfo.headerPks, + headerArray: parsedDataset.datasetInfo.headerArray, headerShow: [], - timeHeaders: datasetObject.timeHeaders, - dateHeaders: datasetObject.dateHeaders, - dateTimeHeaders: datasetObject.dateTimeHeaders, - xlRules: datasetObject.xlRules + timeHeaders: parsedDataset.datasetInfo.timeHeaders, + dateHeaders: parsedDataset.datasetInfo.dateHeaders, + dateTimeHeaders: parsedDataset.datasetInfo.dateTimeHeaders, + xlRules: parsedDataset.datasetInfo.xlRules }) .then((parseResult: ParseResult | undefined) => { console.log('parseResult', parseResult) @@ -304,23 +325,17 @@ export class MultiDatasetComponent implements OnInit { datasource.push(itemObject) }) - this.parsedDatasets.push({ - libds: datasetObject.libds, - parseResult: parseResult, - includeInSubmission: true, - datasetInfo: datasetObject, - datasource: datasource - }) + parsedDataset.datasource = datasource + parsedDataset.parseResult = parseResult + parsedDataset.parsingTable = false } }) .catch((error: string) => { console.warn('Parsing excel file error.', error) - this.parsedDatasets.push({ - libds: datasetObject.libds, - includeInSubmission: false, - datasetInfo: datasetObject - }) + parsedDataset.datasource = [] + parsedDataset.includeInSubmission = false + parsedDataset.parsingTable = false }) } }) @@ -349,13 +364,17 @@ export class MultiDatasetComponent implements OnInit { setTimeout(() => { this.hotInstance = this.hotRegisterer.getInstance('hotInstance') + // Set height of parsed data to full height of the page content area + const contentAreaHeight = this.contentAreaRef.nativeElement.clientHeight + const hotHeight = `${contentAreaHeight - 160}px` + if (this.activeParsedDataset) { this.hotInstance.updateSettings({ data: this.activeParsedDataset.datasource || [], colHeaders: this.activeParsedDataset.datasetInfo.headerColumns, columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(), readOnly: true, - height: '300px', + height: hotHeight || '300px', className: 'htDark' }) } @@ -507,13 +526,32 @@ export class MultiDatasetComponent implements OnInit { * convention. {@link isValidDatasetFormat} */ async onAutoDetectColumns() { - const sheetNames = await this.parseExcelSheetNames() + let passwordError = false - if (sheetNames) { + await this.parseExcelSheetNames() + .then((sheetNames) => { + this.sheetNames = sheetNames + }) + .catch((err) => { + if (err.includes('password')) { + passwordError = true + } + }) + + if (passwordError) { + this.onDiscardFile() + this.eventService.showInfoModal( + 'Locked file', + 'We failed to unlock the file.' + ) + return + } + + if (this.sheetNames) { this.matchedDatasets = [] this.userInputDatasets = '' - sheetNames.forEach((sheetName: string, index: number) => { + this.sheetNames.forEach((sheetName: string, index: number) => { const trimmedSheetname = sheetName.trim() if ( @@ -561,8 +599,8 @@ export class MultiDatasetComponent implements OnInit { this.initHot() } - onSubmittedDatasetClick(submittedDataset: SubmittedDatasetResult) { - this.deselectAllSubmittedDatasets() + onSubmittedCsvDatasetClick(submittedDataset: SubmittedCsvDatasetResult) { + this.deselectAllSubmittedCsvDatasets() submittedDataset.active = true } @@ -571,8 +609,10 @@ export class MultiDatasetComponent implements OnInit { return this.parsedDatasets.find((dataset) => dataset.active) } - public get activeSubmittedDataset(): SubmittedDatasetResult | undefined { - return this.submittedDatasets.find((dataset) => dataset.active) + public get activeSubmittedCsvDataset(): + | SubmittedCsvDatasetResult + | undefined { + return this.submittedCsvDatasets.find((dataset) => dataset.active) } public get notFoundDatasets(): string[] { @@ -606,8 +646,12 @@ export class MultiDatasetComponent implements OnInit { return this.csvFiles.length > 0 } + public get excelsSubmitted(): boolean { + return !!this.parsedDatasets.filter((ds) => ds.submitResult).length + } + public downloadFile(response: any) { - const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log` + const filename = `stagedata-${this.activeSubmittedCsvDataset?.libds}-log` this.helperService.downloadTextFile(filename, JSON.stringify(response)) } @@ -615,7 +659,7 @@ export class MultiDatasetComponent implements OnInit { * Submits attached CSVs which are matched with existing datasets */ async submitCsvFiles() { - let requestsResults: SubmittedDatasetResult[] = [] + let requestsResults: SubmittedCsvDatasetResult[] = [] for (let file of this.csvFiles) { const libds = this.parseDatasetFromCsvName(file.fileName) @@ -626,17 +670,17 @@ export class MultiDatasetComponent implements OnInit { await this.sasService .uploadFile('services/editors/loadfile', [file], { table: libds }) .then( - (res: any) => { - if (typeof res.sasjsAbort !== 'undefined') { - error = res.sasjsAbort + (res: UploadFileResponse) => { + if (typeof res.adapterResponse.sasjsAbort !== 'undefined') { + error = res.adapterResponse.sasjsAbort } else { - success = res + success = res.adapterResponse } }, (err: any) => { console.error('err', err) - error = err + error = err.adapterResponse } ) @@ -647,7 +691,7 @@ export class MultiDatasetComponent implements OnInit { }) } - this.submittedDatasets = requestsResults + this.submittedCsvDatasets = requestsResults } /** @@ -678,7 +722,7 @@ export class MultiDatasetComponent implements OnInit { } /** - * Sends tables to the SAS sequentially + * Sends tables found in an excel to the SAS sequentially * * @param explicitDatasets if empty all datasets will be sent, otherwise only datasets * in the array will be sent. eg. ['lib1.table_1', 'lib1.table_2'] @@ -688,10 +732,6 @@ export class MultiDatasetComponent implements OnInit { this.submitLoading = true - let requestsResults: SubmittedDatasetResult[] = explicitDatasets - ? this.submittedDatasets - : [] - for (let table of this.parsedDatasets) { // Skip the table if no data inside if (!table.parseResult || !table.datasource) continue @@ -733,6 +773,7 @@ export class MultiDatasetComponent implements OnInit { let error let success + let log await this.sasStoreService .updateTable( @@ -741,56 +782,47 @@ export class MultiDatasetComponent implements OnInit { 'SASControlTable', 'editors/stagedata', table.datasetInfo.data.$sasdata, - true + true, + { + debug: true + } ) - .then((res: EditorsStageDataSASResponse) => { - success = res + .then((res: RequestWrapperResponse) => { + success = res.adapterResponse + log = res.log }) .catch((err: any) => { console.error('err', err) - error = err + error = err.adapterResponse + log = err.log }) const requestResult = { success, error, + log, parseResult: table.parseResult, libds: table.libds } - // If explicit datasets are set don't just push to th array - // instead replace if result already exist from before (this might be re-submit) - if (explicitDatasets) { - const existingResultIndex = requestsResults.findIndex( - (result) => result.libds === table.libds - ) - - if (existingResultIndex > -1) { - requestsResults[existingResultIndex] = requestResult - } else { - requestsResults.push(requestResult) - } - } else { - requestsResults.push(requestResult) - } + table.submitResult = requestResult } } - - this.submittedDatasets = requestsResults this.showSubmitReasonModal = false this.submitLoading = false this.deselectAllParsedDatasets() } - async reSubmitTable(activeSubmittedDataset: SubmittedDatasetResult) { + async reSubmitTable(activeSubmittedDataset: ParsedDataset) { // Submit only particular table await this.submitTables([activeSubmittedDataset.libds]) // Activate new resubmitted table - const newSubmittedDataset = this.submittedDatasets.find( + const newSubmittedDataset = this.parsedDatasets.find( (sd) => sd.libds === activeSubmittedDataset.libds ) + if (newSubmittedDataset) newSubmittedDataset.active = true } @@ -815,40 +847,26 @@ export class MultiDatasetComponent implements OnInit { .map((row) => (row ? `${row[0]}.${row[1]}` : '')) } - private parseExcelSheetNames(): Promise { + /** + * Read the file minimally just to get the sheet names, not reading full file + * to help boost the performance + * + * @returns sheet names in string array + */ + private async parseExcelSheetNames(): Promise { return new Promise((resolve, reject) => { - const reader = new FileReader() + if (!this.selectedFile) return resolve([]) - if (!this.selectedFile) { - console.warn('selectedFile is missing') - return resolve([]) - } + this.spreadsheetService + .parseExcelSheetNames(this.selectedFile) + .then((parsed) => { + if (parsed.password) this.selectedFile!.password = parsed.password - reader.onload = (event: ProgressEvent) => { - if (!event?.target) { - console.warn('File reader event.target is missing') - return - } - - const data = event.target.result - const workbook = XLSX.read(data, { - type: 'binary' + return resolve(parsed.sheetNames) + }) + .catch((err) => { + return reject(err) }) - - try { - const sheet_names_list = workbook.SheetNames - - return resolve(sheet_names_list) - } catch (e) { - console.error(e) - } - } - - reader.onerror = function (ex) { - console.log(ex) - } - - reader.readAsBinaryString(this.selectedFile) }) } @@ -977,8 +995,8 @@ export class MultiDatasetComponent implements OnInit { } } - private deselectAllSubmittedDatasets() { - for (let submittedDataset of this.submittedDatasets) { + private deselectAllSubmittedCsvDatasets() { + for (let submittedDataset of this.submittedCsvDatasets) { submittedDataset.active = false } } @@ -1002,16 +1020,28 @@ export interface ParsedDataset { libds: string parseResult?: ParseResult datasetInfo: DatasetsObject + submitResult?: SubmittedDatasetResult datasource?: any[] includeInSubmission: boolean status?: 'success' | 'error' active?: boolean + parsingTable?: boolean } export interface SubmittedDatasetResult { + success: EditorsStageDataSASResponse | undefined + error: any + log?: string +} + +export interface SubmittedCsvDatasetResult { libds: string success: EditorsStageDataSASResponse | undefined error: any - parseResult?: ParseResult active?: boolean } + +export interface SelectedFile extends File { + sizeMB?: number + password?: string +} diff --git a/client/src/app/role/role.component.ts b/client/src/app/role/role.component.ts index cb66d56..6f8477a 100644 --- a/client/src/app/role/role.component.ts +++ b/client/src/app/role/role.component.ts @@ -4,6 +4,7 @@ import { HelperService } from '../services/helper.service' import { Location } from '@angular/common' import { Router, ActivatedRoute } from '@angular/router' import { SasService } from '../services/sas.service' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-role', @@ -47,61 +48,69 @@ export class RoleComponent implements OnInit { } else { if (globals.usernav.roleList === undefined) { this.loading = true - this.sasService.request('usernav/userroles', null).then((res: any) => { - this.loading = false - this.roles = res.roles - globals.usernav.roleList = res.roles - if (this.paramPresent) { - if (this.roles !== undefined) { - let validRole = this.findRole(this.roles, this.paramURI) - if (validRole !== false) { - this.loading = true - let data = { iwant: [{ roleid: this.paramURI }] } - this.sasService - .request('usernav/usermembersbyrole', data) - .then((res: any) => { - this.loading = false - this.roleMembers = res.sasmembers - this.roleMembersCount = res.sasmembers.length - this.roleGroups = res.sasgroups - this.roleGroupsCount = res.sasgroups.length - this.roleUri = validRole.ROLEURI - this.roleName = validRole.ROLENAME - this.roleDesc = validRole.ROLEDESC - }) + this.sasService + .request('usernav/userroles', null) + .then((res: RequestWrapperResponse) => { + this.loading = false + this.roles = res.adapterResponse.roles + globals.usernav.roleList = res.adapterResponse.roles + if (this.paramPresent) { + if (this.roles !== undefined) { + let validRole = this.findRole(this.roles, this.paramURI) + if (validRole !== false) { + this.loading = true + let data = { iwant: [{ roleid: this.paramURI }] } + this.sasService + .request('usernav/usermembersbyrole', data) + .then((res: RequestWrapperResponse) => { + this.loading = false + this.roleMembers = res.adapterResponse.sasmembers + this.roleMembersCount = + res.adapterResponse.sasmembers.length + this.roleGroups = res.adapterResponse.sasgroups + this.roleGroupsCount = + res.adapterResponse.sasgroups.length + this.roleUri = validRole.ROLEURI + this.roleName = validRole.ROLENAME + this.roleDesc = validRole.ROLEDESC + }) + } } } - } - }) + }) } else { this.roles = globals.usernav.roleList this.roleSearch = globals.usernav.roleSearch - this.sasService.request('usernav/userroles', null).then((res: any) => { - this.roles = res.roles - globals.usernav.roleList = res.roles + this.sasService + .request('usernav/userroles', null) + .then((res: RequestWrapperResponse) => { + this.roles = res.adapterResponse.roles + globals.usernav.roleList = res.adapterResponse.roles - if (this.paramPresent) { - if (this.roles !== undefined) { - let validRole = this.findRole(this.roles, this.paramURI) - if (validRole !== false) { - this.loading = true - let data = { iwant: [{ roleid: this.paramURI }] } - this.sasService - .request('usernav/usermembersbyrole', data) - .then((res: any) => { - this.loading = false - this.roleMembers = res.sasmembers - this.roleMembersCount = res.sasmembers.length - this.roleGroups = res.sasgroups - this.roleGroupsCount = res.sasgroups.length - this.roleUri = validRole.ROLEURI - this.roleName = validRole.ROLENAME - this.roleDesc = validRole.ROLEDESC - }) + if (this.paramPresent) { + if (this.roles !== undefined) { + let validRole = this.findRole(this.roles, this.paramURI) + if (validRole !== false) { + this.loading = true + let data = { iwant: [{ roleid: this.paramURI }] } + this.sasService + .request('usernav/usermembersbyrole', data) + .then((res: RequestWrapperResponse) => { + this.loading = false + this.roleMembers = res.adapterResponse.sasmembers + this.roleMembersCount = + res.adapterResponse.sasmembers.length + this.roleGroups = res.adapterResponse.sasgroups + this.roleGroupsCount = + res.adapterResponse.sasgroups.length + this.roleUri = validRole.ROLEURI + this.roleName = validRole.ROLENAME + this.roleDesc = validRole.ROLEDESC + }) + } } } - } - }) + }) } } } @@ -125,12 +134,12 @@ export class RoleComponent implements OnInit { let data = { iwant: [{ roleid: role.ROLEURI }] } this.sasService .request('usernav/usermembersbyrole', data) - .then((res: any) => { + .then((res: RequestWrapperResponse) => { this.loading = false - this.roleMembers = res.sasmembers - this.roleMembersCount = res.sasmembers.length - this.roleGroups = res.sasgroups - this.roleGroupsCount = res.sasgroups.length + this.roleMembers = res.adapterResponse.sasmembers + this.roleMembersCount = res.adapterResponse.sasmembers.length + this.roleGroups = res.adapterResponse.sasgroups + this.roleGroupsCount = res.adapterResponse.sasgroups.length this.roleUri = role.ROLEURI this.roleName = role.ROLENAME this.roleDesc = role.ROLEDESC diff --git a/client/src/app/services/app.service.ts b/client/src/app/services/app.service.ts index 9cb8687..330b9aa 100644 --- a/client/src/app/services/app.service.ts +++ b/client/src/app/services/app.service.ts @@ -9,11 +9,13 @@ import { EnvironmentInfo } from '../system/models/environment-info.model' import { LicenceService } from './licence.service' import { AppSettingsService } from './app-settings.service' import { AppThemes } from '../models/AppSettings' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' +import { AppStoreService } from './app-store.service' @Injectable() export class AppService { public syssite = new BehaviorSubject(null) - private environmentInfo: EnvironmentInfo | null = null + private environmentInfo: EnvironmentInfo = {} constructor( private licenceService: LicenceService, @@ -21,7 +23,8 @@ export class AppService { private sasService: SasService, private loggerService: LoggerService, private appSettingsService: AppSettingsService, - private router: Router + private router: Router, + private appStoreService: AppStoreService ) { this.subscribe() @@ -81,16 +84,19 @@ export class AppService { await this.sasService .request('public/startupservice', null) - .then(async (res: any) => { - this.syssite.next([res.SYSSITE]) + .then(async (res: RequestWrapperResponse) => { + this.syssite.next([res.adapterResponse.SYSSITE]) let missingProps: string[] = [] - if (!res.globvars || (res.globvars && !res.globvars[0])) + if ( + !res.adapterResponse.globvars || + (res.adapterResponse.globvars && !res.adapterResponse.globvars[0]) + ) missingProps.push('Globvars') - if (!res.sasdatasets) missingProps.push('Sasdatasets') - if (!res.saslibs) missingProps.push('Saslibs') - if (!res.xlmaps) missingProps.push('XLMaps') + if (!res.adapterResponse.sasdatasets) missingProps.push('Sasdatasets') + if (!res.adapterResponse.saslibs) missingProps.push('Saslibs') + if (!res.adapterResponse.xlmaps) missingProps.push('XLMaps') if (missingProps.length > 0) { startupServiceError = true @@ -103,23 +109,26 @@ export class AppService { return } + const dcAdapterSettings = this.appStoreService.getDcAdapterSettings() + this.environmentInfo = { - SYSSITE: res.SYSSITE, - SYSSCPL: res.SYSSCPL, - SYSTCPIPHOSTNAME: res.SYSTCPIPHOSTNAME, - SYSVLONG: res.SYSVLONG, - MEMSIZE: res.MEMSIZE, - SYSPROCESSMODE: res.SYSPROCESSMODE, - SYSHOSTNAME: res.SYSHOSTNAME, - SYSUSERID: res.SYSUSERID, - SYSHOSTINFOLONG: res.SYSHOSTINFOLONG, - SYSENCODING: res.SYSENCODING, - AUTOEXEC: res.AUTOEXEC, - ISADMIN: res.globvars[0].ISADMIN, - DC_ADMIN_GROUP: res.globvars[0].DC_ADMIN_GROUP + SYSSITE: res.adapterResponse.SYSSITE, + SYSSCPL: res.adapterResponse.SYSSCPL, + SYSTCPIPHOSTNAME: res.adapterResponse.SYSTCPIPHOSTNAME, + SYSVLONG: res.adapterResponse.SYSVLONG, + MEMSIZE: res.adapterResponse.MEMSIZE, + SYSPROCESSMODE: res.adapterResponse.SYSPROCESSMODE, + SYSHOSTNAME: res.adapterResponse.SYSHOSTNAME, + SYSUSERID: res.adapterResponse.SYSUSERID, + SYSHOSTINFOLONG: res.adapterResponse.SYSHOSTINFOLONG, + SYSENCODING: res.adapterResponse.SYSENCODING, + AUTOEXEC: res.adapterResponse.AUTOEXEC, + ISADMIN: res.adapterResponse.globvars[0].ISADMIN, + DC_ADMIN_GROUP: res.adapterResponse.globvars[0].DC_ADMIN_GROUP, + APP_LOC: dcAdapterSettings?.appLoc } - let libs = res.sasdatasets + let libs = res.adapterResponse.sasdatasets let libGroup: any = {} let libsAndTables let libraries @@ -153,7 +162,7 @@ export class AppService { globals.editor.libsAndTables = libsAndTables } - globals.xlmaps = res.xlmaps.map((xlmap: any) => ({ + globals.xlmaps = res.adapterResponse.xlmaps.map((xlmap: any) => ({ id: xlmap[0], description: xlmap[1], targetDS: xlmap[2] @@ -162,9 +171,9 @@ export class AppService { globals.editor.libraries = libraries globals.editor.startupSet = true - globals.dcLib = res.globvars[0].DCLIB + globals.dcLib = res.adapterResponse.globvars[0].DCLIB - await this.licenceService.activation(res) + await this.licenceService.activation(res.adapterResponse) }) .catch((err: any) => { startupServiceError = true diff --git a/client/src/app/services/sas-store.service.ts b/client/src/app/services/sas-store.service.ts index 54d314e..9209bbf 100644 --- a/client/src/app/services/sas-store.service.ts +++ b/client/src/app/services/sas-store.service.ts @@ -18,6 +18,7 @@ import { isSpecialMissing } from '@sasjs/utils/input/validators' import { Col } from '../shared/dc-validator/models/col.model' import { get } from 'lodash-es' import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Injectable() export class SasStoreService { @@ -58,12 +59,10 @@ export class SasStoreService { ) { const tables: any = {} tables[tableName] = [tableData] - const res: EditorsGetDataSASResponse = await this.sasService.request( - program, - tables - ) + const res: RequestWrapperResponse = + await this.sasService.request(program, tables) const response: EditorsGetDataServiceResponse = { - data: res, + data: res.adapterResponse, libds: libds } return response @@ -84,8 +83,9 @@ export class SasStoreService { tableName: string, program: string, $dataFormats: $DataFormats | null, - suppressErrorSuccessMessages?: boolean - ): Promise { + suppressErrorSuccessMessages?: boolean, + adapterConfig?: any + ): Promise> { // add sp as third argument of createData call let tables: any = { @@ -103,7 +103,7 @@ export class SasStoreService { let res = await this.sasService.request( program, tables, - null, + adapterConfig, { suppressErrorAbortModal: suppressErrorSuccessMessages, suppressSuccessAbortModal: suppressErrorSuccessMessages @@ -127,8 +127,7 @@ export class SasStoreService { ) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request(program, tables)).adapterResponse } /** @@ -152,8 +151,8 @@ export class SasStoreService { * @returns All submits */ public async getSubmitts() { - let res: any = await this.sasService.request('editors/getsubmits', null) - return res + return (await this.sasService.request('editors/getsubmits', null)) + .adapterResponse } /** @@ -161,7 +160,8 @@ export class SasStoreService { * @returns All libraries */ public async viewLibs() { - return this.sasService.request('public/viewlibs', null) + return (await this.sasService.request('public/viewlibs', null)) + .adapterResponse } public async refreshLibInfo(libref: string) { @@ -169,30 +169,22 @@ export class SasStoreService { lib2refresh: [{ libref }] } - return this.sasService.request('public/refreshlibinfo', data) - } - - public async versionHistory(libDataset: any) { - const data = { iwant: [{ LIBDS: libDataset }] } - let res: any = await this.sasService.request( - 'public/getversionhistory', - data - ) - return res + return (await this.sasService.request('public/refreshlibinfo', data)) + .adapterResponse } public async viewTables(lib: any) { let tables = { SASControlTable: [{ MPLIB: lib }] } - let res: any = await this.sasService.request('public/viewtables', tables) - return res + return (await this.sasService.request('public/viewtables', tables)) + .adapterResponse } public async viewData(libDataset: any, filter_pk: any) { let tables = { SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }] } - let res: any = await this.sasService.request('public/viewdata', tables) - return res + return (await this.sasService.request('public/viewdata', tables)) + .adapterResponse } public async viewDataSearch( @@ -213,41 +205,36 @@ export class SasStoreService { } ] } - let res: any = await this.sasService.request('public/viewdata', tables) - return res + + return (await this.sasService.request('public/viewdata', tables)) + .adapterResponse } public async getXLMapRules(id: string) { const tables = { getxlmaps_in: [{ XLMAP_ID: id }] } - const res: any = await this.sasService.request('editors/getxlmaps', tables) - return res - } - - public async getDetails(tableData: any, tableName: string, program: string) { - let tables: any = {} - tables[tableName] = [tableData] - - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request('editors/getxlmaps', tables)) + .adapterResponse } public async showDiffs(tableData: any, tableName: string, program: string) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables, { - useComputeApi: false - }) - return res + return ( + await this.sasService.request(program, tables, { + useComputeApi: false + }) + ).adapterResponse } public async rejecting(tableData: any, tableName: string, program: string) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables, { - useComputeApi: false - }) - return res + return ( + await this.sasService.request(program, tables, { + useComputeApi: false + }) + ).adapterResponse } public async approveTable( @@ -258,15 +245,13 @@ export class SasStoreService { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request(program, tables)).adapterResponse } public async getHistory(tableData: any, tableName: string, program: string) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request(program, tables)).adapterResponse } setQueryVariables(dataset: string, cols: any) { @@ -278,8 +263,8 @@ export class SasStoreService { public async getChangeInfo(tableId: any) { let obj = { TABLE: tableId } let table = { SASControlTable: [obj] } - let res: any = await this.sasService.request('public/getchangeinfo', table) - return res + return (await this.sasService.request('public/getchangeinfo', table)) + .adapterResponse } public async getQueryValues( @@ -304,13 +289,13 @@ export class SasStoreService { tables.FILTERQUERY = filterQuery } - let res: any = await this.sasService - .request('public/getcolvals', tables) - .catch((er: any) => { - throw er - }) - - return res + return ( + await this.sasService + .request('public/getcolvals', tables) + .catch((er: any) => { + throw er + }) + ).adapterResponse } public async saveQuery(libds: string, filterQuery: QueryClause[]) { @@ -319,21 +304,17 @@ export class SasStoreService { filterquery: filterQuery } - let res: any = await this.sasService.request( - 'public/validatefilter', - tables - ) + const res = await this.sasService.request('public/validatefilter', tables) + this.filter.next(res) - return res + + return res.adapterResponse } public async openTable(tableId: string) { let tables = { iwant: [{ table_id: tableId }] } - let res: any = await this.sasService.request( - 'auditors/getstagetable', - tables - ) - return res + return (await this.sasService.request('auditors/getstagetable', tables)) + .adapterResponse } public checkOperator(operator: any, value: any, type: string) { diff --git a/client/src/app/services/sas.service.ts b/client/src/app/services/sas.service.ts index e6c6950..2659daf 100644 --- a/client/src/app/services/sas.service.ts +++ b/client/src/app/services/sas.service.ts @@ -12,8 +12,10 @@ import { ServerType } from '@sasjs/utils/types/serverType' import { DcAdapterSettings } from '../models/DcAdapterSettings' import { AppStoreService } from './app-store.service' import { LoggerService } from './logger.service' -import { RequestWrapperOptions } from '../models/RequestWrapperOptions' +import { RequestWrapperOptions } from '../models/request-wrapper/RequestWrapperOptions' import { ErrorBody } from '../models/ErrorBody' +import { UploadFileResponse } from '../models/UploadFile' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Injectable({ providedIn: 'root' @@ -95,14 +97,16 @@ export class SasService { * @param config additional parameters to force eg. { debug: false } * @param wrapperOptions used to provide options to the request wrapper function * for example to suppress error or success abort modals after request is finished - * @returns + * @returns adapter response or an error. It will return the `log` as well. + * The log could be potentially be wrong if multiple requests happen because the log this + * function return is the last request in the Adapter Array for the given URL. */ public request( url: string, data: any, config?: any, wrapperOptions?: RequestWrapperOptions - ): Promise { + ): Promise> { url = 'services/' + url if (!wrapperOptions) wrapperOptions = {} @@ -117,9 +121,16 @@ export class SasService { }) .then( (res: any) => { + const sasRequest = this.sasjsAdapter + .getSasRequests() + .find((rq) => rq.serviceLink === url) + if (res.login === false) { this.shouldLogin.next(true) - reject(false) + reject({ + adapterResponse: false, + log: sasRequest?.logFile + }) } if (!this.userService.user && res.MF_GETUSER) { @@ -141,7 +152,12 @@ export class SasService { } if (res.status === 404) { - reject({ MESSAGE: res.body || 'SAS Responded with error' }) + reject({ + adapterResponse: { + MESSAGE: res.body || 'SAS Responded with error' + }, + log: sasRequest?.logFile + }) } if (typeof res.sasjsAbort !== 'undefined') { @@ -157,7 +173,12 @@ export class SasService { this.eventService.startupDataLoaded() this.router.navigateByUrl('/deploy') - reject({ error: abortMsg }) + reject({ + adapterResponse: { + error: abortMsg + }, + log: sasRequest?.logFile + }) return } @@ -174,14 +195,26 @@ export class SasService { ) } - reject({ error: abortMsg }) + reject({ + adapterResponse: { + error: abortMsg + }, + log: sasRequest?.logFile + }) } - resolve(res) + resolve({ + adapterResponse: res, + log: sasRequest?.logFile + }) }, (err: { error: ErrorBody | undefined }) => { console.error(err) + const sasRequest = this.sasjsAdapter + .getSasRequests() + .find((rq) => rq.serviceLink === url) + if (err.error) { let errorMessage: string | undefined = err.error.message let log: string | undefined @@ -205,10 +238,18 @@ export class SasService { 'Request error' ) } - reject({ error: errorMessage }) + reject({ + adapterResponse: { + error: errorMessage + }, + log: sasRequest?.logFile + }) } - reject(err) + reject({ + adapterResponse: err, + log: sasRequest?.logFile + }) } ) }) @@ -222,8 +263,35 @@ export class SasService { * @param params Aditional parameters eg. { debug: false } * @returns HTTP Response */ - public uploadFile(sasService: string, files: UploadFile[], params: any) { - return this.sasjsAdapter.uploadFile(sasService, files, params) + public uploadFile( + sasService: string, + files: UploadFile[], + params: any + ): Promise { + return new Promise((resolve, reject) => { + this.sasjsAdapter.uploadFile(sasService, files, params).then( + (res) => { + const sasRequest = this.sasjsAdapter + .getSasRequests() + .find((rq) => rq.serviceLink === 'services/editors/loadfile') + + resolve({ + adapterResponse: res, + log: sasRequest?.logFile + }) + }, + (err) => { + const sasRequest = this.sasjsAdapter + .getSasRequests() + .find((rq) => rq.serviceLink === 'services/editors/loadfile') + + reject({ + response: err, + log: sasRequest?.logFile + }) + } + ) + }) } public async login(username: string, password: string) { diff --git a/client/src/app/services/spreadsheet.service.ts b/client/src/app/services/spreadsheet.service.ts index 778882f..64f6740 100644 --- a/client/src/app/services/spreadsheet.service.ts +++ b/client/src/app/services/spreadsheet.service.ts @@ -1,70 +1,13 @@ import { Injectable } from '@angular/core' -import * as XLSX from '@sheet/crypto' -import { - ExcelPasswordModalService, - Result -} from '../shared/excel-password-modal/excel-password-modal.service' +import { ExcelPasswordModalService } from '../shared/excel-password-modal/excel-password-modal.service' import { EventService } from './event.service' -import { isSpecialMissing } from '@sasjs/utils/input/validators' -import { - dateFormat, - dateToUtcTime, - dateToTime -} from '../editor/utils/date.utils' -import { - excelDateToJSDate, - getMissingHeaders -} from '../editor/utils/grid.utils' -import { isStringNumber, isStringDecimal } from '../editor/utils/types.utils' -import SheetInfo from '../models/SheetInfo' -import { blobToFile } from '../xlmap/utils/file.utils' -import { ExcelRule } from '../models/TableData' -import { DcValidator } from '../shared/dc-validator/dc-validator' import { LicenceService } from './licence.service' -import { FileUploadEncoding } from '../models/FileUploadEncoding' -import { FileUploader } from '../models/FileUploader.class' - -/** - * Used in combination with buffer - */ -const iconv = require('iconv-lite') -/** - * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again - * Which will be send to backend - */ -const Buffer = require('buffer/').Buffer -type AOA = any[][] - -export interface ParseParams { - file: File - dcValidator: DcValidator - /** - * Parse function will manipulate and return the uploader array which can be provided with files already in the queue - * Otherwise new empty instance will be created. - */ - uploader?: FileUploader - headerPks: string[] - headerArray: string[] - headerShow: string[] - timeHeaders: string[] - dateHeaders: string[] - dateTimeHeaders: string[] - xlRules: ExcelRule[] - encoding?: FileUploadEncoding -} - -export interface ParseResult { - /** - * In case of CSV file, won't be returned - */ - data?: any[] - /** - * In case of CSV file, won't be returned - */ - headerShow?: string[] - rangeSheetRes?: SheetInfo - uploader: FileUploader -} +import { SpreadsheetUtil } from '../shared/spreadsheet-util/spreadsheet-util' +import { ParseParams } from '../models/ParseParams.interface' +import { ParseResult } from '../models/ParseResult.interface' +import { OpenOptions } from '../shared/excel-password-modal/models/options.interface' +import { Result } from '../shared/excel-password-modal/models/result.interface' +import * as XLSX from '@sheet/crypto' @Injectable({ providedIn: 'root' @@ -78,829 +21,139 @@ export class SpreadsheetService { private licenceService: LicenceService ) {} - /** - * Parses attached file and searches fo the matching data - * - * @param parseParams params required for parsing the file - * @param onParseStateChange callback used to inform about parsing state - * so the user of the function can update the UI with latest info - * @param onTableFoundEvent callback fired when table range is found in the file - * - * @returns parsed list of files to upload and JSON data ready for HOT usage - */ public parseExcelFile( parseParams: ParseParams, onParseStateChange?: (uploadState: string) => void, onTableFoundEvent?: (info: string) => void ): Promise { - return new Promise((resolve, reject) => { - let data: any[] = [] - const uploader: FileUploader = parseParams.uploader || new FileUploader() - - const file: File = parseParams.file - const filename = file.name - - if (!parseParams.encoding) parseParams.encoding = 'UTF-8' - - if (onParseStateChange) - onParseStateChange(`Loading ${filename} into the browser`) - - let foundData = { - sheet: '' - } - - let fileType = filename.slice( - filename.lastIndexOf('.') + 1, - filename.lastIndexOf('.') + 4 - ) - - if (fileType.toLowerCase() === 'xls') { - let reader: FileReader = new FileReader() - const self = this - reader.onload = async (theFile: any) => { - /* read workbook */ - const bstr = this.toBstr(theFile.target.result) - let wb: XLSX.WorkBook | undefined = undefined - let fileUnlocking: boolean = false - - const xlsxOptions: XLSX.ParsingOptions = { - type: 'binary', - cellDates: false, - cellFormula: true, - cellStyles: true, - cellNF: false, - cellText: false - } - - try { - wb = XLSX.read(bstr, { - ...xlsxOptions - }) - } catch (err: any) { - if (err.message.toLowerCase().includes('password')) { - fileUnlocking = true - - while (fileUnlocking) { - const password = await this.promptExcelPassword() - - if (password) { - try { - wb = XLSX.read(bstr, { - ...xlsxOptions, - password: password - }) - - fileUnlocking = false - } catch (err: any) { - this.excelPasswordModalService.open({ - error: true - }) - - if (!err.message.toLowerCase().includes('password')) { - fileUnlocking = false - } - } - } else { - fileUnlocking = false - } - } - } else { - return reject('Error reading the file') - } - } - - if (!wb) { - return reject('No workbook found.') - } - - /* save data */ - let isComplete: boolean = false - let missingHeaders: string[] = [] - - const csvArrayHeaders: string[] = [ - '_____DELETE__THIS__RECORD_____', - ...parseParams.headerArray - ] - let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) - let csvArrayHeadersMap = csvArrayHeadersLower.reduce( - (map: any, obj: string) => { - map[obj] = -1 - return map - }, - {} - ) - - let csvArrayData: any[] = [] - const rangeSheetRes: SheetInfo = this.getRangeAndSheet( - wb, - parseParams - ) - missingHeaders = rangeSheetRes.missingHeaders - - if (rangeSheetRes.foundData) { - isComplete = true - csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap - const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName] - - if (onParseStateChange) - onParseStateChange( - `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}` - ) - - let startAddress = '' - let endAddress = '' - - for ( - let row = rangeSheetRes.startRow; - row < rangeSheetRes.endRow; - ++row - ) { - const arr: any[] = [] - - csvArrayHeadersLower.forEach((x) => { - const col = csvArrayHeadersMap[x] - const addr = XLSX.utils.encode_cell({ - r: rangeSheetRes.rangeStartRow + row, - c: rangeSheetRes.rangeStartCol + col - }) - - let cell - - if (!ws[addr]) { - cell = { v: '' } - } else { - cell = ws[addr] - } - - if (startAddress === '' && ws[addr]) startAddress = addr - endAddress = addr - - arr.push(cell) - }) - - // If we found at least one non empty value it means it is not empty row - // othervise, it is empty row - let arrNonEmptyValue = arr.find((x) => x.v !== '') - - if (arrNonEmptyValue) csvArrayData.push(arr) - } - - rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}` - - if (onTableFoundEvent) - onTableFoundEvent( - `Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}` - ) - } else { - missingHeaders = rangeSheetRes.missingHeaders - } - - if (missingHeaders.length > 0) { - missingHeaders.sort(function compareSecondColumn(a, b) { - if (a[1] === b[1]) { - return 0 - } else { - return a[1] > b[1] ? -1 : 1 - } - }) - let abortMsg = missingHeaders - .map((x) => x[0]) - .slice(0, 5) - .join('\n') - - uploader.queue.pop() - return reject(abortMsg) - } - - // If first row is empty, that means no data has been found - if (csvArrayData.length === 0 || csvArrayData[0].length === 0) { - let abortMsg = 'No relevant data found in File !' - - uploader.queue.pop() - return reject(abortMsg) - } - - if ( - parseParams.dateTimeHeaders.length > 0 || - parseParams.dateHeaders.length > 0 || - parseParams.timeHeaders.length > 0 - ) { - csvArrayData = this.updateDateTimeCols( - csvArrayHeaders, - csvArrayData, - parseParams - ) - } - - if (parseParams.xlRules.length > 0) { - csvArrayData = this.updateXLRuleCols( - csvArrayHeaders, - csvArrayData, - parseParams - ) - } - - if (!isComplete) { - let abortMsg = '' - - if (missingHeaders.length === 0) { - abortMsg = 'No relevant data found in File !' - } else { - missingHeaders.sort(function compareSecondColumn(a, b) { - if (a[1] === b[1]) { - return 0 - } else { - return a[1] > b[1] ? -1 : 1 - } - }) - abortMsg = missingHeaders - .map((x) => x[0]) - .slice(0, 5) - .join('\n') - } - - // abort message is fired, return undefined - uploader.queue.pop() - return reject(abortMsg) - } else { - parseParams.headerShow = csvArrayHeaders - csvArrayData = csvArrayData.map((row: any) => - row.map((col: any) => (col.t === 'n' ? col.v : col.w)) - ) - - csvArrayData = csvArrayData.map((row: any) => { - return row.map((col: any, index: number) => { - if (!col && col !== 0) col = '' - - /** - * Keeping this for the reference - * Code below used to convert JSON to CSV - * now the XLSX is converting to CSV - */ - // if (isNaN(col)) { - // // Match and replace the double quotes, ignore the first and last char - // // in case they are double quotes already - // col = col.replace(/(? -1 || - // col.search(/\r|\n/g) > -1 - // ) { - // // Missing quotes at the end - // if (col.search(/"$/g) < 0) { - // col = col + '"' // So we add them - // } - - // // Missing quotes at the start - // if (col.search(/^"/g) < 0) { - // col = '"' + col // So we add them - // } - // } - // } - - const colName = parseParams.headerShow[index] - const colRule = parseParams.dcValidator?.getRule(colName) - - if (colRule?.type === 'numeric') { - if (isSpecialMissing(col) && !col.includes('.')) - col = '.' + col - } - - return col - }) - }) - - data = csvArrayData - - // Apply licence rows limitation if exists, it is only affecting data - // which will be send to SAS - const strippedCsvArrayData = csvArrayData.slice( - 0, - this.licenceState.value.submit_rows_limit - ) - // To submit to sas service, we need clean version of CSV of file - // attached. XLSX will do the parsing and heavy lifting - // First we create worksheet of json (data we extracted) - let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { - skipHeader: true - }) - // create CSV to be uploaded from worksheet - let csvContentClean = XLSX.utils.sheet_to_csv(ws) - // Prepend headers - csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean - - // Blob from which CSV file will be created depending of the selected - // encoding - let blob: Blob - - if (parseParams.encoding === 'WLATIN1') { - // WLATIN1 - let encoded = iconv.decode( - Buffer.from(csvContentClean), - 'CP-1252' - ) - blob = new Blob([encoded], { type: 'application/csv' }) - } else { - // UTF-8 - blob = new Blob([csvContentClean], { type: 'application/csv' }) - } - - let newCSVFile: File = blobToFile(blob, filename + '.csv') - uploader.addToQueue([newCSVFile]) - } - - if (data.length === 0) { - return reject( - `Table in the file is empty. Data found on sheet: ${foundData.sheet}` - ) - } - - return resolve({ - uploader, - data, - rangeSheetRes, - headerShow: parseParams.headerShow - }) - } - reader.readAsArrayBuffer(file) - } else if (fileType.toLowerCase() === 'csv') { - if (this.licenceState.value.submit_rows_limit !== Infinity) { - uploader.queue.pop() - return reject( - 'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io' - ) - } - - if (parseParams.encoding === 'WLATIN1') { - let reader = new FileReader() - const self = this - // Closure to capture the file information. - reader.onload = (theFile: any) => { - let encoded = iconv.decode( - Buffer.from(theFile.target.result), - 'CP-1252' - ) - let blob = new Blob([encoded], { type: fileType }) - let encodedFile: File = blobToFile(blob, filename) - uploader.queue.pop() - uploader.addToQueue([encodedFile]) - - resolve({ - uploader - }) - } - - reader.readAsArrayBuffer(file) - } else { - return resolve({ - uploader - }) - } - } else { - let abortMsg = - 'Invalid file type "' + - filename + - '". Please upload csv or excel file.' - - uploader.queue.pop() - return reject(abortMsg) - } + const spreadSheetUtil = new SpreadsheetUtil({ + licenceState: this.licenceState }) + + return spreadSheetUtil.parseExcelFile( + parseParams, + this.promptExcelPassword, + onParseStateChange, + onTableFoundEvent + ) } /** - * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers - * @param wb Excel workbook - * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range + * Read the file minimally just to get the sheet names, not reading full file + * to help boost the performance + * + * @returns sheet names in string array */ - private getRangeAndSheet( - wb: XLSX.WorkBook, - parseParams: ParseParams - ): SheetInfo { - let data = [] + public async parseExcelSheetNames(file: File): Promise<{ + sheetNames: string[] + password?: string + }> { + return new Promise((resolve, reject) => { + const reader = new FileReader() - let rangeStartRow: number = 0 - let rangeStartCol: number = 0 - let startRow: number = -1 - let endRow: number = -1 - let sheetName: string = '' - let isComplete = false - let missingHeaders: string[] = [] - const csvArrayHeaders: string[] = [ - '_____DELETE__THIS__RECORD_____', - ...parseParams.headerArray - ] - let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) - let csvArrayHeadersMap = csvArrayHeadersLower.reduce( - (map: any, obj: string) => { - map[obj] = -1 - return map - }, - {} - ) - - wb.SheetNames.forEach((element: string) => { - // Checking for required data in each sheet in workbook/ - if (isComplete) { - return + if (!file) { + console.warn('file is missing') + return resolve({ sheetNames: [] }) } - missingHeaders = [] - sheetName = element - const ws: XLSX.WorkSheet = wb.Sheets[sheetName] - - data = XLSX.utils.sheet_to_json(ws, { - header: 1, - blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong - defval: '' - }) - - if (data.length <= 1) { - return - } - - let tempArr: string[] = [] - parseParams.headerArray.forEach(() => tempArr.push('')) - data.push(tempArr) - - let foundHeaders = false - - data.forEach((row: any, index: number) => { - if (isComplete) { + reader.onload = async (event: ProgressEvent) => { + if (!event?.target) { + console.warn('File reader event.target is missing') return } - if (foundHeaders) { - let isDataEnd = true - let isPkNull = false + let wb: XLSX.WorkBook | undefined = undefined + let fileUnlocking: boolean = false + let password: string | undefined + const data = event.target.result - csvArrayHeadersLower.forEach((x) => { - const col = csvArrayHeadersMap[x] - - if (row[col] !== '' && row[col] !== undefined) { - isDataEnd = false - } else { - if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) { - isPkNull = true - } - } + try { + wb = XLSX.read(data, { + // Load file minimally to parse sheets + bookSheets: true, + type: 'binary' }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + fileUnlocking = true - if (isDataEnd || isPkNull) { - endRow = index - isComplete = true - } else { - if (startRow === -1) { - startRow = index - } - } - } else { - const rowLowerCase: string[] = row.map((x: any) => - x.toString().toLowerCase() - ) + let passwordError = false - // If in file there is no delete column, remove it from search of missing. - // This way delete column will be optional to provide in file - if (!rowLowerCase.includes('_____delete__this__record_____')) { - const deleteIndex = csvArrayHeadersLower.indexOf( - '_____delete__this__record_____' - ) - - if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1) - } - - foundHeaders = true - - csvArrayHeadersLower.forEach((x) => { - if (rowLowerCase.indexOf(x) === -1) { - foundHeaders = false - } - }) - - let result = [] - - result = this.findValidHeaders( - rowLowerCase, - csvArrayHeadersLower, - index, - sheetName, - parseParams - ) - - if (result[0] === false) { - foundHeaders = false - - if (result[1].length > 0) { - result[1].forEach((data: string) => { - missingHeaders.push(data) + while (fileUnlocking) { + password = await this.promptExcelPassword({ + error: passwordError }) - } - } else { - csvArrayHeadersMap = result[1] - } - } - }) - if (isComplete) { - this.update_sheet_range(ws) - const worksheetSel = ws['!ref'] + if (password) { + try { + wb = XLSX.read(data, { + // Load file minimally to parse sheets + bookSheets: true, + type: 'binary', + password: password + }) - if (worksheetSel) { - const range = XLSX.utils.decode_range(ws['!ref'] || '') - rangeStartRow = range.s.r - rangeStartCol = range.s.c - } - } - }) + fileUnlocking = false + passwordError = false + } catch (err: any) { + passwordError = true - // If start row is still -1 that means first row of found range is empty - if (startRow === -1) isComplete = false + if (!err.message.toLowerCase().includes('password')) { + fileUnlocking = false + } + } - const returnObj: SheetInfo = { - foundData: isComplete, - sheetName, - startRow, - endRow, - csvArrayHeadersMap, - missingHeaders, - rangeStartRow, - rangeStartCol - } - - return returnObj - } - - private findValidHeaders( - row: string[], - headers: string[], - rowNumber: number, - tabName: string, - parseParams: ParseParams - ): Array { - let headersFound = false - const missingErrorArray = [] - let j = 0 - - while (j < row.length) { - if (headersFound) { - // return; - } else { - if (headers.indexOf(row[j]) !== -1) { - let breakIndex - let rowStart = 0 - let rowEnd = 0 - let arrStart = 0 - let foundHeadersArray: string[] = [] - let spaceBreak = false - - for (let i = j; i < row.length; i++) { - if ( - row[i] === '' || - (foundHeadersArray.indexOf(row[i]) !== -1 && - this.isColHeader(row[i], parseParams.headerArray)) - ) { - if (row[i] === '') { - spaceBreak = true + if (!password) + return reject('Invalid password, failed to decrypt the file') + } else { + fileUnlocking = false + return reject('No password provided') } - - breakIndex = i - break - } else { - foundHeadersArray.push(row[i]) - } - } - - let tempArray: string[] = [] - - if (breakIndex !== undefined) { - tempArray = row.slice(j, breakIndex) - arrStart = j - rowEnd = breakIndex - - if (spaceBreak) { - rowStart = j - j = breakIndex - } else { - rowStart = j - j = breakIndex - 1 } } else { - tempArray = row.slice(j) - rowStart = j - arrStart = j - rowEnd = row.length - j = row.length - } - - let foundHeaders = true - - //We check if there are missing headers - headers.forEach((x) => { - if (tempArray.indexOf(x) === -1) { - foundHeaders = false - } - }) - - if (foundHeaders) { - headersFound = true - - let mapHeaders: any[] = headers - - let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) { - map[obj] = -1 - return map - }, {}) - - let temp = row.slice(rowStart, rowEnd) - - headers.forEach((x) => { - csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart - }) - - return [true, csvArrayHeadersMap] - } else { - let missingHeaders = getMissingHeaders(tempArray, headers) - - let missingMessage = 'TAB(' + tabName + ')' - missingErrorArray.push([ - missingMessage + - ' - ' + - missingHeaders[1].join(',') + - ' ( missing ' + - missingHeaders[0].join(',') + - ' )', - missingHeaders[1].length - ]) + return reject('Error reading the file') } } + + if (!wb) return reject('Error parsing the workbook') + + try { + const sheetNames = wb.SheetNames + + return resolve({ + sheetNames: sheetNames, + password: password + }) + } catch (e) { + console.error(e) + } } - j++ - } - return [false, missingErrorArray] + + reader.onerror = function (ex) { + console.log(ex) + } + + reader.readAsBinaryString(file) + }) } - private isColHeader(col: string, headerArray: string[]) { - return headerArray.indexOf(col.toUpperCase()) > -1 - } - - /** - * Function that updates the !ref range value provided in official docs. - * @param ws worksheet to be updated - */ - private update_sheet_range(ws: XLSX.WorkSheet) { - const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } } - - Object.keys(ws) - .filter(function (x) { - return x.charAt(0) != '!' - }) - .map(XLSX.utils.decode_cell) - .forEach(function (x: any) { - range.s.c = Math.min(range.s.c, x.c) - range.s.r = Math.min(range.s.r, x.r) - range.e.c = Math.max(range.e.c, x.c) - range.e.r = Math.max(range.e.r, x.r) - }) - - ws['!ref'] = XLSX.utils.encode_range(range) + public bytesToMB(size: number): number { + return parseFloat((size / (1024 * 1024)).toFixed(2)) } /** * When excel is password protected we will display the password prompt for user to type password in. * @returns Password user input or undefined if discarded by user */ - private promptExcelPassword(): Promise { + private promptExcelPassword = ( + options?: OpenOptions + ): Promise => { return new Promise((resolve, reject) => { - this.excelPasswordModalService.open().subscribe((result: Result) => { - resolve(result.password) - }) + this.excelPasswordModalService + .open(options) + .subscribe((result: Result) => { + resolve(result.password) + }) }) } - - private updateDateTimeCols( - headers: any, - data: any, - parseParams: ParseParams - ) { - if (parseParams.dateHeaders.length > 0) { - const dateCols: number[] = [] - parseParams.dateHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - dateCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - dateCols.forEach((element) => { - const obj = row[element] - if (isStringNumber(obj.v)) { - const date = excelDateToJSDate(Number(obj.v)) - - obj.v = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - } else { - if (obj && obj.v && obj.v.toString().indexOf(':') === -1) { - const date = new Date(obj.v) - if (date.toUTCString() !== 'Invalid Date') { - obj.v = dateFormat(date) - } - } - } - row[element] = obj - }) - }) - } - if (parseParams.timeHeaders.length > 0) { - let timeCols: number[] = [] - parseParams.timeHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - timeCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - timeCols.forEach((element) => { - const obj = row[element] - if ( - isStringNumber(obj.v) || - isStringDecimal(obj.v) || - obj.v.includes('E-') - ) { - const date = excelDateToJSDate(Number(obj.v)) - - obj.v = dateToUtcTime(date) - } - row[element] = obj - }) - }) - } - if (parseParams.dateTimeHeaders.length > 0) { - let dateTimeCols: number[] = [] - parseParams.dateTimeHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - dateTimeCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - dateTimeCols.forEach((element) => { - const obj = row[element] - if (isStringNumber(obj.v) || isStringDecimal(obj.v)) { - const date = excelDateToJSDate(Number(obj.v)) - obj.v = dateFormat(date) + ' ' + dateToUtcTime(date) - } else { - if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) { - let str = obj.v.substring(0, obj.v.indexOf(':')) - str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1) - obj.v = str - } - const date = new Date(obj.v) - if (date.toUTCString() !== 'Invalid Date') { - obj.v = dateFormat(date) + ' ' + dateToTime(date) - } - } - row[element] = obj - }) - }) - } - return data - } - - private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) { - if (parseParams.xlRules.length > 0) { - const xlRuleCols: any = [] - parseParams.xlRules.forEach((element: any) => { - if (headers.indexOf(element.XL_COLUMN) !== -1) { - element['index'] = headers.indexOf(element.XL_COLUMN) - xlRuleCols.push(element) - } - }) - data.forEach((row: any[]) => { - xlRuleCols.forEach((element: any) => { - const obj = row[element.index] - if (element.XL_RULE === 'FORMULA') { - if ('f' in obj) { - if (obj['t'] === 'n') { - obj['v'] = '=' + obj['f'] - } else { - obj['w'] = '=' + obj['f'] - } - } - } - row[element] = obj - }) - }) - } - return data - } - - private toBstr(res: any) { - let bytes = new Uint8Array(res) - let binary = '' - let length = bytes.byteLength - for (let i = 0; i < length; i++) { - binary += String.fromCharCode(bytes[i]) - } - return binary - } } diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.html b/client/src/app/shared/excel-password-modal/excel-password-modal.component.html index 28a48f6..9bdea64 100644 --- a/client/src/app/shared/excel-password-modal/excel-password-modal.component.html +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.html @@ -1,4 +1,4 @@ - +
Please enter password:

-

- Sorry that didn't work, try again. -

- - +
+

+ Sorry that didn't work, try again. +

+
+ +
+ + +
- +
diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss b/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss new file mode 100644 index 0000000..dddaa9f --- /dev/null +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss @@ -0,0 +1,18 @@ +.excel-password-root { + ::ng-deep { + .modal { + z-index: 1060; + } + } +} + +.modal-footer { + display: flex; + align-items: center; + justify-content: space-between; + + .buttons { + display: flex; + gap: 5px; + } +} \ No newline at end of file diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts b/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts index d78a5c8..b37912d 100644 --- a/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts @@ -1,24 +1,24 @@ import { Component } from '@angular/core' import { Observable } from 'rxjs' -import { - ExcelPasswordModalService, - Options -} from './excel-password-modal.service' +import { ExcelPasswordModalService } from './excel-password-modal.service' +import { Options } from './models/options.interface' @Component({ selector: 'app-excel-password-modal', + styleUrls: ['./excel-password-modal.component.scss'], templateUrl: './excel-password-modal.component.html' }) export class ExcelPasswordModalComponent { - options$: Observable + options$: Observable = this.excelPasswordModalService.optionsSubject$ fileUnlockError: boolean = false - constructor(private excelPasswordModalService: ExcelPasswordModalService) { - this.options$ = this.excelPasswordModalService.optionsSubject$ - } + passwordInput: string = '' + + constructor(private excelPasswordModalService: ExcelPasswordModalService) {} close(password?: string) { + this.passwordInput = '' this.excelPasswordModalService.close(password) } } diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts b/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts index 6f0e398..880123b 100644 --- a/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts @@ -1,17 +1,7 @@ import { Injectable } from '@angular/core' import { Subject, Observable } from 'rxjs' - -export interface Options extends OpenOptions { - open: boolean -} - -export interface Result { - password: string | undefined -} - -export interface OpenOptions { - error?: boolean -} +import { OpenOptions, Options } from './models/options.interface' +import { Result } from './models/result.interface' @Injectable({ providedIn: 'root' diff --git a/client/src/app/shared/excel-password-modal/models/options.interface.ts b/client/src/app/shared/excel-password-modal/models/options.interface.ts new file mode 100644 index 0000000..9502cbd --- /dev/null +++ b/client/src/app/shared/excel-password-modal/models/options.interface.ts @@ -0,0 +1,7 @@ +export interface OpenOptions { + error?: boolean +} + +export interface Options extends OpenOptions { + open: boolean +} diff --git a/client/src/app/shared/excel-password-modal/models/result.interface.ts b/client/src/app/shared/excel-password-modal/models/result.interface.ts new file mode 100644 index 0000000..3f0a324 --- /dev/null +++ b/client/src/app/shared/excel-password-modal/models/result.interface.ts @@ -0,0 +1,3 @@ +export interface Result { + password: string | undefined +} diff --git a/client/src/app/shared/sidebar/sidebar.component.scss b/client/src/app/shared/sidebar/sidebar.component.scss index 381c606..822f421 100644 --- a/client/src/app/shared/sidebar/sidebar.component.scss +++ b/client/src/app/shared/sidebar/sidebar.component.scss @@ -2,7 +2,6 @@ $sidebarWidth: 272px; .clr-vertical-nav .nav-link.active { background-color: transparent; - padding-left: 5px; } clr-vertical-nav { diff --git a/client/src/app/shared/spreadsheet-util/random.class.ts b/client/src/app/shared/spreadsheet-util/random.class.ts new file mode 100644 index 0000000..a18f48c --- /dev/null +++ b/client/src/app/shared/spreadsheet-util/random.class.ts @@ -0,0 +1,3 @@ +export class RandomClass { + constructor(jo: string) {} +} diff --git a/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts new file mode 100644 index 0000000..1b3168d --- /dev/null +++ b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts @@ -0,0 +1,954 @@ +import { isSpecialMissing } from '@sasjs/utils/input/validators' +import { + dateFormat, + dateToUtcTime, + dateToTime +} from 'src/app/editor/utils/date.utils' +import { + getMissingHeaders, + excelDateToJSDate +} from 'src/app/editor/utils/grid.utils' +import { + isStringNumber, + isStringDecimal +} from 'src/app/editor/utils/types.utils' +import { FileUploader } from 'src/app/models/FileUploader.class' +import SheetInfo from 'src/app/models/SheetInfo' +import { blobToFile } from 'src/app/xlmap/utils/file.utils' +import * as XLSX from '@sheet/crypto' +import { LicenceState } from 'src/app/models/LicenceState' +import { BehaviorSubject } from 'rxjs' +import { ParseParams } from 'src/app/models/ParseParams.interface' +import { ParseResult } from 'src/app/models/ParseResult.interface' +import { OpenOptions } from '../excel-password-modal/models/options.interface' + +/** + * Used in combination with buffer + */ +import * as iconv from 'iconv-lite' +/** + * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again + * Which will be send to backend + */ +import { Buffer } from 'buffer' +type AOA = any[][] + +export interface ConstructorParams { + licenceState: BehaviorSubject +} + +export class SpreadsheetUtil { + private licenceState: BehaviorSubject + + constructor(params: ConstructorParams) { + this.licenceState = params.licenceState + } + + /** + * Parses attached file and searches fo the matching data + * + * @param parseParams params required for parsing the file + * @param onParseStateChange callback used to inform about parsing state + * so the user of the function can update the UI with latest info + * @param onTableFoundEvent callback fired when table range is found in the file + * + * @returns parsed list of files to upload and JSON data ready for HOT usage + */ + public parseExcelFile( + parseParams: ParseParams, + promptExcelPassword: (options?: OpenOptions) => Promise, + onParseStateChange?: (uploadState: string) => void, + onTableFoundEvent?: (info: string) => void + ): Promise { + return new Promise((resolve, reject) => { + // If file size is bigger then 2 MB we need to load its bytes in chunks + const sizeInMB = this.bytesToMB(parseParams.file.size) + + if (sizeInMB > 2) { + } + + let data: any[] = [] + const uploader: FileUploader = parseParams.uploader || new FileUploader() + + const file: File = parseParams.file + const filename = file.name + + if (!parseParams.encoding) parseParams.encoding = 'UTF-8' + + if (onParseStateChange) + onParseStateChange(`Loading ${filename} into the browser`) + + let foundData = { + sheet: '' + } + + let fileType = filename.slice( + filename.lastIndexOf('.') + 1, + filename.lastIndexOf('.') + 4 + ) + + if (fileType.toLowerCase() === 'xls') { + let reader: FileReader = new FileReader() + + const self = this + reader.onload = async (theFile: any) => { + /* read workbook */ + const bstr = this.toBstr(theFile.target.result) + let wb: XLSX.WorkBook | undefined = undefined + let fileUnlocking: boolean = false + + const xlsxOptions: XLSX.ParsingOptions = { + type: 'binary', + cellDates: false, + cellFormula: true, + cellStyles: true, + cellNF: false, + cellText: false, + password: parseParams.password + } + + try { + wb = await this.xlsxRead(bstr, { + ...xlsxOptions + }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + fileUnlocking = true + + let passwordError = false + + while (fileUnlocking) { + const password = await promptExcelPassword({ + error: passwordError + }) + + if (password) { + try { + wb = await this.xlsxRead(bstr, { + ...xlsxOptions, + password: password + }) + + fileUnlocking = false + passwordError = false + } catch (err: any) { + passwordError = true + + if (!err.message.toLowerCase().includes('password')) { + fileUnlocking = false + } + } + } else { + fileUnlocking = false + } + } + } else { + return reject('Error reading the file') + } + } + + if (!wb) { + return reject('No workbook found.') + } + + /* save data */ + let isComplete: boolean = false + let missingHeaders: string[] = [] + + const csvArrayHeaders: string[] = [ + '_____DELETE__THIS__RECORD_____', + ...parseParams.headerArray + ] + + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) + let csvArrayHeadersMap = csvArrayHeadersLower.reduce( + (map: any, obj: string) => { + map[obj] = -1 + return map + }, + {} + ) + + let csvArrayData: any[] = [] + const rangeSheetRes: SheetInfo = this.getRangeAndSheet( + wb, + parseParams + ) + missingHeaders = rangeSheetRes.missingHeaders + + if (rangeSheetRes.foundData) { + isComplete = true + csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap + const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName] + + if (onParseStateChange) + onParseStateChange( + `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}` + ) + + let startAddress = '' + let endAddress = '' + + for ( + let row = rangeSheetRes.startRow; + row < rangeSheetRes.endRow; + ++row + ) { + const arr: any[] = [] + + csvArrayHeadersLower.forEach((x) => { + const col = csvArrayHeadersMap[x] + const addr = XLSX.utils.encode_cell({ + r: rangeSheetRes.rangeStartRow + row, + c: rangeSheetRes.rangeStartCol + col + }) + + let cell + + if (!ws[addr]) { + cell = { v: '' } + } else { + cell = ws[addr] + } + + if (startAddress === '' && ws[addr]) startAddress = addr + endAddress = addr + + arr.push(cell) + }) + + // If we found at least one non empty value it means it is not empty row + // othervise, it is empty row + let arrNonEmptyValue = arr.find((x) => x.v !== '') + + if (arrNonEmptyValue) csvArrayData.push(arr) + } + + rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}` + + if (onTableFoundEvent) + onTableFoundEvent( + `Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}` + ) + } else { + missingHeaders = rangeSheetRes.missingHeaders + } + + if (missingHeaders.length > 0) { + missingHeaders.sort(function compareSecondColumn(a, b) { + if (a[1] === b[1]) { + return 0 + } else { + return a[1] > b[1] ? -1 : 1 + } + }) + let abortMsg = missingHeaders + .map((x) => x[0]) + .slice(0, 5) + .join('\n') + + uploader.queue.pop() + return reject(abortMsg) + } + + // If first row is empty, that means no data has been found + if (csvArrayData.length === 0 || csvArrayData[0].length === 0) { + let abortMsg = 'No relevant data found in File !' + + uploader.queue.pop() + return reject(abortMsg) + } + + if ( + parseParams.dateTimeHeaders.length > 0 || + parseParams.dateHeaders.length > 0 || + parseParams.timeHeaders.length > 0 + ) { + csvArrayData = this.updateDateTimeCols( + csvArrayHeaders, + csvArrayData, + parseParams + ) + } + + if (parseParams.xlRules.length > 0) { + csvArrayData = this.updateXLRuleCols( + csvArrayHeaders, + csvArrayData, + parseParams + ) + } + + if (!isComplete) { + let abortMsg = '' + + if (missingHeaders.length === 0) { + abortMsg = 'No relevant data found in File !' + } else { + missingHeaders.sort(function compareSecondColumn(a, b) { + if (a[1] === b[1]) { + return 0 + } else { + return a[1] > b[1] ? -1 : 1 + } + }) + abortMsg = missingHeaders + .map((x) => x[0]) + .slice(0, 5) + .join('\n') + } + + // abort message is fired, return undefined + uploader.queue.pop() + return reject(abortMsg) + } else { + parseParams.headerShow = csvArrayHeaders + csvArrayData = csvArrayData.map((row: any) => + row.map((col: any) => (col.t === 'n' ? col.v : col.w)) + ) + + csvArrayData = csvArrayData.map((row: any) => { + return row.map((col: any, index: number) => { + if (!col && col !== 0) col = '' + + /** + * Keeping this for the reference + * Code below used to convert JSON to CSV + * now the XLSX is converting to CSV + */ + // if (isNaN(col)) { + // // Match and replace the double quotes, ignore the first and last char + // // in case they are double quotes already + // col = col.replace(/(? -1 || + // col.search(/\r|\n/g) > -1 + // ) { + // // Missing quotes at the end + // if (col.search(/"$/g) < 0) { + // col = col + '"' // So we add them + // } + + // // Missing quotes at the start + // if (col.search(/^"/g) < 0) { + // col = '"' + col // So we add them + // } + // } + // } + + const colName = parseParams.headerShow[index] + const colRule = parseParams.dcValidator?.getRule(colName) + + if (colRule?.type === 'numeric') { + if (isSpecialMissing(col) && !col.includes('.')) + col = '.' + col + } + + return col + }) + }) + + data = csvArrayData + + // Apply licence rows limitation if exists, it is only affecting data + // which will be send to SAS + const strippedCsvArrayData = csvArrayData.slice( + 0, + this.licenceState.value.submit_rows_limit + ) + // To submit to sas service, we need clean version of CSV of file + // attached. XLSX will do the parsing and heavy lifting + // First we create worksheet of json (data we extracted) + let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { + skipHeader: true + }) + + // create CSV to be uploaded from worksheet + let csvContentClean = XLSX.utils.sheet_to_csv(ws) + // Prepend headers + csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean + + // Blob from which CSV file will be created depending of the selected + // encoding + let blob: Blob + + if (parseParams.encoding === 'WLATIN1') { + // WLATIN1 + let encoded = iconv.decode( + Buffer.from(csvContentClean), + 'CP-1252' + ) + blob = new Blob([encoded], { type: 'application/csv' }) + } else { + // UTF-8 + blob = new Blob([csvContentClean], { type: 'application/csv' }) + } + + let newCSVFile: File = blobToFile(blob, filename + '.csv') + uploader.addToQueue([newCSVFile]) + } + + if (data.length === 0) { + return reject( + `Table in the file is empty. Data found on sheet: ${foundData.sheet}` + ) + } + + return resolve({ + uploader, + data, + rangeSheetRes, + headerShow: parseParams.headerShow + }) + } + + reader.readAsArrayBuffer(file) + } else if (fileType.toLowerCase() === 'csv') { + if (this.licenceState.value.submit_rows_limit !== Infinity) { + uploader.queue.pop() + return reject( + 'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io' + ) + } + + if (parseParams.encoding === 'WLATIN1') { + let reader = new FileReader() + const self = this + // Closure to capture the file information. + reader.onload = (theFile: any) => { + let encoded = iconv.decode( + Buffer.from(theFile.target.result), + 'CP-1252' + ) + let blob = new Blob([encoded], { type: fileType }) + let encodedFile: File = blobToFile(blob, filename) + uploader.queue.pop() + uploader.addToQueue([encodedFile]) + + resolve({ + uploader + }) + } + + reader.readAsArrayBuffer(file) + } else { + return resolve({ + uploader + }) + } + } else { + let abortMsg = + 'Invalid file type "' + + filename + + '". Please upload csv or excel file.' + + uploader.queue.pop() + return reject(abortMsg) + } + }) + } + + public bytesToMB(size: number): number { + return parseFloat((size / (1024 * 1024)).toFixed(2)) + } + + /** + * XLSX Read wrapper which uses Web Worker to read the file and not block + * the UI while reading. It will allow reading bigger files. + * If worker fails, fallback is regular file read. + * @param data + * @param opts + * @returns + */ + private xlsxRead( + data: any, + opts?: XLSX.ParsingOptions | undefined + ): Promise { + return new Promise((resolve, reject) => { + if (opts && opts.password) { + console.info('Not using worker to parse the XLSX - has password') + // At the moment worker can't use crypto version of SheetJS because of + // 'global not defined' issue + return resolve(XLSX.read(data, opts)) + } + + if (typeof Worker === 'undefined') { + console.info( + 'Not using worker to parse the XLSX - no Worker available in this environment' + ) + // Web workers are not supported in this environment. + // You should add a fallback so that your program still executes correctly. + return resolve(XLSX.read(data, opts)) + } + + // Ultimately use Web Worker to parse the excel + console.info('Using worker to parse the XLSX') + + const worker = new Worker( + new URL('../../spreadsheet.worker', import.meta.url) + ) + + worker.onmessage = ({ data }) => { + if (data.event === 'reading_end') { + resolve(data.workbook) + } else if (data.error) { + reject(data.error) + } else { + console.info( + 'Worker failed to parse the XLSX - fallback to non worker parsing' + ) + // Fallback to reading without Worker + resolve(XLSX.read(data, opts)) + } + } + + worker.postMessage({ + data, + opts + }) + + // Big timeout (10 minutes) in case Worker fails and no response + // and read the file with fallback method without worker + setTimeout(() => { + return resolve(XLSX.read(data, opts)) + }, 600 * 1000) // 10 minutes + }) + } + + /** + * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers + * @param wb Excel workbook + * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range + */ + private getRangeAndSheet( + wb: XLSX.WorkBook, + parseParams: ParseParams + ): SheetInfo { + let data = [] + + let rangeStartRow: number = 0 + let rangeStartCol: number = 0 + let startRow: number = -1 + let endRow: number = -1 + let sheetName: string = '' + let isComplete = false + let missingHeaders: string[] = [] + const csvArrayHeaders: string[] = [ + '_____DELETE__THIS__RECORD_____', + ...parseParams.headerArray + ] + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) + let csvArrayHeadersMap = csvArrayHeadersLower.reduce( + (map: any, obj: string) => { + map[obj] = -1 + return map + }, + {} + ) + + wb.SheetNames.forEach((element: string) => { + // Checking for required data in each sheet in workbook/ + if (isComplete) { + return + } + + missingHeaders = [] + sheetName = element + const ws: XLSX.WorkSheet = wb.Sheets[sheetName] + + data = XLSX.utils.sheet_to_json(ws, { + header: 1, + blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong + defval: '' + }) + + if (data.length <= 1) { + return + } + + let tempArr: string[] = [] + parseParams.headerArray.forEach(() => tempArr.push('')) + data.push(tempArr) + + let foundHeaders = false + + data.forEach((row: any, index: number) => { + if (isComplete) { + return + } + + if (foundHeaders) { + let isDataEnd = true + let isPkNull = false + + csvArrayHeadersLower.forEach((x) => { + const col = csvArrayHeadersMap[x] + + if (row[col] !== '' && row[col] !== undefined) { + isDataEnd = false + } else { + if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) { + isPkNull = true + } + } + }) + + if (isDataEnd || isPkNull) { + endRow = index + isComplete = true + } else { + if (startRow === -1) { + startRow = index + } + } + } else { + const rowLowerCase: string[] = row.map((x: any) => + x.toString().toLowerCase() + ) + + // If in file there is no delete column, remove it from search of missing. + // This way delete column will be optional to provide in file + if (!rowLowerCase.includes('_____delete__this__record_____')) { + const deleteIndex = csvArrayHeadersLower.indexOf( + '_____delete__this__record_____' + ) + + if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1) + } + + foundHeaders = true + + csvArrayHeadersLower.forEach((x) => { + if (rowLowerCase.indexOf(x) === -1) { + foundHeaders = false + } + }) + + let result = [] + + result = this.findValidHeaders( + rowLowerCase, + csvArrayHeadersLower, + index, + sheetName, + parseParams + ) + + if (result[0] === false) { + foundHeaders = false + + if (result[1].length > 0) { + result[1].forEach((data: string) => { + missingHeaders.push(data) + }) + } + } else { + csvArrayHeadersMap = result[1] + } + } + }) + + if (isComplete) { + this.update_sheet_range(ws) + const worksheetSel = ws['!ref'] + + if (worksheetSel) { + const range = XLSX.utils.decode_range(ws['!ref'] || '') + rangeStartRow = range.s.r + rangeStartCol = range.s.c + } + } + }) + + // If start row is still -1 that means first row of found range is empty + if (startRow === -1) isComplete = false + + const returnObj: SheetInfo = { + foundData: isComplete, + sheetName, + startRow, + endRow, + csvArrayHeadersMap, + missingHeaders, + rangeStartRow, + rangeStartCol + } + + return returnObj + } + + private findValidHeaders( + row: string[], + headers: string[], + rowNumber: number, + tabName: string, + parseParams: ParseParams + ): Array { + let headersFound = false + const missingErrorArray = [] + let j = 0 + + while (j < row.length) { + if (headersFound) { + // return; + } else { + if (headers.indexOf(row[j]) !== -1) { + let breakIndex + let rowStart = 0 + let rowEnd = 0 + let arrStart = 0 + let foundHeadersArray: string[] = [] + let spaceBreak = false + + for (let i = j; i < row.length; i++) { + if ( + row[i] === '' || + (foundHeadersArray.indexOf(row[i]) !== -1 && + this.isColHeader(row[i], parseParams.headerArray)) + ) { + if (row[i] === '') { + spaceBreak = true + } + + breakIndex = i + break + } else { + foundHeadersArray.push(row[i]) + } + } + + let tempArray: string[] = [] + + if (breakIndex !== undefined) { + tempArray = row.slice(j, breakIndex) + arrStart = j + rowEnd = breakIndex + + if (spaceBreak) { + rowStart = j + j = breakIndex + } else { + rowStart = j + j = breakIndex - 1 + } + } else { + tempArray = row.slice(j) + rowStart = j + arrStart = j + rowEnd = row.length + j = row.length + } + + let foundHeaders = true + + //We check if there are missing headers + headers.forEach((x) => { + if (tempArray.indexOf(x) === -1) { + foundHeaders = false + } + }) + + if (foundHeaders) { + headersFound = true + + let mapHeaders: any[] = headers + + let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) { + map[obj] = -1 + return map + }, {}) + + let temp = row.slice(rowStart, rowEnd) + + headers.forEach((x) => { + csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart + }) + + return [true, csvArrayHeadersMap] + } else { + let missingHeaders = getMissingHeaders(tempArray, headers) + + let missingMessage = 'TAB(' + tabName + ')' + missingErrorArray.push([ + missingMessage + + ' - ' + + missingHeaders[1].join(',') + + ' ( missing ' + + missingHeaders[0].join(',') + + ' )', + missingHeaders[1].length + ]) + } + } + } + j++ + } + return [false, missingErrorArray] + } + + private isColHeader(col: string, headerArray: string[]) { + return headerArray.indexOf(col.toUpperCase()) > -1 + } + + /** + * Function that updates the !ref range value provided in official docs. + * @param ws worksheet to be updated + */ + private update_sheet_range(ws: XLSX.WorkSheet) { + const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } } + + Object.keys(ws) + .filter(function (x) { + return x.charAt(0) != '!' + }) + .map(XLSX.utils.decode_cell) + .forEach(function (x: any) { + range.s.c = Math.min(range.s.c, x.c) + range.s.r = Math.min(range.s.r, x.r) + range.e.c = Math.max(range.e.c, x.c) + range.e.r = Math.max(range.e.r, x.r) + }) + + ws['!ref'] = XLSX.utils.encode_range(range) + } + + /** + * When excel is password protected we will display the password prompt for user to type password in. + * @returns Password user input or undefined if discarded by user + */ + // private promptExcelPassword(): Promise { + // return new Promise((resolve, reject) => { + // this.excelPasswordModalService.open().subscribe((result: Result) => { + // resolve(result.password) + // }) + // }) + // } + + private updateDateTimeCols( + headers: any, + data: any, + parseParams: ParseParams + ) { + if (parseParams.dateHeaders.length > 0) { + const dateCols: number[] = [] + parseParams.dateHeaders.forEach((element: any) => { + if (headers.indexOf(element) !== -1) { + dateCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + dateCols.forEach((element) => { + const obj = row[element] + if (isStringNumber(obj.v)) { + const date = excelDateToJSDate(Number(obj.v)) + + obj.v = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + } else { + if (obj && obj.v && obj.v.toString().indexOf(':') === -1) { + const date = new Date(obj.v) + if (date.toUTCString() !== 'Invalid Date') { + obj.v = dateFormat(date) + } + } + } + row[element] = obj + }) + }) + } + if (parseParams.timeHeaders.length > 0) { + let timeCols: number[] = [] + parseParams.timeHeaders.forEach((element: any) => { + if (headers.indexOf(element) !== -1) { + timeCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + timeCols.forEach((element) => { + const obj = row[element] + if ( + isStringNumber(obj.v) || + isStringDecimal(obj.v) || + obj.v.includes('E-') + ) { + const date = excelDateToJSDate(Number(obj.v)) + + obj.v = dateToUtcTime(date) + } + row[element] = obj + }) + }) + } + if (parseParams.dateTimeHeaders.length > 0) { + let dateTimeCols: number[] = [] + parseParams.dateTimeHeaders.forEach((element: any) => { + if (headers.indexOf(element) !== -1) { + dateTimeCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + dateTimeCols.forEach((element) => { + const obj = row[element] + if (isStringNumber(obj.v) || isStringDecimal(obj.v)) { + const date = excelDateToJSDate(Number(obj.v)) + obj.v = dateFormat(date) + ' ' + dateToUtcTime(date) + } else { + if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) { + let str = obj.v.substring(0, obj.v.indexOf(':')) + str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1) + obj.v = str + } + const date = new Date(obj.v) + if (date.toUTCString() !== 'Invalid Date') { + obj.v = dateFormat(date) + ' ' + dateToTime(date) + } + } + row[element] = obj + }) + }) + } + return data + } + + private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) { + if (parseParams.xlRules.length > 0) { + const xlRuleCols: any = [] + parseParams.xlRules.forEach((element: any) => { + if (headers.indexOf(element.XL_COLUMN) !== -1) { + element['index'] = headers.indexOf(element.XL_COLUMN) + xlRuleCols.push(element) + } + }) + data.forEach((row: any[]) => { + xlRuleCols.forEach((element: any) => { + const obj = row[element.index] + if (element.XL_RULE === 'FORMULA') { + if ('f' in obj) { + if (obj['t'] === 'n') { + obj['v'] = '=' + obj['f'] + } else { + obj['w'] = '=' + obj['f'] + } + } + } + row[element] = obj + }) + }) + } + return data + } + + private toBstr(res: any) { + let bytes = new Uint8Array(res) + let binary = '' + let length = bytes.byteLength + for (let i = 0; i < length; i++) { + binary += String.fromCharCode(bytes[i]) + } + return binary + } +} diff --git a/client/src/app/shared/terms/terms.component.ts b/client/src/app/shared/terms/terms.component.ts index d23519d..312f97c 100644 --- a/client/src/app/shared/terms/terms.component.ts +++ b/client/src/app/shared/terms/terms.component.ts @@ -8,6 +8,7 @@ import { import { SasService } from '../../services/sas.service' import * as marked from 'marked' import { EULA } from 'src/environments/_eula' +import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-terms', @@ -51,8 +52,12 @@ export class TermsComponent implements OnInit, AfterViewInit { this.sasService .request(`public/registeruser`, table) - .then((res: any) => { - if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') { + .then((res: RequestWrapperResponse) => { + if ( + res.adapterResponse.return && + res.adapterResponse.return[0] && + res.adapterResponse.return[0].MSG === 'SUCCESS' + ) { location.reload() } }) diff --git a/client/src/app/spreadsheet.worker.ts b/client/src/app/spreadsheet.worker.ts new file mode 100644 index 0000000..dfa7874 --- /dev/null +++ b/client/src/app/spreadsheet.worker.ts @@ -0,0 +1,30 @@ +/// + +/** + * We use normal version of the XLSX (SheetJS) + * Because at the moment "@sheet/crypto" can't work in the Web Worker environment + * Because of the missing "global" variable. + */ +import * as XLSX from 'xlsx' + +addEventListener('message', ({ data }) => { + const { data: xldata, opts: xlopts } = data as { + data: any + opts?: any + } + + try { + const workbook = XLSX.read(xldata, xlopts) + + postMessage({ + event: 'reading_end', + workbook: workbook + }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + postMessage({ + error: err + }) + } + } +}) diff --git a/client/src/app/stage/stage.component.ts b/client/src/app/stage/stage.component.ts index 76bfb70..39e123d 100644 --- a/client/src/app/stage/stage.component.ts +++ b/client/src/app/stage/stage.component.ts @@ -8,6 +8,7 @@ import { HotTableInterface } from '../models/HotTable.interface' import { LicenceService } from '../services/licence.service' import { globals } from '../_globals' import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-stage', @@ -180,10 +181,12 @@ export class StageComponent implements OnInit { this.sasService .request('editors/restore', data) - .then((res: EditorsRestoreServiceResponse) => { - if (res.restore_out) { + .then((res: RequestWrapperResponse) => { + if (res.adapterResponse.restore_out) { this.route.navigate([`/stage`]).then(() => { - this.route.navigate([`/stage/${res.restore_out[0].LOADREF}`]) + this.route.navigate([ + `/stage/${res.adapterResponse.restore_out[0].LOADREF}` + ]) }) } }) diff --git a/client/src/app/system/models/environment-info.model.ts b/client/src/app/system/models/environment-info.model.ts index 4dcae15..c550cf8 100644 --- a/client/src/app/system/models/environment-info.model.ts +++ b/client/src/app/system/models/environment-info.model.ts @@ -1,15 +1,16 @@ export interface EnvironmentInfo { - SYSSITE: string - SYSSCPL: string - SYSTCPIPHOSTNAME: string - SYSVLONG: string - MEMSIZE: string - SYSPROCESSMODE: string - SYSHOSTNAME: string - SYSUSERID: string - SYSHOSTINFOLONG: string - SYSENCODING: string - AUTOEXEC: string - ISADMIN: number - DC_ADMIN_GROUP: string + SYSSITE?: string + SYSSCPL?: string + SYSTCPIPHOSTNAME?: string + SYSVLONG?: string + MEMSIZE?: string + SYSPROCESSMODE?: string + SYSHOSTNAME?: string + SYSUSERID?: string + SYSHOSTINFOLONG?: string + SYSENCODING?: string + AUTOEXEC?: string + ISADMIN?: number + DC_ADMIN_GROUP?: string + APP_LOC?: string } diff --git a/client/src/app/system/system.component.html b/client/src/app/system/system.component.html index 33691af..2a8763b 100644 --- a/client/src/app/system/system.component.html +++ b/client/src/app/system/system.component.html @@ -77,6 +77,10 @@ DC Admin Group: {{ environmentInfo?.DC_ADMIN_GROUP }}

+

+ App Location: + {{ environmentInfo?.APP_LOC }} +

diff --git a/client/src/app/system/system.component.ts b/client/src/app/system/system.component.ts index 5f2f50d..b294782 100644 --- a/client/src/app/system/system.component.ts +++ b/client/src/app/system/system.component.ts @@ -9,6 +9,7 @@ import { AppInfo } from './models/app-info.model' import { EnvironmentInfo } from './models/environment-info.model' import { AppSettingsService } from '../services/app-settings.service' import { AppSettings } from '../models/AppSettings' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-system', @@ -51,7 +52,7 @@ export class SystemComponent implements OnInit { this.environmentInfo = this.appService.getEnvironmentInfo() this.settings = this.appSettingsService.settings.value - if (this.environmentInfo) { + if (this.environmentInfo.AUTOEXEC) { this.environmentInfo.AUTOEXEC = decodeURIComponent( this.environmentInfo.AUTOEXEC ) @@ -83,12 +84,12 @@ export class SystemComponent implements OnInit { this.sasService .request('admin/refreshcatalog', null) - .then((res: any) => { - this.response = this.parseResponse(res) + .then((res: RequestWrapperResponse) => { + this.response = this.parseResponse(res.adapterResponse) this.responseModal = true }) .catch((err: any) => { - this.response = this.parseResponse(err) + this.response = this.parseResponse(err.adapterResponse) this.responseModal = true }) .finally(() => { @@ -101,12 +102,12 @@ export class SystemComponent implements OnInit { this.sasService .request('admin/refreshtablelineage', null) - .then((res: any) => { - this.response = this.parseResponse(res) + .then((res: RequestWrapperResponse) => { + this.response = this.parseResponse(res.adapterResponse) this.responseModal = true }) .catch((err: any) => { - this.response = this.parseResponse(err) + this.response = this.parseResponse(err.adapterResponse) this.responseModal = true }) .finally(() => { diff --git a/client/src/app/user/user.component.ts b/client/src/app/user/user.component.ts index bc7ee21..136d142 100644 --- a/client/src/app/user/user.component.ts +++ b/client/src/app/user/user.component.ts @@ -6,6 +6,7 @@ import { Location } from '@angular/common' import { SasService } from '../services/sas.service' import { SASjsConfig } from '@sasjs/adapter' import { ServerType } from '@sasjs/utils/types/serverType' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-user', @@ -89,10 +90,10 @@ export class UserComponent implements OnInit { } else { this.sasService .request('usernav/usermembers', null) - .then((res: any) => { + .then((res: RequestWrapperResponse) => { this.loading = false - this.users = res.users - globals.usernav.userList = res.users + this.users = res.adapterResponse.users + globals.usernav.userList = res.adapterResponse.users }) } } else { @@ -141,18 +142,18 @@ export class UserComponent implements OnInit { this.sasService .request('usernav/usergroupsbymember', data) - .then((res: any) => { + .then((res: RequestWrapperResponse) => { this.loading = false switch (this.serverType) { case ServerType.Sas9: { - this.userInfo = res.info[0] - this.userEmails = res.emails - this.userEmailsCount = res.emails.length - this.userRoles = res.roles - this.userRolesCount = res.roles.length - this.userLogins = res.logins - this.userLoginsCount = res.logins.length + this.userInfo = res.adapterResponse.info[0] + this.userEmails = res.adapterResponse.emails + this.userEmailsCount = res.adapterResponse.emails.length + this.userRoles = res.adapterResponse.roles + this.userRolesCount = res.adapterResponse.roles.length + this.userLogins = res.adapterResponse.logins + this.userLoginsCount = res.adapterResponse.logins.length break } @@ -162,7 +163,7 @@ export class UserComponent implements OnInit { (userAr) => userAr.URI === uri ) } else { - const group = res.groups[0] + const group = res.adapterResponse.groups[0] this.userInfo = { URI: group.ID, @@ -175,9 +176,9 @@ export class UserComponent implements OnInit { } } - this.userData = res - this.userGroups = res.groups - this.userGroupsCount = res.groups.length + this.userData = res.adapterResponse + this.userGroups = res.adapterResponse.groups + this.userGroupsCount = res.adapterResponse.groups.length }) } } @@ -238,18 +239,18 @@ export class UserComponent implements OnInit { let data = { iwant: [{ uri: user.URI }] } this.sasService .request('usernav/usergroupsbymember', data) - .then((res: any) => { + .then((res: RequestWrapperResponse) => { this.loading = false switch (this.serverType) { case ServerType.Sas9: { - this.userInfo = res.info[0] - this.userEmails = res.emails - this.userEmailsCount = res.emails.length - this.userRoles = res.roles - this.userRolesCount = res.roles.length - this.userLogins = res.logins - this.userLoginsCount = res.logins.length + this.userInfo = res.adapterResponse.info[0] + this.userEmails = res.adapterResponse.emails + this.userEmailsCount = res.adapterResponse.emails.length + this.userRoles = res.adapterResponse.roles + this.userRolesCount = res.adapterResponse.roles.length + this.userLogins = res.adapterResponse.logins + this.userLoginsCount = res.adapterResponse.logins.length break } @@ -259,7 +260,7 @@ export class UserComponent implements OnInit { (userAr) => userAr.URI === user.URI ) } else { - const group = res.groups[0] + const group = res.adapterResponse.groups[0] this.userInfo = { URI: group.ID, @@ -272,9 +273,9 @@ export class UserComponent implements OnInit { } } - this.userData = res - this.userGroups = res.groups - this.userGroupsCount = res.groups.length + this.userData = res.adapterResponse + this.userGroups = res.adapterResponse.groups + this.userGroupsCount = res.adapterResponse.groups.length }) } } diff --git a/client/src/app/viewer/viewer.component.ts b/client/src/app/viewer/viewer.component.ts index 40116d8..10ba9fb 100644 --- a/client/src/app/viewer/viewer.component.ts +++ b/client/src/app/viewer/viewer.component.ts @@ -37,6 +37,7 @@ import { DataFormat } from '../models/sas/common/DateFormat' import { Libinfo } from '../models/sas/common/Libinfo' import { LicenceService } from '../services/licence.service' import { Location } from '@angular/common' +import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' @Component({ selector: 'app-viewer', diff --git a/client/src/app/xlmap/xlmap.component.ts b/client/src/app/xlmap/xlmap.component.ts index bee8bd0..0b84151 100644 --- a/client/src/app/xlmap/xlmap.component.ts +++ b/client/src/app/xlmap/xlmap.component.ts @@ -22,6 +22,7 @@ import { } from '../services' import { getCellAddress, getFinishingCell } from './utils/xl.utils' import { blobToFile, byteArrayToBinaryString } from './utils/file.utils' +import { UploadFileResponse } from '../models/UploadFile' interface XLMapRule { XLMAP_ID: string @@ -340,9 +341,9 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit { .uploadFile(uploadUrl, filesToUpload, { table: this.selectedXLMap.targetDS }) - .then((res: any) => { - if (res.sasjsAbort) { - const abortRes = res + .then((res: UploadFileResponse) => { + if (res.adapterResponse.sasjsAbort) { + const abortRes = res.adapterResponse const abortMsg = abortRes.sasjsAbort[0].MSG const macMsg = abortRes.sasjsAbort[0].MAC @@ -351,14 +352,14 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit { SYSERRORTEXT: abortRes.SYSERRORTEXT, MAC: macMsg }) - } else if (res.sasparams) { - const params = res.sasparams[0] + } else if (res.adapterResponse.sasparams) { + const params = res.adapterResponse.sasparams[0] const tableId = params.DSID this.router.navigateByUrl('/stage/' + tableId) } }) .catch((err: any) => { - this.eventService.catchResponseError('file upload', err) + this.eventService.catchResponseError('file upload', err.response) }) .finally(() => { this.status = Status.ReadyToSubmit diff --git a/client/tsconfig.worker.json b/client/tsconfig.worker.json new file mode 100644 index 0000000..22dc454 --- /dev/null +++ b/client/tsconfig.worker.json @@ -0,0 +1,15 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/worker", + "lib": [ + "es2018", + "webworker" + ], + "types": [] + }, + "include": [ + "src/**/*.worker.ts" + ] +}