From 4218da91cd193aa45346ad7e34ccc00ca89df4fb Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Fri, 5 Jul 2024 15:45:06 +0200 Subject: [PATCH] fix: implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height --- client/package-lock.json | 95 +++++++++++++++++++ client/package.json | 1 + .../multi-dataset.component.html | 2 +- .../multi-dataset/multi-dataset.component.ts | 11 ++- client/src/app/services/sas-store.service.ts | 76 +++++---------- .../spreadsheet-util/spreadsheet-util.ts | 67 +++++++++---- client/src/app/spreadsheet.worker.ts | 26 ++++- client/src/app/viewer/viewer.component.ts | 8 +- 8 files changed, 207 insertions(+), 79 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index d9815cb..acb8b16 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -52,6 +52,7 @@ "tslib": "^2.3.0", "vm": "^0.1.0", "webpack": "^5.91.0", + "xlsx": "^0.18.5", "zone.js": "~0.14.4" }, "devDependencies": { @@ -7122,6 +7123,14 @@ "node": ">=8.9.0" } }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/agent-base": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", @@ -8259,6 +8268,18 @@ "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", "dev": true }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -8579,6 +8600,14 @@ "integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==", "dev": true }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -8966,6 +8995,17 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/create-ecdh": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", @@ -11578,6 +11618,14 @@ "node": ">= 0.6" } }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -19340,6 +19388,17 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/sshpk": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", @@ -21323,6 +21382,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -21469,6 +21544,26 @@ } } }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/xmldoc": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz", diff --git a/client/package.json b/client/package.json index 4f6c77e..964c897 100644 --- a/client/package.json +++ b/client/package.json @@ -80,6 +80,7 @@ "tslib": "^2.3.0", "vm": "^0.1.0", "webpack": "^5.91.0", + "xlsx": "^0.18.5", "zone.js": "~0.14.4" }, "devDependencies": { diff --git a/client/src/app/multi-dataset/multi-dataset.component.html b/client/src/app/multi-dataset/multi-dataset.component.html index 9d3fd2b..182c2c4 100644 --- a/client/src/app/multi-dataset/multi-dataset.component.html +++ b/client/src/app/multi-dataset/multi-dataset.component.html @@ -114,7 +114,7 @@ --> -
+
{ this.hotInstance = this.hotRegisterer.getInstance('hotInstance') + // Set height of parsed data to full height of the page content area + const contentAreaHeight = this.contentAreaRef.nativeElement.clientHeight + const hotHeight = `${contentAreaHeight-160}px` + if (this.activeParsedDataset) { this.hotInstance.updateSettings({ data: this.activeParsedDataset.datasource || [], colHeaders: this.activeParsedDataset.datasetInfo.headerColumns, columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(), readOnly: true, - height: '300px', + height: hotHeight || '300px', className: 'htDark' }) } diff --git a/client/src/app/services/sas-store.service.ts b/client/src/app/services/sas-store.service.ts index afebe5e..73bde2e 100644 --- a/client/src/app/services/sas-store.service.ts +++ b/client/src/app/services/sas-store.service.ts @@ -127,8 +127,7 @@ export class SasStoreService { ) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request(program, tables)).adapterResponse } /** @@ -152,8 +151,7 @@ export class SasStoreService { * @returns All submits */ public async getSubmitts() { - let res: any = await this.sasService.request('editors/getsubmits', null) - return res + return (await this.sasService.request('editors/getsubmits', null)).adapterResponse } /** @@ -161,7 +159,7 @@ export class SasStoreService { * @returns All libraries */ public async viewLibs() { - return this.sasService.request('public/viewlibs', null) + return (await this.sasService.request('public/viewlibs', null)).adapterResponse } public async refreshLibInfo(libref: string) { @@ -169,30 +167,19 @@ export class SasStoreService { lib2refresh: [{ libref }] } - return this.sasService.request('public/refreshlibinfo', data) - } - - public async versionHistory(libDataset: any) { - const data = { iwant: [{ LIBDS: libDataset }] } - let res: any = await this.sasService.request( - 'public/getversionhistory', - data - ) - return res + return (await this.sasService.request('public/refreshlibinfo', data)).adapterResponse } public async viewTables(lib: any) { let tables = { SASControlTable: [{ MPLIB: lib }] } - let res: any = await this.sasService.request('public/viewtables', tables) - return res + return (await this.sasService.request('public/viewtables', tables)).adapterResponse } public async viewData(libDataset: any, filter_pk: any) { let tables = { SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }] } - let res: any = await this.sasService.request('public/viewdata', tables) - return res + return (await this.sasService.request('public/viewdata', tables)).adapterResponse } public async viewDataSearch( @@ -213,41 +200,30 @@ export class SasStoreService { } ] } - let res: any = await this.sasService.request('public/viewdata', tables) - return res + + return (await this.sasService.request('public/viewdata', tables)).adapterResponse } public async getXLMapRules(id: string) { const tables = { getxlmaps_in: [{ XLMAP_ID: id }] } - const res: any = await this.sasService.request('editors/getxlmaps', tables) - return res - } - - public async getDetails(tableData: any, tableName: string, program: string) { - let tables: any = {} - tables[tableName] = [tableData] - - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request('editors/getxlmaps', tables)).adapterResponse } public async showDiffs(tableData: any, tableName: string, program: string) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables, { + return (await this.sasService.request(program, tables, { useComputeApi: false - }) - return res + })).adapterResponse } public async rejecting(tableData: any, tableName: string, program: string) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables, { + return (await this.sasService.request(program, tables, { useComputeApi: false - }) - return res + })).adapterResponse } public async approveTable( @@ -258,15 +234,13 @@ export class SasStoreService { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request(program, tables)).adapterResponse } public async getHistory(tableData: any, tableName: string, program: string) { let tables: any = {} tables[tableName] = [tableData] - let res: any = await this.sasService.request(program, tables) - return res + return (await this.sasService.request(program, tables)).adapterResponse } setQueryVariables(dataset: string, cols: any) { @@ -278,8 +252,7 @@ export class SasStoreService { public async getChangeInfo(tableId: any) { let obj = { TABLE: tableId } let table = { SASControlTable: [obj] } - let res: any = await this.sasService.request('public/getchangeinfo', table) - return res + return (await this.sasService.request('public/getchangeinfo', table)).adapterResponse } public async getQueryValues( @@ -304,13 +277,11 @@ export class SasStoreService { tables.FILTERQUERY = filterQuery } - let res: any = await this.sasService + return (await this.sasService .request('public/getcolvals', tables) .catch((er: any) => { throw er - }) - - return res + })).adapterResponse } public async saveQuery(libds: string, filterQuery: QueryClause[]) { @@ -319,21 +290,22 @@ export class SasStoreService { filterquery: filterQuery } - let res: any = await this.sasService.request( + const res = await this.sasService.request( 'public/validatefilter', tables ) + this.filter.next(res) - return res + + return res.adapterResponse } public async openTable(tableId: string) { let tables = { iwant: [{ table_id: tableId }] } - let res: any = await this.sasService.request( + return (await this.sasService.request( 'auditors/getstagetable', tables - ) - return res + )).adapterResponse } public checkOperator(operator: any, value: any, type: string) { diff --git a/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts index 96aa44f..7b1a1ad 100644 --- a/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts +++ b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts @@ -108,10 +108,7 @@ export class SpreadsheetUtil { } try { - // wb = this.xlsxRead(bstr, { - // ...xlsxOptions - // }) - wb = XLSX.read(bstr, { + wb = await this.xlsxRead(bstr, { ...xlsxOptions }) } catch (err: any) { @@ -127,7 +124,7 @@ export class SpreadsheetUtil { if (password) { try { - wb = XLSX.read(bstr, { + wb = await this.xlsxRead(bstr, { ...xlsxOptions, password: password }) @@ -162,6 +159,7 @@ export class SpreadsheetUtil { '_____DELETE__THIS__RECORD_____', ...parseParams.headerArray ] + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) let csvArrayHeadersMap = csvArrayHeadersLower.reduce( (map: any, obj: string) => { @@ -364,6 +362,7 @@ export class SpreadsheetUtil { let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { skipHeader: true }) + // create CSV to be uploaded from worksheet let csvContentClean = XLSX.utils.sheet_to_csv(ws) // Prepend headers @@ -402,7 +401,7 @@ export class SpreadsheetUtil { headerShow: parseParams.headerShow }) } - console.log('before read file as array buffer') + reader.readAsArrayBuffer(file) } else if (fileType.toLowerCase() === 'csv') { if (this.licenceState.value.submit_rows_limit !== Infinity) { @@ -453,28 +452,64 @@ export class SpreadsheetUtil { return parseFloat((size / (1024 * 1024)).toFixed(2)) } + /** + * XLSX Read wrapper which uses Web Worker to read the file and not block + * the UI while reading. It will allow reading bigger files. + * If worker fails, fallback is regular file read. + * @param data + * @param opts + * @returns + */ private xlsxRead( data: any, opts?: XLSX.ParsingOptions | undefined - ): XLSX.WorkBook { - if (typeof Worker !== 'undefined') { - // Create a new + ): Promise { + return new Promise((resolve, reject) => { + if (opts && opts.password) { + console.info('Not using worker to parse the XLSX - has password') + // At the moment worker can't use crypto version of SheetJS because of + // 'global not defined' issue + return resolve(XLSX.read(data, opts)) + } + + if (typeof Worker === 'undefined') { + console.info('Not using worker to parse the XLSX - no Worker available in this environment') + // Web workers are not supported in this environment. + // You should add a fallback so that your program still executes correctly. + return resolve(XLSX.read(data, opts)) + } + + // Ultimately use Web Worker to parse the excel + console.info('Using worker to parse the XLSX') + const worker = new Worker( new URL('../../spreadsheet.worker', import.meta.url) ) - worker.onmessage = ({ data }) => {} + + worker.onmessage = ({ data }) => { + if (data.event === 'reading_end') { + resolve(data.workbook) + } else if (data.error) { + reject(data.error) + } + else { + console.info('Worker failed to parse the XLSX - fallback to non worker parsing') + // Fallback to reading without Worker + resolve(XLSX.read(data, opts)) + } + } worker.postMessage({ data, opts }) - return XLSX.read(data, opts) // Put in worker - } else { - // Web workers are not supported in this environment. - // You should add a fallback so that your program still executes correctly. - return XLSX.read(data, opts) - } + // Big timeout (10 minutes) in case Worker fails and no response + // and read the file with fallback method without worker + setTimeout(() => { + return resolve(XLSX.read(data, opts)) + }, 600 * 1000) // 10 minutes + }) } /** diff --git a/client/src/app/spreadsheet.worker.ts b/client/src/app/spreadsheet.worker.ts index 98b37b1..dfa7874 100644 --- a/client/src/app/spreadsheet.worker.ts +++ b/client/src/app/spreadsheet.worker.ts @@ -1,12 +1,30 @@ /// -import * as XLSX from '@sheet/crypto' +/** + * We use normal version of the XLSX (SheetJS) + * Because at the moment "@sheet/crypto" can't work in the Web Worker environment + * Because of the missing "global" variable. + */ +import * as XLSX from 'xlsx' addEventListener('message', ({ data }) => { - const input = data as { + const { data: xldata, opts: xlopts } = data as { data: any - opts?: XLSX.ParsingOptions | undefined + opts?: any } - console.log('input', input) + try { + const workbook = XLSX.read(xldata, xlopts) + + postMessage({ + event: 'reading_end', + workbook: workbook + }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + postMessage({ + error: err + }) + } + } }) diff --git a/client/src/app/viewer/viewer.component.ts b/client/src/app/viewer/viewer.component.ts index 0fc784b..6943e22 100644 --- a/client/src/app/viewer/viewer.component.ts +++ b/client/src/app/viewer/viewer.component.ts @@ -294,9 +294,9 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit { this.sasStoreService.refreshLibInfo(this.lib).then( async ( - res: RequestWrapperResponse + res: PublicRefreshlibinfoServiceResponse ) => { - this.libinfo = res.adapterResponse.libinfo + this.libinfo = res.libinfo globals.viewer.libinfo = this.libinfo const library = this.libraries.find((x) => x.LIBRARYREF === this.lib) @@ -1130,8 +1130,8 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit { await this.sasStoreService .viewLibs() - .then((res: RequestWrapperResponse) => { - this.libraries = res.adapterResponse.saslibs + .then((res: PublicViewlibsServiceResponse) => { + this.libraries = res.saslibs globals.viewer.libraries = this.libraries globals.viewer.startupSet = true