dc/dc
dc/dc
3
1
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases 35 Wiki Activity

fix: implemented the new request wrapper usage, added XLSX read with a Web Worker, multi load preview data, full height
Some checks failed
Build / Build-and-ng-test (pull_request) Failing after 43s
Details

Browse Source
This commit is contained in:
Mihajlo Medjedovic 2024-07-05 15:45:06 +02:00
parent 857b94f44f
commit 4218da91cd
8 changed files with 207 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
client/package-lock.json generated
View File

@ -52,6 +52,7 @@
"tslib": "^2.3.0",
"vm": "^0.1.0",
"webpack": "^5.91.0",
"xlsx": "^0.18.5",
"zone.js": "~0.14.4"
},
"devDependencies": {
@ -7122,6 +7123,14 @@
"node": ">=8.9.0"
}
},
"node_modules/adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/agent-base": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
@ -8259,6 +8268,18 @@
"integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
"dev": true
},
"node_modules/cfb": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
"integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
"dependencies": {
"adler-32": "~1.3.0",
"crc-32": "~1.2.0"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@ -8579,6 +8600,14 @@
"integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==",
"dev": true
},
"node_modules/codepage": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
"integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@ -8966,6 +8995,17 @@
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/crc-32": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
"integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
"bin": {
"crc32": "bin/crc32.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/create-ecdh": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
@ -11578,6 +11618,14 @@
"node": ">= 0.6"
}
},
"node_modules/frac": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/fraction.js": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz",
@ -19340,6 +19388,17 @@
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
"dev": true
},
"node_modules/ssf": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
"dependencies": {
"frac": "~1.1.2"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/sshpk": {
"version": "1.18.0",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@ -21323,6 +21382,22 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/wmf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word-wrap": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@ -21469,6 +21544,26 @@
}
}
},
"node_modules/xlsx": {
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"dependencies": {
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
},
"bin": {
"xlsx": "bin/xlsx.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/xmldoc": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz",

1
client/package.json
View File

@ -80,6 +80,7 @@
"tslib": "^2.3.0",
"vm": "^0.1.0",
"webpack": "^5.91.0",
"xlsx": "^0.18.5",
"zone.js": "~0.14.4"
},
"devDependencies": {

2
client/src/app/multi-dataset/multi-dataset.component.html
View File

@ -114,7 +114,7 @@
</div> -->
</app-sidebar>
<div class="content-area">
<div #contentArea class="content-area">
<div class="card no-borders h-100 d-flex clr-flex-column">
<div
class="header-row clr-row justify-content-between clr-justify-content-center w-100 m-0"

11
client/src/app/multi-dataset/multi-dataset.component.ts
View File

@ -1,8 +1,10 @@
import {
ChangeDetectorRef,
Component,
ElementRef,
HostBinding,
OnInit
OnInit,
ViewChild
} from '@angular/core'
import {
EventService,
@ -36,6 +38,7 @@ import { ParseResult } from '../models/ParseResult.interface'
})
export class MultiDatasetComponent implements OnInit {
@HostBinding('class.content-container') contentContainerClass = true
@ViewChild('contentArea', { static: true }) contentAreaRef!: ElementRef
public licenceState = this.licenceService.licenceState
public Infinity = Infinity
@ -361,13 +364,17 @@ export class MultiDatasetComponent implements OnInit {
setTimeout(() => {
this.hotInstance = this.hotRegisterer.getInstance('hotInstance')
// Set height of parsed data to full height of the page content area
const contentAreaHeight = this.contentAreaRef.nativeElement.clientHeight
const hotHeight = `${contentAreaHeight-160}px`
if (this.activeParsedDataset) {
this.hotInstance.updateSettings({
data: this.activeParsedDataset.datasource || [],
colHeaders: this.activeParsedDataset.datasetInfo.headerColumns,
columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(),
readOnly: true,
height: '300px',
height: hotHeight || '300px',
className: 'htDark'
})
}

76
client/src/app/services/sas-store.service.ts
View File

@ -127,8 +127,7 @@ export class SasStoreService {
) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request(program, tables)).adapterResponse
}
/**
@ -152,8 +151,7 @@ export class SasStoreService {
* @returns All submits
*/
public async getSubmitts() {
let res: any = await this.sasService.request('editors/getsubmits', null)
return res
return (await this.sasService.request('editors/getsubmits', null)).adapterResponse
}
/**
@ -161,7 +159,7 @@ export class SasStoreService {
* @returns All libraries
*/
public async viewLibs() {
return this.sasService.request('public/viewlibs', null)
return (await this.sasService.request('public/viewlibs', null)).adapterResponse
}
public async refreshLibInfo(libref: string) {
@ -169,30 +167,19 @@ export class SasStoreService {
lib2refresh: [{ libref }]
}
return this.sasService.request('public/refreshlibinfo', data)
}
public async versionHistory(libDataset: any) {
const data = { iwant: [{ LIBDS: libDataset }] }
let res: any = await this.sasService.request(
'public/getversionhistory',
data
)
return res
return (await this.sasService.request('public/refreshlibinfo', data)).adapterResponse
}
public async viewTables(lib: any) {
let tables = { SASControlTable: [{ MPLIB: lib }] }
let res: any = await this.sasService.request('public/viewtables', tables)
return res
return (await this.sasService.request('public/viewtables', tables)).adapterResponse
}
public async viewData(libDataset: any, filter_pk: any) {
let tables = {
SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }]
}
let res: any = await this.sasService.request('public/viewdata', tables)
return res
return (await this.sasService.request('public/viewdata', tables)).adapterResponse
}
public async viewDataSearch(
@ -213,41 +200,30 @@ export class SasStoreService {
}
]
}
let res: any = await this.sasService.request('public/viewdata', tables)
return res
return (await this.sasService.request('public/viewdata', tables)).adapterResponse
}
public async getXLMapRules(id: string) {
const tables = {
getxlmaps_in: [{ XLMAP_ID: id }]
}
const res: any = await this.sasService.request('editors/getxlmaps', tables)
return res
}
public async getDetails(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request('editors/getxlmaps', tables)).adapterResponse
}
public async showDiffs(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables, {
return (await this.sasService.request(program, tables, {
useComputeApi: false
})
return res
})).adapterResponse
}
public async rejecting(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables, {
return (await this.sasService.request(program, tables, {
useComputeApi: false
})
return res
})).adapterResponse
}
public async approveTable(
@ -258,15 +234,13 @@ export class SasStoreService {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request(program, tables)).adapterResponse
}
public async getHistory(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request(program, tables)).adapterResponse
}
setQueryVariables(dataset: string, cols: any) {
@ -278,8 +252,7 @@ export class SasStoreService {
public async getChangeInfo(tableId: any) {
let obj = { TABLE: tableId }
let table = { SASControlTable: [obj] }
let res: any = await this.sasService.request('public/getchangeinfo', table)
return res
return (await this.sasService.request('public/getchangeinfo', table)).adapterResponse
}
public async getQueryValues(
@ -304,13 +277,11 @@ export class SasStoreService {
tables.FILTERQUERY = filterQuery
}
let res: any = await this.sasService
return (await this.sasService
.request('public/getcolvals', tables)
.catch((er: any) => {
throw er
})
return res
})).adapterResponse
}
public async saveQuery(libds: string, filterQuery: QueryClause[]) {
@ -319,21 +290,22 @@ export class SasStoreService {
filterquery: filterQuery
}
let res: any = await this.sasService.request(
const res = await this.sasService.request(
'public/validatefilter',
tables
)
this.filter.next(res)
return res
return res.adapterResponse
}
public async openTable(tableId: string) {
let tables = { iwant: [{ table_id: tableId }] }
let res: any = await this.sasService.request(
return (await this.sasService.request(
'auditors/getstagetable',
tables
)
return res
)).adapterResponse
}
public checkOperator(operator: any, value: any, type: string) {

67
client/src/app/shared/spreadsheet-util/spreadsheet-util.ts
View File

@ -108,10 +108,7 @@ export class SpreadsheetUtil {
}
try {
// wb = this.xlsxRead(bstr, {
// ...xlsxOptions
// })
wb = XLSX.read(bstr, {
wb = await this.xlsxRead(bstr, {
...xlsxOptions
})
} catch (err: any) {
@ -127,7 +124,7 @@ export class SpreadsheetUtil {
if (password) {
try {
wb = XLSX.read(bstr, {
wb = await this.xlsxRead(bstr, {
...xlsxOptions,
password: password
})
@ -162,6 +159,7 @@ export class SpreadsheetUtil {
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
@ -364,6 +362,7 @@ export class SpreadsheetUtil {
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
skipHeader: true
})
// create CSV to be uploaded from worksheet
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
// Prepend headers
@ -402,7 +401,7 @@ export class SpreadsheetUtil {
headerShow: parseParams.headerShow
})
}
console.log('before read file as array buffer')
reader.readAsArrayBuffer(file)
} else if (fileType.toLowerCase() === 'csv') {
if (this.licenceState.value.submit_rows_limit !== Infinity) {
@ -453,28 +452,64 @@ export class SpreadsheetUtil {
return parseFloat((size / (1024 * 1024)).toFixed(2))
}
/**
* XLSX Read wrapper which uses Web Worker to read the file and not block
* the UI while reading. It will allow reading bigger files.
* If worker fails, fallback is regular file read.
* @param data
* @param opts
* @returns
*/
private xlsxRead(
data: any,
opts?: XLSX.ParsingOptions | undefined
): XLSX.WorkBook {
if (typeof Worker !== 'undefined') {
// Create a new
): Promise<XLSX.WorkBook> {
return new Promise((resolve, reject) => {
if (opts && opts.password) {
console.info('Not using worker to parse the XLSX - has password')
// At the moment worker can't use crypto version of SheetJS because of
// 'global not defined' issue
return resolve(XLSX.read(data, opts))
}
if (typeof Worker === 'undefined') {
console.info('Not using worker to parse the XLSX - no Worker available in this environment')
// Web workers are not supported in this environment.
// You should add a fallback so that your program still executes correctly.
return resolve(XLSX.read(data, opts))
}
// Ultimately use Web Worker to parse the excel
console.info('Using worker to parse the XLSX')
const worker = new Worker(
new URL('../../spreadsheet.worker', import.meta.url)
)
worker.onmessage = ({ data }) => {}
worker.onmessage = ({ data }) => {
if (data.event === 'reading_end') {
resolve(data.workbook)
} else if (data.error) {
reject(data.error)
}
else {
console.info('Worker failed to parse the XLSX - fallback to non worker parsing')
// Fallback to reading without Worker
resolve(XLSX.read(data, opts))
}
}
worker.postMessage({
data,
opts
})
return XLSX.read(data, opts) // Put in worker
} else {
// Web workers are not supported in this environment.
// You should add a fallback so that your program still executes correctly.
return XLSX.read(data, opts)
}
// Big timeout (10 minutes) in case Worker fails and no response
// and read the file with fallback method without worker
setTimeout(() => {
return resolve(XLSX.read(data, opts))
}, 600 * 1000) // 10 minutes
})
}
/**

26
client/src/app/spreadsheet.worker.ts
View File

@ -1,12 +1,30 @@
/// <reference lib="webworker" />
import * as XLSX from '@sheet/crypto'
/**
* We use normal version of the XLSX (SheetJS)
* Because at the moment "@sheet/crypto" can't work in the Web Worker environment
* Because of the missing "global" variable.
*/
import * as XLSX from 'xlsx'
addEventListener('message', ({ data }) => {
const input = data as {
const { data: xldata, opts: xlopts } = data as {
data: any
opts?: XLSX.ParsingOptions | undefined
opts?: any
}
console.log('input', input)
try {
const workbook = XLSX.read(xldata, xlopts)
postMessage({
event: 'reading_end',
workbook: workbook
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
postMessage({
error: err
})
}
}
})

8
client/src/app/viewer/viewer.component.ts
View File

@ -294,9 +294,9 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
this.sasStoreService.refreshLibInfo(this.lib).then(
async (
res: RequestWrapperResponse<PublicRefreshlibinfoServiceResponse>
res: PublicRefreshlibinfoServiceResponse
) => {
this.libinfo = res.adapterResponse.libinfo
this.libinfo = res.libinfo
globals.viewer.libinfo = this.libinfo
const library = this.libraries.find((x) => x.LIBRARYREF === this.lib)
@ -1130,8 +1130,8 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
await this.sasStoreService
.viewLibs()
.then((res: RequestWrapperResponse<PublicViewlibsServiceResponse>) => {
this.libraries = res.adapterResponse.saslibs
.then((res: PublicViewlibsServiceResponse) => {
this.libraries = res.saslibs
globals.viewer.libraries = this.libraries
globals.viewer.startupSet = true