Merge pull request 'XLSX Performance Improvements (multi load)' (#118) from multi-load-performance into main
Some checks failed
Release / Build-production-and-ng-test (push) Successful in 3m55s
Release / Build-and-test-development (push) Failing after 8m2s
Release / release (push) Has been skipped

Reviewed-on: #118
This commit is contained in:
mihajlo 2024-07-26 11:34:20 +00:00
commit 403d08c86a
42 changed files with 2081 additions and 1415 deletions

View File

@ -69,7 +69,8 @@
],
"scripts": [
"node_modules/marked/marked.min.js"
]
],
"webWorkerTsConfig": "tsconfig.worker.json"
},
"configurations": {
"production": {
@ -148,7 +149,8 @@
"src/styles.scss"
],
"scripts": [],
"karmaConfig": "karma.conf.js"
"karmaConfig": "karma.conf.js",
"webWorkerTsConfig": "tsconfig.worker.json"
}
},
"lint": {

View File

@ -52,6 +52,7 @@
"tslib": "^2.3.0",
"vm": "^0.1.0",
"webpack": "^5.91.0",
"xlsx": "^0.18.5",
"zone.js": "~0.14.4"
},
"devDependencies": {
@ -7122,6 +7123,14 @@
"node": ">=8.9.0"
}
},
"node_modules/adler-32": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
"integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/agent-base": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
@ -8259,6 +8268,18 @@
"integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==",
"dev": true
},
"node_modules/cfb": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
"integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
"dependencies": {
"adler-32": "~1.3.0",
"crc-32": "~1.2.0"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@ -8579,6 +8600,14 @@
"integrity": "sha512-c5or4P6erEA69TxaxTNcHUNcIn+oyxSRTOWV+pSYF+z4epXqNvwvJ70XPGjPNgue83oAFAPBRQYwpAJ/Hpe/Sg==",
"dev": true
},
"node_modules/codepage": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
"integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
@ -8966,6 +8995,17 @@
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/crc-32": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
"integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
"bin": {
"crc32": "bin/crc32.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/create-ecdh": {
"version": "4.0.4",
"resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
@ -11578,6 +11618,14 @@
"node": ">= 0.6"
}
},
"node_modules/frac": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
"integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/fraction.js": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz",
@ -19340,6 +19388,17 @@
"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
"dev": true
},
"node_modules/ssf": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
"integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
"dependencies": {
"frac": "~1.1.2"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/sshpk": {
"version": "1.18.0",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz",
@ -21323,6 +21382,22 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/wmf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
"integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
"integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/word-wrap": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
@ -21469,6 +21544,26 @@
}
}
},
"node_modules/xlsx": {
"version": "0.18.5",
"resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
"integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
"dependencies": {
"adler-32": "~1.3.0",
"cfb": "~1.2.1",
"codepage": "~1.15.0",
"crc-32": "~1.2.1",
"ssf": "~0.11.2",
"wmf": "~1.0.1",
"word": "~0.3.0"
},
"bin": {
"xlsx": "bin/xlsx.njs"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/xmldoc": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/xmldoc/-/xmldoc-1.3.0.tgz",

View File

@ -80,6 +80,7 @@
"tslib": "^2.3.0",
"vm": "^0.1.0",
"webpack": "^5.91.0",
"xlsx": "^0.18.5",
"zone.js": "~0.14.4"
},
"devDependencies": {

View File

@ -1,6 +1,7 @@
import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core'
import SASjs, { SASjsConfig } from '@sasjs/adapter'
import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
import { DeployService } from 'src/app/services/deploy.service'
import { EventService } from 'src/app/services/event.service'
import { LoggerService } from 'src/app/services/logger.service'
@ -303,10 +304,10 @@ export class ManualComponent implements OnInit {
this.sasService
.request('public/startupservice', null)
.then((res: any) => {
this.loggerService.log(res)
.then((res: RequestWrapperResponse) => {
this.loggerService.log(res.adapterResponse)
if (res.saslibs) {
if (res.adapterResponse.saslibs) {
this.validationState = 'success'
} else {
this.validationState = 'error'

View File

@ -3,6 +3,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'
import SASjs, { SASjsConfig } from '@sasjs/adapter'
import { ServerType } from '@sasjs/utils/types/serverType'
import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
import { SASGroup } from 'src/app/models/sas/public-getgroups.model'
import { SASjsApiServerInfo } from 'src/app/models/sasjs-api/SASjsApiServerInfo.model'
import { SasService } from 'src/app/services/sas.service'
@ -68,11 +69,11 @@ export class SasjsConfiguratorComponent implements OnInit {
this.loading = true
this.sasService.request('usernav/usergroupsbymember', null).then(
(res: any) => {
this.METAPERSON = res.MF_GETUSER
this.SYSUSERID = res.SYSUSERID
this.SYSHOSTNAME = res.SYSHOSTNAME
this.SYSVLONG = res.SYSVLONG
(res: RequestWrapperResponse) => {
this.METAPERSON = res.adapterResponse.MF_GETUSER
this.SYSUSERID = res.adapterResponse.SYSUSERID
this.SYSHOSTNAME = res.adapterResponse.SYSHOSTNAME
this.SYSVLONG = res.adapterResponse.SYSVLONG
/*
We would like to present a default DCPATH (deployment path) to the
@ -88,12 +89,14 @@ export class SasjsConfiguratorComponent implements OnInit {
*/
this.dcDirectory =
this.tmpDirectories[
['L', 'H', 'A', 'S'].includes(res.SYSSCPL.substring(0, 1))
['L', 'H', 'A', 'S'].includes(
res.adapterResponse.SYSSCPL.substring(0, 1)
)
? 'linux'
: 'windows'
]
this.dcAdminGroupList = res.groups
this.dcAdminGroupList = res.adapterResponse.groups
this.dcAdminGroup = this.dcAdminGroupList[0].GROUPNAME
this.loading = false

View File

@ -57,10 +57,10 @@ import { LicenceService } from '../services/licence.service'
import * as numbro from 'numbro'
import * as languages from 'numbro/dist/languages.min'
import { FileUploadEncoding } from '../models/FileUploadEncoding'
import {
ParseResult,
SpreadsheetService
} from '../services/spreadsheet.service'
import { SpreadsheetService } from '../services/spreadsheet.service'
import { UploadFileResponse } from '../models/UploadFile'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
import { ParseResult } from '../models/ParseResult.interface'
@Component({
selector: 'app-editor',
@ -571,20 +571,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.sasService
.uploadFile(this.uploadUrl, filesToUpload, { table: this.libds })
.then(
(res: any) => {
if (typeof res.sasjsAbort === 'undefined') {
if (typeof res.sasparams === 'undefined') {
(res: UploadFileResponse) => {
if (typeof res.adapterResponse.sasjsAbort === 'undefined') {
if (typeof res.adapterResponse.sasparams === 'undefined') {
return
} else {
this.uploadLoading = false
let params = res.sasparams[0]
let params = res.adapterResponse.sasparams[0]
this.successEnable = true
this.tableId = params.DSID
this.router.navigateByUrl('/stage/' + this.tableId)
}
} else {
// handle succesfull response
const abortRes = res
const abortRes = res.adapterResponse
const abortMsg = abortRes.sasjsAbort[0].MSG
const macMsg = abortRes.sasjsAbort[0].MAC
@ -609,7 +609,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.fileUploadInputCompList.first.nativeElement.value = ''
}
this.uploader.queue = []
this.eventService.catchResponseError('file upload', err)
this.eventService.catchResponseError(
'file upload',
err.adapterResponse
)
}
)
}
@ -1561,9 +1564,11 @@ export class EditorComponent implements OnInit, AfterViewInit {
'editors/stagedata',
this.$dataFormats
)
.then((res: any) => {
if (typeof res.sasparams !== 'undefined') {
this.router.navigateByUrl('/stage/' + res.sasparams[0].DSID)
.then((res: RequestWrapperResponse) => {
if (typeof res.adapterResponse.sasparams !== 'undefined') {
this.router.navigateByUrl(
'/stage/' + res.adapterResponse.sasparams[0].DSID
)
return true
}
@ -1591,7 +1596,10 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.disableSubmit = true
this.submit = false
let errorText = typeof err === 'string' ? err : JSON.stringify(err)
let errorText =
typeof err.adapterRespnse === 'string'
? err.adapterRespnse
: JSON.stringify(err.adapterRespnse)
this.eventService.showAbortModal(
'editors/stagedata',
@ -1983,8 +1991,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
suppressSuccessAbortModal: true,
suppressErrorAbortModal: true
})
.then((res: any) => {
const colSource = res.dynamic_values.map(
.then((res: RequestWrapperResponse) => {
const colSource = res.adapterResponse.dynamic_values.map(
(el: any) => el[this.cellValidationFields.RAW_VALUE]
)
@ -1998,8 +2006,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
...this.cellValidationSource[validationSourceIndex],
row: row,
col: column,
values: res.dynamic_values,
extended_values: res.dynamic_extended_values
values: res.adapterResponse.dynamic_values,
extended_values: res.adapterResponse.dynamic_extended_values
}
}

View File

@ -6,6 +6,7 @@ import { ServerType } from '@sasjs/utils/types/serverType'
import { HelperService } from '../services/helper.service'
import { SasService } from '../services/sas.service'
import { globals } from '../_globals'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-group',
@ -82,10 +83,12 @@ export class GroupComponent implements OnInit {
globals.usernav.groupList = groups
})
} else {
this.sasService.request('public/getgroups', null).then((res: any) => {
this.sasService
.request('public/getgroups', null)
.then((res: RequestWrapperResponse) => {
this.loading = false
this.groups = res.groups
globals.usernav.groupList = res.groups
this.groups = res.adapterResponse.groups
globals.usernav.groupList = res.adapterResponse.groups
})
}
} else {
@ -128,14 +131,15 @@ export class GroupComponent implements OnInit {
let data = { iwant: [{ groupid: this.paramURI }] }
this.sasService
.request('usernav/usermembersbygroup', data)
.then((res: any) => {
this.groupMembers = res.sasmembers
this.groupMemberCount = res.sasmembers.length
if (res.sasmembers[0] !== undefined) {
.then((res: RequestWrapperResponse) => {
this.groupMembers = res.adapterResponse.sasmembers
this.groupMemberCount = res.adapterResponse.sasmembers.length
if (res.adapterResponse.sasmembers[0] !== undefined) {
this.loading = false
this.groupUri = res.sasmembers[0].URIMEM || this.paramURI
this.groupName = res.sasmembers[0].GROUPNAME
this.groupDesc = res.sasmembers[0].GROUPDESC
this.groupUri =
res.adapterResponse.sasmembers[0].URIMEM || this.paramURI
this.groupName = res.adapterResponse.sasmembers[0].GROUPNAME
this.groupDesc = res.adapterResponse.sasmembers[0].GROUPDESC
if (!this.groupName) {
this.groupName = this.paramURI
@ -202,13 +206,13 @@ export class GroupComponent implements OnInit {
this.sasService
.request('usernav/usermembersbygroup', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
this.groupUri = group.GROUPURI
this.groupName = group.GROUPNAME
this.groupDesc = group.GROUPDESC
this.groupMembers = res.sasmembers
this.groupMemberCount = res.sasmembers.length
this.groupMembers = res.adapterResponse.sasmembers
this.groupMemberCount = res.adapterResponse.sasmembers.length
})
}
}

View File

@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'
import { ActivatedRoute, Router } from '@angular/router'
import { AppService, LicenceService, SasService } from '../services'
import { LicenseKeyData } from '../models/LicenseKeyData'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
enum LicenseActions {
key = 'key',
@ -116,8 +117,12 @@ export class LicensingComponent implements OnInit {
this.sasService
.request('admin/registerkey', table)
.then((res: any) => {
if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
.then((res: RequestWrapperResponse) => {
if (
res.adapterResponse.return &&
res.adapterResponse.return[0] &&
res.adapterResponse.return[0].MSG === 'SUCCESS'
) {
location.replace(location.href.split('#')[0])
}
})

View File

@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
import * as saveSvg from 'save-svg-as-png'
import { LoggerService } from '../services/logger.service'
import { LicenceService } from '../services/licence.service'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
const moment = require('moment')
@Component({
@ -115,8 +116,8 @@ export class LineageComponent {
await this.sasService
.request('lineage/getmetacols', libTable)
.then((res: any) => {
this.columnsList = res.metacols
.then((res: RequestWrapperResponse) => {
this.columnsList = res.adapterResponse.metacols
if (this.columnsList && this.columnsList.length > 0) {
// this.column = this.columnsList[0]['COLURI']
@ -174,8 +175,8 @@ export class LineageComponent {
let libTable = { SASControlTable: [{ liburi: $event }] }
await this.sasService
.request('lineage/getmetatables', libTable)
.then((res: any) => {
this.tablesList = res.metatables
.then((res: RequestWrapperResponse) => {
this.tablesList = res.adapterResponse.metatables
if (this.tablesList && this.tablesList.length > 0) {
library['tables'] = this.tablesList
@ -295,8 +296,8 @@ export class LineageComponent {
} else {
await this.sasService
.request('public/viewlibs', null)
.then((res: any) => {
this.libraryList = res.saslibs
.then((res: RequestWrapperResponse) => {
this.libraryList = res.adapterResponse.saslibs
this.helperService.displayLibraries(this.libraryList)
if (this.libraryList) {
@ -402,8 +403,8 @@ export class LineageComponent {
return new Promise<void>((resolve, reject) => {
this.sasService
.request('lineage/fetchtablelineage', libTable)
.then(async (res: any) => {
if (res.flatdata.length > 0) {
.then(async (res: RequestWrapperResponse) => {
if (res.adapterResponse.flatdata.length > 0) {
if (this.licenceService.checkLineageLimit()) {
this.eventService.showInfoModal(
'Notice',
@ -421,20 +422,22 @@ export class LineageComponent {
}
this.lineageTableName =
res.info[0].LIBREF + '.' + res.info[0].TABLENAME
res.adapterResponse.info[0].LIBREF +
'.' +
res.adapterResponse.info[0].TABLENAME
let dotArray = res.finalfinal
let dotArray = res.adapterResponse.finalfinal
let vizTmp: string = ''
for (let i = 0; i < dotArray.length; i++) {
vizTmp += unescape(dotArray[i].LINE) + '\n'
}
this.flatdata = res.flatdata
this.flatdata = res.adapterResponse.flatdata
if (this.libraryList) {
let libraryToSelect = this.libraryList.find((library: any) =>
res.info[0].LIBURI.toUpperCase().includes(
res.adapterResponse.info[0].LIBURI.toUpperCase().includes(
library.LIBRARYID.toUpperCase()
)
)
@ -450,7 +453,7 @@ export class LineageComponent {
if (libraryToSelect['tables']) {
tableToSelect = libraryToSelect['tables'].find((table: any) =>
table.TABLEURI.toUpperCase().includes(
res.info[0].TABLEID.toUpperCase()
res.adapterResponse.info[0].TABLEID.toUpperCase()
)
)
@ -495,10 +498,10 @@ export class LineageComponent {
.replace(/\sds:/g, '\nds:')
.replace(/\s\n/g, '\n')
this.idlookup = res.idlookup
this.idlookup = res.adapterResponse.idlookup
if (res.finalfinal.length > this.largeDotFileLimit) {
this.largeDotFileLines = res.finalfinal.length
if (res.adapterResponse.finalfinal.length > this.largeDotFileLimit) {
this.largeDotFileLines = res.adapterResponse.finalfinal.length
} else {
this.buildGraph()
}
@ -619,8 +622,8 @@ export class LineageComponent {
return new Promise<void>((resolve, reject) => {
this.sasService
.request('lineage/fetchcollineage', libTable)
.then(async (res: any) => {
if (res.flatdata.length > 0) {
.then(async (res: RequestWrapperResponse) => {
if (res.adapterResponse.flatdata.length > 0) {
if (this.licenceService.checkLineageLimit()) {
this.eventService.showInfoModal(
'Notice',
@ -631,18 +634,21 @@ export class LineageComponent {
}
}
if (typeof res === 'string') {
if (typeof res.adapterResponse === 'string') {
this.vizInput = 'digraph G {SAS Error}'
this.buildGraph()
return
}
this.lineageTableName = res.info[0].LIBREF + '.' + res.info[0].TABNAME
this.lineageColumnName = res.info[0].COLNAME
this.lineageTableName =
res.adapterResponse.info[0].LIBREF +
'.' +
res.adapterResponse.info[0].TABNAME
this.lineageColumnName = res.adapterResponse.info[0].COLNAME
this.idlookup = res.idlookup
this.idlookup = res.adapterResponse.idlookup
let dotArray = res.fromsas
let dotArray = res.adapterResponse.fromsas
let vizTmp: string = ''
for (let i = 0; i < dotArray.length; i++) {
vizTmp += unescape(dotArray[i].STRING) + '\n'
@ -653,11 +659,11 @@ export class LineageComponent {
.replace(/\sds:/g, '\nds:')
.replace(/\s\n/g, '\n')
this.flatdata = res.flatdata
this.flatdata = res.adapterResponse.flatdata
if (this.libraryList) {
let libraryToSelect = this.libraryList.find((library: any) =>
res.info[0]?.LIBURI?.toUpperCase()?.includes(
res.adapterResponse.info[0]?.LIBURI?.toUpperCase()?.includes(
library?.LIBRARYID?.toUpperCase()
)
)
@ -672,7 +678,8 @@ export class LineageComponent {
if (libraryToSelect['tables']) {
tableToSelect = libraryToSelect['tables'].find(
(table: any) => table.TABLEURI === res.info[0].TABURI
(table: any) =>
table.TABLEURI === res.adapterResponse.info[0].TABURI
)
if (tableToSelect) {
@ -714,8 +721,8 @@ export class LineageComponent {
}
}
if (res.fromsas.length > this.largeDotFileLimit) {
this.largeDotFileLines = res.fromsas.length
if (res.adapterResponse.fromsas.length > this.largeDotFileLimit) {
this.largeDotFileLines = res.adapterResponse.fromsas.length
} else {
this.buildGraph()
}

View File

@ -9,6 +9,7 @@ import { SasService } from '../services/sas.service'
import { globals } from '../_globals'
import { Injectable } from '@angular/core'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
interface MetaData {
NAME: any
@ -109,26 +110,34 @@ export class MetadataComponent implements OnInit {
this.metatypesLoading = false
this.metaDataSearch = globals.metadata.metaDataSearch
} else {
this.sasService.request('metanav/metatypes', null).then((res: any) => {
this.metaDataList = res.types
this.sasService
.request('metanav/metatypes', null)
.then((res: RequestWrapperResponse) => {
this.metaDataList = res.adapterResponse.types
globals.metadata.metaDataList = this.metaDataList
this.loading = false
this.metatypesLoading = false
})
this.sasService.request('metanav/metarepos', null).then((res: any) => {
this.sasService
.request('metanav/metarepos', null)
.then((res: RequestWrapperResponse) => {
let foundation = false
this.repositories = []
for (let index = 0; index < res.outrepos.length; index++) {
this.repositories.push(res.outrepos[index].NAME)
if (res.outrepos[index].NAME === 'Foundation') {
for (
let index = 0;
index < res.adapterResponse.outrepos.length;
index++
) {
this.repositories.push(res.adapterResponse.outrepos[index].NAME)
if (res.adapterResponse.outrepos[index].NAME === 'Foundation') {
foundation = true
}
}
if (foundation) {
this.repository = 'Foundation'
} else {
this.repository = res.outrepos[0].NAME
this.repository = res.adapterResponse.outrepos[0].NAME
}
globals.metadata.metaRepositories = this.repositories
globals.metadata.selectedRepository = this.repository
@ -183,8 +192,10 @@ export class MetadataComponent implements OnInit {
const data: any = {
SASControlTable: [{ metatype: $event, repo: this.repository }]
}
this.sasService.request('metanav/metaobjects', data).then((res: any) => {
this.metaObjectList = res.objects
this.sasService
.request('metanav/metaobjects', data)
.then((res: RequestWrapperResponse) => {
this.metaObjectList = res.adapterResponse.objects
this.getMetaObjectAttributes(this.metaObjectSize)
this.loading = false
this.assoTypeSelected = $event
@ -197,12 +208,16 @@ export class MetadataComponent implements OnInit {
let data: any = {
SASControlTable: [{ objecturi: $event }]
}
this.sasService.request('metanav/metadetails', data).then((res: any) => {
this.metaObjectAssociations = res.associations
this.root$ = of(this.getAssosiationsCount(res.associations))
this.sasService
.request('metanav/metadetails', data)
.then((res: RequestWrapperResponse) => {
this.metaObjectAssociations = res.adapterResponse.associations
this.root$ = of(
this.getAssosiationsCount(res.adapterResponse.associations)
)
this.showAcc = true
this.showTable = true
let metaObjectName = res.attributes.find(
let metaObjectName = res.adapterResponse.attributes.find(
(x: any) => x.NAME === 'Name'
).VALUE
this.assoObjectSelected = metaObjectName
@ -221,7 +236,7 @@ export class MetadataComponent implements OnInit {
url + '/object/' + $event.slice(1 + $event.indexOf('\\'))
)
}
this.metaObjectAttributes = res.attributes
this.metaObjectAttributes = res.adapterResponse.attributes
})
}
@ -229,8 +244,10 @@ export class MetadataComponent implements OnInit {
let data: any = {
SASControlTable: [{ objecturi: $event }]
}
this.sasService.request('metanav/metadetails', data).then((res: any) => {
this.metaObjectAttributes = res.attributes
this.sasService
.request('metanav/metadetails', data)
.then((res: RequestWrapperResponse) => {
this.metaObjectAttributes = res.adapterResponse.attributes
this.showTable = true
})
}
@ -244,7 +261,7 @@ export class MetadataComponent implements OnInit {
details: []
})
}
let assocObj = assosiationsHash.get(assosiation.ASSOC)
let assocObj: any = assosiationsHash.get(assosiation.ASSOC)
assocObj.count++
assocObj.details.push({
ASSOCURI: assosiation.ASSOCURI,
@ -254,7 +271,7 @@ export class MetadataComponent implements OnInit {
})
}
let assocGrouped: Array<any> = []
assosiationsHash.forEach(function (val, key) {
assosiationsHash.forEach(function (val: any, key) {
assocGrouped.push({
ASSOC: key,
count: val.count,
@ -294,9 +311,9 @@ export class MetadataComponent implements OnInit {
}
return this.sasService
.request('metanav/metadetails', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.showTable = true
this.metaObjectAttributes = res.attributes
this.metaObjectAttributes = res.adapterResponse.attributes
this.assoObjectSelected = asso.NAME
let url = this.router.url
if (this.objectRoute) {
@ -314,7 +331,7 @@ export class MetadataComponent implements OnInit {
asso.ASSOCURI.slice(1 + asso.ASSOCURI.indexOf('\\'))
)
}
return this.getAssosiationsCount(res.associations)
return this.getAssosiationsCount(res.adapterResponse.associations)
})
}

View File

@ -0,0 +1,23 @@
import { DcValidator } from '../shared/dc-validator/dc-validator'
import { FileUploadEncoding } from './FileUploadEncoding'
import { FileUploader } from './FileUploader.class'
import { ExcelRule } from './TableData'
export interface ParseParams {
file: File
password?: string
dcValidator: DcValidator
/**
* Parse function will manipulate and return the uploader array which can be provided with files already in the queue
* Otherwise new empty instance will be created.
*/
uploader?: FileUploader
headerPks: string[]
headerArray: string[]
headerShow: string[]
timeHeaders: string[]
dateHeaders: string[]
dateTimeHeaders: string[]
xlRules: ExcelRule[]
encoding?: FileUploadEncoding
}

View File

@ -0,0 +1,15 @@
import { FileUploader } from './FileUploader.class'
import SheetInfo from './SheetInfo'
export interface ParseResult {
/**
* In case of CSV file, won't be returned
*/
data?: any[]
/**
* In case of CSV file, won't be returned
*/
headerShow?: string[]
rangeSheetRes?: SheetInfo
uploader: FileUploader
}

View File

@ -0,0 +1,4 @@
export interface UploadFileResponse {
adapterResponse: any
log?: string
}

View File

@ -0,0 +1,4 @@
export interface RequestWrapperResponse<responseType = any> {
adapterResponse: responseType
log?: string
}

View File

@ -22,8 +22,8 @@
/>
</div>
<ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
<div class="text-center mb-10">
<ng-container *ngIf="parsedDatasets.length && !submittedCsvDatasets.length">
<div *ngIf="!excelsSubmitted" class="text-center mb-10">
<button (click)="onDiscard()" class="btn btn-danger btn-sm mr-10">
Discard
</button>
@ -41,30 +41,54 @@
<clr-tree-node *ngFor="let dataset of parsedDatasets">
<button
(click)="onParsedDatasetClick(dataset)"
class="clr-treenode-link whitespace-nowrap"
class="clr-treenode-link whitespace-nowrap d-flex clr-align-items-center"
[class.active]="dataset.active"
>
<ng-container *ngIf="dataset.submitResult">
<cds-icon
*ngIf="!(dataset.datasource && dataset.parseResult)"
*ngIf="dataset.submitResult.error"
status="danger"
shape="exclamation-circle"
></cds-icon>
<cds-icon
*ngIf="dataset.datasource && dataset.parseResult"
*ngIf="dataset.submitResult.success"
status="success"
shape="check-circle"
></cds-icon>
</ng-container>
<ng-container *ngIf="!dataset.submitResult">
<ng-container *ngIf="dataset.datasource">
<cds-icon
*ngIf="!(dataset.datasource.length && dataset.parseResult)"
status="danger"
shape="exclamation-circle"
></cds-icon>
<cds-icon
*ngIf="dataset.datasource.length && dataset.parseResult"
shape="table"
></cds-icon>
{{ dataset.libds }}
</ng-container>
<ng-container *ngIf="!dataset.datasource">
<cds-icon *ngIf="!dataset.parsingTable" shape="table"></cds-icon>
<clr-spinner *ngIf="dataset.parsingTable" clrSmall></clr-spinner>
</ng-container>
</ng-container>
<span class="ml-5"> {{ dataset.libds }} </span>
</button>
</clr-tree-node>
</clr-tree>
</ng-container>
<ng-container *ngIf="submittedDatasets.length">
<ng-container *ngIf="submittedCsvDatasets.length">
<p cds-text="caption" class="ml-10 mb-10 mt-10">Submitted tables:</p>
<clr-tree>
<clr-tree-node *ngFor="let dataset of submittedDatasets">
<clr-tree-node *ngFor="let dataset of submittedCsvDatasets">
<button
(click)="onSubmittedDatasetClick(dataset)"
(click)="onSubmittedCsvDatasetClick(dataset)"
class="clr-treenode-link whitespace-nowrap"
[class.active]="dataset.active"
>
@ -90,7 +114,7 @@
</div> -->
</app-sidebar>
<div class="content-area">
<div #contentArea class="content-area">
<div class="card no-borders h-100 d-flex clr-flex-column">
<div
class="header-row clr-row justify-content-between clr-justify-content-center w-100 m-0"
@ -116,7 +140,7 @@
<ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
<div class="d-flex clr-justify-content-center mt-15">
<div class="dataset-input-wrapper">
<p cds-text="secondary regular" class="mb-20">
<p cds-text="secondary regular" class="mb-5">
Selected file: <strong>{{ selectedFile.name }}</strong>
<clr-tooltip>
<cds-icon
@ -129,6 +153,9 @@
<clr-tooltip-content> Discard the file </clr-tooltip-content>
</clr-tooltip>
</p>
<p cds-text="secondary regular" class="mb-20">
File size: <strong>{{ selectedFile.sizeMB }} MB</strong>
</p>
<p cds-text="secondary regular" class="mb-15">
Paste or type the list of datasets to upload:
</p>
@ -151,7 +178,7 @@
<div class="dataset-selection-actions text-right mt-10">
<button
(click)="onUploadFile()"
(click)="onStartParsingFile()"
id="continue-btn"
class="btn btn-primary btn-sm"
[disabled]="!matchedDatasets.length"
@ -164,7 +191,9 @@
</div>
</ng-container>
<ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
<ng-container
*ngIf="parsedDatasets.length && !submittedCsvDatasets.length"
>
<div
*ngIf="!activeParsedDataset"
class="no-table-selected pointer-events-none"
@ -180,7 +209,68 @@
</div>
<ng-container *ngIf="activeParsedDataset">
<div class="d-flex clr-justify-content-between p-10">
<div
*ngIf="activeParsedDataset.submitResult"
class="d-flex clr-justify-content-between p-10 mt-15 submission-results"
>
<div>
<p cds-text="secondary regular" class="mb-10">
Submit Status:
<span
*ngIf="activeParsedDataset.submitResult?.success"
class="color-green"
><strong>SUCCESS</strong></span
>
<span
*ngIf="activeParsedDataset.submitResult?.error"
class="color-red"
><strong>ERROR</strong></span
>
</p>
<p
*ngIf="activeParsedDataset.submitResult?.error"
cds-text="secondary regular"
>
Error details:
</p>
</div>
<div>
<button
*ngIf="
!submittingCsv && activeParsedDataset.submitResult?.error
"
(click)="reSubmitTable(activeParsedDataset)"
class="btn btn-primary mt-10"
[clrLoading]="submitLoading"
>
Resubmit
</button>
<button
(click)="
downloadFile(
activeParsedDataset.submitResult.log ||
activeParsedDataset.submitResult.success ||
activeParsedDataset.submitResult.error
)
"
class="btn btn-primary-outline mt-10"
>
Download log
</button>
</div>
</div>
<div
*ngIf="activeParsedDataset.submitResult?.error"
class="error-field mt-15"
>
<div class="log-wrapper">
{{ activeParsedDataset.submitResult?.error | json }}
</div>
</div>
<div class="d-flex clr-justify-content-between p-10 mt-15">
<div>
<p cds-text="secondary regular" class="mb-10">
Found in range:
@ -197,7 +287,17 @@
</ng-container>
<ng-container *ngIf="!activeParsedDataset.parseResult">
<strong>No data found</strong>
<strong *ngIf="!activeParsedDataset.parsingTable"
>No data found</strong
>
<span
*ngIf="activeParsedDataset.parsingTable"
class="d-flex clr-align-items-center"
>
<strong>Searching for the data...</strong>
<clr-spinner class="ml-5" clrSmall></clr-spinner>
</span>
</ng-container>
</p>
<p cds-text="secondary regular">
@ -262,9 +362,9 @@
</ng-container>
</ng-container>
<ng-container *ngIf="submittedDatasets.length">
<ng-container *ngIf="submittedCsvDatasets.length">
<div
*ngIf="!activeSubmittedDataset"
*ngIf="!activeSubmittedCsvDataset"
class="no-table-selected pointer-events-none"
>
<clr-icon
@ -278,31 +378,17 @@
</div>
</ng-container>
<ng-container *ngIf="activeSubmittedDataset">
<ng-container *ngIf="activeSubmittedCsvDataset">
<div class="d-flex clr-justify-content-between p-10">
<div>
<p
*ngIf="activeSubmittedDataset.parseResult"
cds-text="secondary regular"
class="mb-10"
>
Found in range:
<strong
>"{{
activeSubmittedDataset.parseResult.rangeSheetRes?.sheetName
}}"!{{
activeSubmittedDataset.parseResult.rangeSheetRes?.rangeAddress
}}</strong
>
</p>
<p cds-text="secondary regular" class="mb-10">
Matched with dataset:
<strong>
<clr-tooltip>
<a
clrTooltipTrigger
[routerLink]="'/editor/' + activeSubmittedDataset.libds"
>{{ activeSubmittedDataset.libds }}</a
[routerLink]="'/editor/' + activeSubmittedCsvDataset.libds"
>{{ activeSubmittedCsvDataset.libds }}</a
>
<clr-tooltip-content
[clrPosition]="'top-right'"
@ -315,15 +401,17 @@
</p>
<p cds-text="secondary regular" class="mb-10">
Status:
<span *ngIf="activeSubmittedDataset.success" class="color-green"
<span
*ngIf="activeSubmittedCsvDataset.success"
class="color-green"
><strong>SUCCESS</strong></span
>
<span *ngIf="activeSubmittedDataset.error" class="color-red"
<span *ngIf="activeSubmittedCsvDataset.error" class="color-red"
><strong>ERROR</strong></span
>
</p>
<p
*ngIf="activeSubmittedDataset.error"
*ngIf="activeSubmittedCsvDataset.error"
cds-text="secondary regular"
>
Error details:
@ -331,18 +419,11 @@
</div>
<div>
<button
*ngIf="!submittingCsv && activeSubmittedDataset.error"
(click)="reSubmitTable(activeSubmittedDataset)"
class="btn btn-primary mt-10"
[clrLoading]="submitLoading"
>
Resubmit
</button>
<button
(click)="
downloadFile(
activeSubmittedDataset.success || activeSubmittedDataset.error
activeSubmittedCsvDataset.success ||
activeSubmittedCsvDataset.error
)
"
class="btn btn-primary-outline mt-10"
@ -352,9 +433,9 @@
</div>
</div>
<div *ngIf="activeSubmittedDataset.error" class="error-field mt-15">
<div *ngIf="activeSubmittedCsvDataset.error" class="error-field mt-15">
<div class="log-wrapper">
{{ activeSubmittedDataset.error | json }}
{{ activeSubmittedCsvDataset.error | json }}
</div>
</div>
</ng-container>

View File

@ -48,3 +48,7 @@
.licence-limit-notice {
color: var(--cds-alias-status-warning-dark);
}
.submission-results {
border-bottom: 1px solid #d3d3d3;
}

View File

@ -1,8 +1,10 @@
import {
ChangeDetectorRef,
Component,
ElementRef,
HostBinding,
OnInit
OnInit,
ViewChild
} from '@angular/core'
import {
EventService,
@ -11,17 +13,13 @@ import {
SasService,
SasStoreService
} from '../services'
import * as XLSX from '@sheet/crypto'
import { globals } from '../_globals'
import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
import { DcValidator } from '../shared/dc-validator/dc-validator'
import { ExcelRule } from '../models/TableData'
import { HotTableInterface } from '../models/HotTable.interface'
import { Col } from '../shared/dc-validator/models/col.model'
import {
ParseResult,
SpreadsheetService
} from '../services/spreadsheet.service'
import { SpreadsheetService } from '../services/spreadsheet.service'
import Handsontable from 'handsontable'
import { HotTableRegisterer } from '@handsontable/angular'
import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
@ -29,6 +27,9 @@ import { CellChange, ChangeSource } from 'handsontable/common'
import { baseAfterGetColHeader } from '../shared/utils/hot.utils'
import { ColumnSettings } from 'handsontable/settings'
import { UploadFile } from '@sasjs/adapter'
import { UploadFileResponse } from '../models/UploadFile'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
import { ParseResult } from '../models/ParseResult.interface'
@Component({
selector: 'app-multi-dataset',
@ -37,6 +38,7 @@ import { UploadFile } from '@sasjs/adapter'
})
export class MultiDatasetComponent implements OnInit {
@HostBinding('class.content-container') contentContainerClass = true
@ViewChild('contentArea', { static: true }) contentAreaRef!: ElementRef
public licenceState = this.licenceService.licenceState
public Infinity = Infinity
@ -48,15 +50,19 @@ export class MultiDatasetComponent implements OnInit {
public csvFiles: UploadFile[] = []
public csvSubmitting: boolean = false
public selectedFile: File | null = null
public autoDetectingColumns: boolean = false
public selectedFile: SelectedFile | null = null
public parsedDatasets: ParsedDataset[] = []
public submittedDatasets: SubmittedDatasetResult[] = []
public submittedCsvDatasets: SubmittedCsvDatasetResult[] = []
public datasetsLoading: boolean = false
public uploadLoading: boolean = false
public submitLoading: boolean = false
public matchedDatasets: string[] = []
public sheetNames: string[] = []
public userInputDatasets: string = ''
public libsAndTables: {
@ -194,6 +200,11 @@ export class MultiDatasetComponent implements OnInit {
// For EXCEL if multiple files, we only take one (the first one)
this.selectedFile = event.target.files[0]
if (this.selectedFile)
this.selectedFile.sizeMB = this.spreadsheetService.bytesToMB(
this.selectedFile.size
)
this.initUserInputHot()
this.onAutoDetectColumns()
} else if (matchedExtension === 'csv') {
@ -246,7 +257,7 @@ export class MultiDatasetComponent implements OnInit {
this.userInputDatasets = ''
}
async onUploadFile() {
async onStartParsingFile() {
this.uploadLoading = true
const datasetFetchingPromises: Promise<
@ -271,18 +282,28 @@ export class MultiDatasetComponent implements OnInit {
const datasetObjects = this.buildDatasetsObjects(datasets)
for (let datasetObject of datasetObjects) {
datasetObjects.forEach((datasetObject) => {
this.parsedDatasets.push({
libds: datasetObject.libds,
includeInSubmission: true,
datasetInfo: datasetObject,
parsingTable: true
})
})
for (let parsedDataset of this.parsedDatasets) {
this.spreadsheetService
.parseExcelFile({
file: this.selectedFile!,
dcValidator: datasetObject.dcValidator!,
headerPks: datasetObject.headerPks,
headerArray: datasetObject.headerArray,
password: this.selectedFile!.password || undefined,
dcValidator: parsedDataset.datasetInfo.dcValidator!,
headerPks: parsedDataset.datasetInfo.headerPks,
headerArray: parsedDataset.datasetInfo.headerArray,
headerShow: [],
timeHeaders: datasetObject.timeHeaders,
dateHeaders: datasetObject.dateHeaders,
dateTimeHeaders: datasetObject.dateTimeHeaders,
xlRules: datasetObject.xlRules
timeHeaders: parsedDataset.datasetInfo.timeHeaders,
dateHeaders: parsedDataset.datasetInfo.dateHeaders,
dateTimeHeaders: parsedDataset.datasetInfo.dateTimeHeaders,
xlRules: parsedDataset.datasetInfo.xlRules
})
.then((parseResult: ParseResult | undefined) => {
console.log('parseResult', parseResult)
@ -304,23 +325,17 @@ export class MultiDatasetComponent implements OnInit {
datasource.push(itemObject)
})
this.parsedDatasets.push({
libds: datasetObject.libds,
parseResult: parseResult,
includeInSubmission: true,
datasetInfo: datasetObject,
datasource: datasource
})
parsedDataset.datasource = datasource
parsedDataset.parseResult = parseResult
parsedDataset.parsingTable = false
}
})
.catch((error: string) => {
console.warn('Parsing excel file error.', error)
this.parsedDatasets.push({
libds: datasetObject.libds,
includeInSubmission: false,
datasetInfo: datasetObject
})
parsedDataset.datasource = []
parsedDataset.includeInSubmission = false
parsedDataset.parsingTable = false
})
}
})
@ -349,13 +364,17 @@ export class MultiDatasetComponent implements OnInit {
setTimeout(() => {
this.hotInstance = this.hotRegisterer.getInstance('hotInstance')
// Set height of parsed data to full height of the page content area
const contentAreaHeight = this.contentAreaRef.nativeElement.clientHeight
const hotHeight = `${contentAreaHeight - 160}px`
if (this.activeParsedDataset) {
this.hotInstance.updateSettings({
data: this.activeParsedDataset.datasource || [],
colHeaders: this.activeParsedDataset.datasetInfo.headerColumns,
columns: this.activeParsedDataset.datasetInfo.dcValidator?.getRules(),
readOnly: true,
height: '300px',
height: hotHeight || '300px',
className: 'htDark'
})
}
@ -507,13 +526,32 @@ export class MultiDatasetComponent implements OnInit {
* convention. {@link isValidDatasetFormat}
*/
async onAutoDetectColumns() {
const sheetNames = await this.parseExcelSheetNames()
let passwordError = false
if (sheetNames) {
await this.parseExcelSheetNames()
.then((sheetNames) => {
this.sheetNames = sheetNames
})
.catch((err) => {
if (err.includes('password')) {
passwordError = true
}
})
if (passwordError) {
this.onDiscardFile()
this.eventService.showInfoModal(
'Locked file',
'We failed to unlock the file.'
)
return
}
if (this.sheetNames) {
this.matchedDatasets = []
this.userInputDatasets = ''
sheetNames.forEach((sheetName: string, index: number) => {
this.sheetNames.forEach((sheetName: string, index: number) => {
const trimmedSheetname = sheetName.trim()
if (
@ -561,8 +599,8 @@ export class MultiDatasetComponent implements OnInit {
this.initHot()
}
onSubmittedDatasetClick(submittedDataset: SubmittedDatasetResult) {
this.deselectAllSubmittedDatasets()
onSubmittedCsvDatasetClick(submittedDataset: SubmittedCsvDatasetResult) {
this.deselectAllSubmittedCsvDatasets()
submittedDataset.active = true
}
@ -571,8 +609,10 @@ export class MultiDatasetComponent implements OnInit {
return this.parsedDatasets.find((dataset) => dataset.active)
}
public get activeSubmittedDataset(): SubmittedDatasetResult | undefined {
return this.submittedDatasets.find((dataset) => dataset.active)
public get activeSubmittedCsvDataset():
| SubmittedCsvDatasetResult
| undefined {
return this.submittedCsvDatasets.find((dataset) => dataset.active)
}
public get notFoundDatasets(): string[] {
@ -606,8 +646,12 @@ export class MultiDatasetComponent implements OnInit {
return this.csvFiles.length > 0
}
public get excelsSubmitted(): boolean {
return !!this.parsedDatasets.filter((ds) => ds.submitResult).length
}
public downloadFile(response: any) {
const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log`
const filename = `stagedata-${this.activeSubmittedCsvDataset?.libds}-log`
this.helperService.downloadTextFile(filename, JSON.stringify(response))
}
@ -615,7 +659,7 @@ export class MultiDatasetComponent implements OnInit {
* Submits attached CSVs which are matched with existing datasets
*/
async submitCsvFiles() {
let requestsResults: SubmittedDatasetResult[] = []
let requestsResults: SubmittedCsvDatasetResult[] = []
for (let file of this.csvFiles) {
const libds = this.parseDatasetFromCsvName(file.fileName)
@ -626,17 +670,17 @@ export class MultiDatasetComponent implements OnInit {
await this.sasService
.uploadFile('services/editors/loadfile', [file], { table: libds })
.then(
(res: any) => {
if (typeof res.sasjsAbort !== 'undefined') {
error = res.sasjsAbort
(res: UploadFileResponse) => {
if (typeof res.adapterResponse.sasjsAbort !== 'undefined') {
error = res.adapterResponse.sasjsAbort
} else {
success = res
success = res.adapterResponse
}
},
(err: any) => {
console.error('err', err)
error = err
error = err.adapterResponse
}
)
@ -647,7 +691,7 @@ export class MultiDatasetComponent implements OnInit {
})
}
this.submittedDatasets = requestsResults
this.submittedCsvDatasets = requestsResults
}
/**
@ -678,7 +722,7 @@ export class MultiDatasetComponent implements OnInit {
}
/**
* Sends tables to the SAS sequentially
* Sends tables found in an excel to the SAS sequentially
*
* @param explicitDatasets if empty all datasets will be sent, otherwise only datasets
* in the array will be sent. eg. ['lib1.table_1', 'lib1.table_2']
@ -688,10 +732,6 @@ export class MultiDatasetComponent implements OnInit {
this.submitLoading = true
let requestsResults: SubmittedDatasetResult[] = explicitDatasets
? this.submittedDatasets
: []
for (let table of this.parsedDatasets) {
// Skip the table if no data inside
if (!table.parseResult || !table.datasource) continue
@ -733,6 +773,7 @@ export class MultiDatasetComponent implements OnInit {
let error
let success
let log
await this.sasStoreService
.updateTable(
@ -741,56 +782,47 @@ export class MultiDatasetComponent implements OnInit {
'SASControlTable',
'editors/stagedata',
table.datasetInfo.data.$sasdata,
true
true,
{
debug: true
}
)
.then((res: EditorsStageDataSASResponse) => {
success = res
.then((res: RequestWrapperResponse<EditorsStageDataSASResponse>) => {
success = res.adapterResponse
log = res.log
})
.catch((err: any) => {
console.error('err', err)
error = err
error = err.adapterResponse
log = err.log
})
const requestResult = {
success,
error,
log,
parseResult: table.parseResult,
libds: table.libds
}
// If explicit datasets are set don't just push to th array
// instead replace if result already exist from before (this might be re-submit)
if (explicitDatasets) {
const existingResultIndex = requestsResults.findIndex(
(result) => result.libds === table.libds
)
if (existingResultIndex > -1) {
requestsResults[existingResultIndex] = requestResult
} else {
requestsResults.push(requestResult)
}
} else {
requestsResults.push(requestResult)
table.submitResult = requestResult
}
}
}
this.submittedDatasets = requestsResults
this.showSubmitReasonModal = false
this.submitLoading = false
this.deselectAllParsedDatasets()
}
async reSubmitTable(activeSubmittedDataset: SubmittedDatasetResult) {
async reSubmitTable(activeSubmittedDataset: ParsedDataset) {
// Submit only particular table
await this.submitTables([activeSubmittedDataset.libds])
// Activate new resubmitted table
const newSubmittedDataset = this.submittedDatasets.find(
const newSubmittedDataset = this.parsedDatasets.find(
(sd) => sd.libds === activeSubmittedDataset.libds
)
if (newSubmittedDataset) newSubmittedDataset.active = true
}
@ -815,40 +847,26 @@ export class MultiDatasetComponent implements OnInit {
.map((row) => (row ? `${row[0]}.${row[1]}` : ''))
}
private parseExcelSheetNames(): Promise<string[]> {
/**
* Read the file minimally just to get the sheet names, not reading full file
* to help boost the performance
*
* @returns sheet names in string array
*/
private async parseExcelSheetNames(): Promise<string[]> {
return new Promise((resolve, reject) => {
const reader = new FileReader()
if (!this.selectedFile) return resolve([])
if (!this.selectedFile) {
console.warn('selectedFile is missing')
return resolve([])
}
this.spreadsheetService
.parseExcelSheetNames(this.selectedFile)
.then((parsed) => {
if (parsed.password) this.selectedFile!.password = parsed.password
reader.onload = (event: ProgressEvent<FileReader>) => {
if (!event?.target) {
console.warn('File reader event.target is missing')
return
}
const data = event.target.result
const workbook = XLSX.read(data, {
type: 'binary'
return resolve(parsed.sheetNames)
})
.catch((err) => {
return reject(err)
})
try {
const sheet_names_list = workbook.SheetNames
return resolve(sheet_names_list)
} catch (e) {
console.error(e)
}
}
reader.onerror = function (ex) {
console.log(ex)
}
reader.readAsBinaryString(this.selectedFile)
})
}
@ -977,8 +995,8 @@ export class MultiDatasetComponent implements OnInit {
}
}
private deselectAllSubmittedDatasets() {
for (let submittedDataset of this.submittedDatasets) {
private deselectAllSubmittedCsvDatasets() {
for (let submittedDataset of this.submittedCsvDatasets) {
submittedDataset.active = false
}
}
@ -1002,16 +1020,28 @@ export interface ParsedDataset {
libds: string
parseResult?: ParseResult
datasetInfo: DatasetsObject
submitResult?: SubmittedDatasetResult
datasource?: any[]
includeInSubmission: boolean
status?: 'success' | 'error'
active?: boolean
parsingTable?: boolean
}
export interface SubmittedDatasetResult {
success: EditorsStageDataSASResponse | undefined
error: any
log?: string
}
export interface SubmittedCsvDatasetResult {
libds: string
success: EditorsStageDataSASResponse | undefined
error: any
parseResult?: ParseResult
active?: boolean
}
export interface SelectedFile extends File {
sizeMB?: number
password?: string
}

View File

@ -4,6 +4,7 @@ import { HelperService } from '../services/helper.service'
import { Location } from '@angular/common'
import { Router, ActivatedRoute } from '@angular/router'
import { SasService } from '../services/sas.service'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-role',
@ -47,10 +48,12 @@ export class RoleComponent implements OnInit {
} else {
if (globals.usernav.roleList === undefined) {
this.loading = true
this.sasService.request('usernav/userroles', null).then((res: any) => {
this.sasService
.request('usernav/userroles', null)
.then((res: RequestWrapperResponse) => {
this.loading = false
this.roles = res.roles
globals.usernav.roleList = res.roles
this.roles = res.adapterResponse.roles
globals.usernav.roleList = res.adapterResponse.roles
if (this.paramPresent) {
if (this.roles !== undefined) {
let validRole = this.findRole(this.roles, this.paramURI)
@ -59,12 +62,14 @@ export class RoleComponent implements OnInit {
let data = { iwant: [{ roleid: this.paramURI }] }
this.sasService
.request('usernav/usermembersbyrole', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
this.roleMembers = res.sasmembers
this.roleMembersCount = res.sasmembers.length
this.roleGroups = res.sasgroups
this.roleGroupsCount = res.sasgroups.length
this.roleMembers = res.adapterResponse.sasmembers
this.roleMembersCount =
res.adapterResponse.sasmembers.length
this.roleGroups = res.adapterResponse.sasgroups
this.roleGroupsCount =
res.adapterResponse.sasgroups.length
this.roleUri = validRole.ROLEURI
this.roleName = validRole.ROLENAME
this.roleDesc = validRole.ROLEDESC
@ -76,9 +81,11 @@ export class RoleComponent implements OnInit {
} else {
this.roles = globals.usernav.roleList
this.roleSearch = globals.usernav.roleSearch
this.sasService.request('usernav/userroles', null).then((res: any) => {
this.roles = res.roles
globals.usernav.roleList = res.roles
this.sasService
.request('usernav/userroles', null)
.then((res: RequestWrapperResponse) => {
this.roles = res.adapterResponse.roles
globals.usernav.roleList = res.adapterResponse.roles
if (this.paramPresent) {
if (this.roles !== undefined) {
@ -88,12 +95,14 @@ export class RoleComponent implements OnInit {
let data = { iwant: [{ roleid: this.paramURI }] }
this.sasService
.request('usernav/usermembersbyrole', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
this.roleMembers = res.sasmembers
this.roleMembersCount = res.sasmembers.length
this.roleGroups = res.sasgroups
this.roleGroupsCount = res.sasgroups.length
this.roleMembers = res.adapterResponse.sasmembers
this.roleMembersCount =
res.adapterResponse.sasmembers.length
this.roleGroups = res.adapterResponse.sasgroups
this.roleGroupsCount =
res.adapterResponse.sasgroups.length
this.roleUri = validRole.ROLEURI
this.roleName = validRole.ROLENAME
this.roleDesc = validRole.ROLEDESC
@ -125,12 +134,12 @@ export class RoleComponent implements OnInit {
let data = { iwant: [{ roleid: role.ROLEURI }] }
this.sasService
.request('usernav/usermembersbyrole', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
this.roleMembers = res.sasmembers
this.roleMembersCount = res.sasmembers.length
this.roleGroups = res.sasgroups
this.roleGroupsCount = res.sasgroups.length
this.roleMembers = res.adapterResponse.sasmembers
this.roleMembersCount = res.adapterResponse.sasmembers.length
this.roleGroups = res.adapterResponse.sasgroups
this.roleGroupsCount = res.adapterResponse.sasgroups.length
this.roleUri = role.ROLEURI
this.roleName = role.ROLENAME
this.roleDesc = role.ROLEDESC

View File

@ -9,11 +9,13 @@ import { EnvironmentInfo } from '../system/models/environment-info.model'
import { LicenceService } from './licence.service'
import { AppSettingsService } from './app-settings.service'
import { AppThemes } from '../models/AppSettings'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
import { AppStoreService } from './app-store.service'
@Injectable()
export class AppService {
public syssite = new BehaviorSubject<string[] | null>(null)
private environmentInfo: EnvironmentInfo | null = null
private environmentInfo: EnvironmentInfo = {}
constructor(
private licenceService: LicenceService,
@ -21,7 +23,8 @@ export class AppService {
private sasService: SasService,
private loggerService: LoggerService,
private appSettingsService: AppSettingsService,
private router: Router
private router: Router,
private appStoreService: AppStoreService
) {
this.subscribe()
@ -81,16 +84,19 @@ export class AppService {
await this.sasService
.request('public/startupservice', null)
.then(async (res: any) => {
this.syssite.next([res.SYSSITE])
.then(async (res: RequestWrapperResponse) => {
this.syssite.next([res.adapterResponse.SYSSITE])
let missingProps: string[] = []
if (!res.globvars || (res.globvars && !res.globvars[0]))
if (
!res.adapterResponse.globvars ||
(res.adapterResponse.globvars && !res.adapterResponse.globvars[0])
)
missingProps.push('Globvars')
if (!res.sasdatasets) missingProps.push('Sasdatasets')
if (!res.saslibs) missingProps.push('Saslibs')
if (!res.xlmaps) missingProps.push('XLMaps')
if (!res.adapterResponse.sasdatasets) missingProps.push('Sasdatasets')
if (!res.adapterResponse.saslibs) missingProps.push('Saslibs')
if (!res.adapterResponse.xlmaps) missingProps.push('XLMaps')
if (missingProps.length > 0) {
startupServiceError = true
@ -103,23 +109,26 @@ export class AppService {
return
}
const dcAdapterSettings = this.appStoreService.getDcAdapterSettings()
this.environmentInfo = {
SYSSITE: res.SYSSITE,
SYSSCPL: res.SYSSCPL,
SYSTCPIPHOSTNAME: res.SYSTCPIPHOSTNAME,
SYSVLONG: res.SYSVLONG,
MEMSIZE: res.MEMSIZE,
SYSPROCESSMODE: res.SYSPROCESSMODE,
SYSHOSTNAME: res.SYSHOSTNAME,
SYSUSERID: res.SYSUSERID,
SYSHOSTINFOLONG: res.SYSHOSTINFOLONG,
SYSENCODING: res.SYSENCODING,
AUTOEXEC: res.AUTOEXEC,
ISADMIN: res.globvars[0].ISADMIN,
DC_ADMIN_GROUP: res.globvars[0].DC_ADMIN_GROUP
SYSSITE: res.adapterResponse.SYSSITE,
SYSSCPL: res.adapterResponse.SYSSCPL,
SYSTCPIPHOSTNAME: res.adapterResponse.SYSTCPIPHOSTNAME,
SYSVLONG: res.adapterResponse.SYSVLONG,
MEMSIZE: res.adapterResponse.MEMSIZE,
SYSPROCESSMODE: res.adapterResponse.SYSPROCESSMODE,
SYSHOSTNAME: res.adapterResponse.SYSHOSTNAME,
SYSUSERID: res.adapterResponse.SYSUSERID,
SYSHOSTINFOLONG: res.adapterResponse.SYSHOSTINFOLONG,
SYSENCODING: res.adapterResponse.SYSENCODING,
AUTOEXEC: res.adapterResponse.AUTOEXEC,
ISADMIN: res.adapterResponse.globvars[0].ISADMIN,
DC_ADMIN_GROUP: res.adapterResponse.globvars[0].DC_ADMIN_GROUP,
APP_LOC: dcAdapterSettings?.appLoc
}
let libs = res.sasdatasets
let libs = res.adapterResponse.sasdatasets
let libGroup: any = {}
let libsAndTables
let libraries
@ -153,7 +162,7 @@ export class AppService {
globals.editor.libsAndTables = libsAndTables
}
globals.xlmaps = res.xlmaps.map((xlmap: any) => ({
globals.xlmaps = res.adapterResponse.xlmaps.map((xlmap: any) => ({
id: xlmap[0],
description: xlmap[1],
targetDS: xlmap[2]
@ -162,9 +171,9 @@ export class AppService {
globals.editor.libraries = libraries
globals.editor.startupSet = true
globals.dcLib = res.globvars[0].DCLIB
globals.dcLib = res.adapterResponse.globvars[0].DCLIB
await this.licenceService.activation(res)
await this.licenceService.activation(res.adapterResponse)
})
.catch((err: any) => {
startupServiceError = true

View File

@ -18,6 +18,7 @@ import { isSpecialMissing } from '@sasjs/utils/input/validators'
import { Col } from '../shared/dc-validator/models/col.model'
import { get } from 'lodash-es'
import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.model'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Injectable()
export class SasStoreService {
@ -58,12 +59,10 @@ export class SasStoreService {
) {
const tables: any = {}
tables[tableName] = [tableData]
const res: EditorsGetDataSASResponse = await this.sasService.request(
program,
tables
)
const res: RequestWrapperResponse<EditorsGetDataSASResponse> =
await this.sasService.request(program, tables)
const response: EditorsGetDataServiceResponse = {
data: res,
data: res.adapterResponse,
libds: libds
}
return response
@ -84,8 +83,9 @@ export class SasStoreService {
tableName: string,
program: string,
$dataFormats: $DataFormats | null,
suppressErrorSuccessMessages?: boolean
): Promise<EditorsStageDataSASResponse> {
suppressErrorSuccessMessages?: boolean,
adapterConfig?: any
): Promise<RequestWrapperResponse<EditorsStageDataSASResponse>> {
// add sp as third argument of createData call
let tables: any = {
@ -103,7 +103,7 @@ export class SasStoreService {
let res = await this.sasService.request<EditorsStageDataSASResponse>(
program,
tables,
null,
adapterConfig,
{
suppressErrorAbortModal: suppressErrorSuccessMessages,
suppressSuccessAbortModal: suppressErrorSuccessMessages
@ -127,8 +127,7 @@ export class SasStoreService {
) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request(program, tables)).adapterResponse
}
/**
@ -152,8 +151,8 @@ export class SasStoreService {
* @returns All submits
*/
public async getSubmitts() {
let res: any = await this.sasService.request('editors/getsubmits', null)
return res
return (await this.sasService.request('editors/getsubmits', null))
.adapterResponse
}
/**
@ -161,7 +160,8 @@ export class SasStoreService {
* @returns All libraries
*/
public async viewLibs() {
return this.sasService.request('public/viewlibs', null)
return (await this.sasService.request('public/viewlibs', null))
.adapterResponse
}
public async refreshLibInfo(libref: string) {
@ -169,30 +169,22 @@ export class SasStoreService {
lib2refresh: [{ libref }]
}
return this.sasService.request('public/refreshlibinfo', data)
}
public async versionHistory(libDataset: any) {
const data = { iwant: [{ LIBDS: libDataset }] }
let res: any = await this.sasService.request(
'public/getversionhistory',
data
)
return res
return (await this.sasService.request('public/refreshlibinfo', data))
.adapterResponse
}
public async viewTables(lib: any) {
let tables = { SASControlTable: [{ MPLIB: lib }] }
let res: any = await this.sasService.request('public/viewtables', tables)
return res
return (await this.sasService.request('public/viewtables', tables))
.adapterResponse
}
public async viewData(libDataset: any, filter_pk: any) {
let tables = {
SASControlTable: [{ LIBDS: libDataset, FILTER_RK: filter_pk }]
}
let res: any = await this.sasService.request('public/viewdata', tables)
return res
return (await this.sasService.request('public/viewdata', tables))
.adapterResponse
}
public async viewDataSearch(
@ -213,41 +205,36 @@ export class SasStoreService {
}
]
}
let res: any = await this.sasService.request('public/viewdata', tables)
return res
return (await this.sasService.request('public/viewdata', tables))
.adapterResponse
}
public async getXLMapRules(id: string) {
const tables = {
getxlmaps_in: [{ XLMAP_ID: id }]
}
const res: any = await this.sasService.request('editors/getxlmaps', tables)
return res
}
public async getDetails(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request('editors/getxlmaps', tables))
.adapterResponse
}
public async showDiffs(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables, {
return (
await this.sasService.request(program, tables, {
useComputeApi: false
})
return res
).adapterResponse
}
public async rejecting(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables, {
return (
await this.sasService.request(program, tables, {
useComputeApi: false
})
return res
).adapterResponse
}
public async approveTable(
@ -258,15 +245,13 @@ export class SasStoreService {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request(program, tables)).adapterResponse
}
public async getHistory(tableData: any, tableName: string, program: string) {
let tables: any = {}
tables[tableName] = [tableData]
let res: any = await this.sasService.request(program, tables)
return res
return (await this.sasService.request(program, tables)).adapterResponse
}
setQueryVariables(dataset: string, cols: any) {
@ -278,8 +263,8 @@ export class SasStoreService {
public async getChangeInfo(tableId: any) {
let obj = { TABLE: tableId }
let table = { SASControlTable: [obj] }
let res: any = await this.sasService.request('public/getchangeinfo', table)
return res
return (await this.sasService.request('public/getchangeinfo', table))
.adapterResponse
}
public async getQueryValues(
@ -304,13 +289,13 @@ export class SasStoreService {
tables.FILTERQUERY = filterQuery
}
let res: any = await this.sasService
return (
await this.sasService
.request('public/getcolvals', tables)
.catch((er: any) => {
throw er
})
return res
).adapterResponse
}
public async saveQuery(libds: string, filterQuery: QueryClause[]) {
@ -319,21 +304,17 @@ export class SasStoreService {
filterquery: filterQuery
}
let res: any = await this.sasService.request(
'public/validatefilter',
tables
)
const res = await this.sasService.request('public/validatefilter', tables)
this.filter.next(res)
return res
return res.adapterResponse
}
public async openTable(tableId: string) {
let tables = { iwant: [{ table_id: tableId }] }
let res: any = await this.sasService.request(
'auditors/getstagetable',
tables
)
return res
return (await this.sasService.request('auditors/getstagetable', tables))
.adapterResponse
}
public checkOperator(operator: any, value: any, type: string) {

View File

@ -12,8 +12,10 @@ import { ServerType } from '@sasjs/utils/types/serverType'
import { DcAdapterSettings } from '../models/DcAdapterSettings'
import { AppStoreService } from './app-store.service'
import { LoggerService } from './logger.service'
import { RequestWrapperOptions } from '../models/RequestWrapperOptions'
import { RequestWrapperOptions } from '../models/request-wrapper/RequestWrapperOptions'
import { ErrorBody } from '../models/ErrorBody'
import { UploadFileResponse } from '../models/UploadFile'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Injectable({
providedIn: 'root'
@ -95,14 +97,16 @@ export class SasService {
* @param config additional parameters to force eg. { debug: false }
* @param wrapperOptions used to provide options to the request wrapper function
* for example to suppress error or success abort modals after request is finished
* @returns
* @returns adapter response or an error. It will return the `log` as well.
* The log could be potentially be wrong if multiple requests happen because the log this
* function return is the last request in the Adapter Array for the given URL.
*/
public request<responseType = any>(
url: string,
data: any,
config?: any,
wrapperOptions?: RequestWrapperOptions
): Promise<responseType> {
): Promise<RequestWrapperResponse<responseType>> {
url = 'services/' + url
if (!wrapperOptions) wrapperOptions = {}
@ -117,9 +121,16 @@ export class SasService {
})
.then(
(res: any) => {
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === url)
if (res.login === false) {
this.shouldLogin.next(true)
reject(false)
reject({
adapterResponse: false,
log: sasRequest?.logFile
})
}
if (!this.userService.user && res.MF_GETUSER) {
@ -141,7 +152,12 @@ export class SasService {
}
if (res.status === 404) {
reject({ MESSAGE: res.body || 'SAS Responded with error' })
reject({
adapterResponse: {
MESSAGE: res.body || 'SAS Responded with error'
},
log: sasRequest?.logFile
})
}
if (typeof res.sasjsAbort !== 'undefined') {
@ -157,7 +173,12 @@ export class SasService {
this.eventService.startupDataLoaded()
this.router.navigateByUrl('/deploy')
reject({ error: abortMsg })
reject({
adapterResponse: {
error: abortMsg
},
log: sasRequest?.logFile
})
return
}
@ -174,14 +195,26 @@ export class SasService {
)
}
reject({ error: abortMsg })
reject({
adapterResponse: {
error: abortMsg
},
log: sasRequest?.logFile
})
}
resolve(res)
resolve({
adapterResponse: res,
log: sasRequest?.logFile
})
},
(err: { error: ErrorBody | undefined }) => {
console.error(err)
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === url)
if (err.error) {
let errorMessage: string | undefined = err.error.message
let log: string | undefined
@ -205,10 +238,18 @@ export class SasService {
'Request error'
)
}
reject({ error: errorMessage })
reject({
adapterResponse: {
error: errorMessage
},
log: sasRequest?.logFile
})
}
reject(err)
reject({
adapterResponse: err,
log: sasRequest?.logFile
})
}
)
})
@ -222,8 +263,35 @@ export class SasService {
* @param params Aditional parameters eg. { debug: false }
* @returns HTTP Response
*/
public uploadFile(sasService: string, files: UploadFile[], params: any) {
return this.sasjsAdapter.uploadFile(sasService, files, params)
public uploadFile(
sasService: string,
files: UploadFile[],
params: any
): Promise<UploadFileResponse> {
return new Promise((resolve, reject) => {
this.sasjsAdapter.uploadFile(sasService, files, params).then(
(res) => {
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === 'services/editors/loadfile')
resolve({
adapterResponse: res,
log: sasRequest?.logFile
})
},
(err) => {
const sasRequest = this.sasjsAdapter
.getSasRequests()
.find((rq) => rq.serviceLink === 'services/editors/loadfile')
reject({
response: err,
log: sasRequest?.logFile
})
}
)
})
}
public async login(username: string, password: string) {

View File

@ -1,70 +1,13 @@
import { Injectable } from '@angular/core'
import * as XLSX from '@sheet/crypto'
import {
ExcelPasswordModalService,
Result
} from '../shared/excel-password-modal/excel-password-modal.service'
import { ExcelPasswordModalService } from '../shared/excel-password-modal/excel-password-modal.service'
import { EventService } from './event.service'
import { isSpecialMissing } from '@sasjs/utils/input/validators'
import {
dateFormat,
dateToUtcTime,
dateToTime
} from '../editor/utils/date.utils'
import {
excelDateToJSDate,
getMissingHeaders
} from '../editor/utils/grid.utils'
import { isStringNumber, isStringDecimal } from '../editor/utils/types.utils'
import SheetInfo from '../models/SheetInfo'
import { blobToFile } from '../xlmap/utils/file.utils'
import { ExcelRule } from '../models/TableData'
import { DcValidator } from '../shared/dc-validator/dc-validator'
import { LicenceService } from './licence.service'
import { FileUploadEncoding } from '../models/FileUploadEncoding'
import { FileUploader } from '../models/FileUploader.class'
/**
* Used in combination with buffer
*/
const iconv = require('iconv-lite')
/**
* In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
* Which will be send to backend
*/
const Buffer = require('buffer/').Buffer
type AOA = any[][]
export interface ParseParams {
file: File
dcValidator: DcValidator
/**
* Parse function will manipulate and return the uploader array which can be provided with files already in the queue
* Otherwise new empty instance will be created.
*/
uploader?: FileUploader
headerPks: string[]
headerArray: string[]
headerShow: string[]
timeHeaders: string[]
dateHeaders: string[]
dateTimeHeaders: string[]
xlRules: ExcelRule[]
encoding?: FileUploadEncoding
}
export interface ParseResult {
/**
* In case of CSV file, won't be returned
*/
data?: any[]
/**
* In case of CSV file, won't be returned
*/
headerShow?: string[]
rangeSheetRes?: SheetInfo
uploader: FileUploader
}
import { SpreadsheetUtil } from '../shared/spreadsheet-util/spreadsheet-util'
import { ParseParams } from '../models/ParseParams.interface'
import { ParseResult } from '../models/ParseResult.interface'
import { OpenOptions } from '../shared/excel-password-modal/models/options.interface'
import { Result } from '../shared/excel-password-modal/models/result.interface'
import * as XLSX from '@sheet/crypto'
@Injectable({
providedIn: 'root'
@ -78,90 +21,93 @@ export class SpreadsheetService {
private licenceService: LicenceService
) {}
/**
* Parses attached file and searches fo the matching data
*
* @param parseParams params required for parsing the file
* @param onParseStateChange callback used to inform about parsing state
* so the user of the function can update the UI with latest info
* @param onTableFoundEvent callback fired when table range is found in the file
*
* @returns parsed list of files to upload and JSON data ready for HOT usage
*/
public parseExcelFile(
parseParams: ParseParams,
onParseStateChange?: (uploadState: string) => void,
onTableFoundEvent?: (info: string) => void
): Promise<ParseResult | undefined> {
return new Promise((resolve, reject) => {
let data: any[] = []
const uploader: FileUploader = parseParams.uploader || new FileUploader()
const spreadSheetUtil = new SpreadsheetUtil({
licenceState: this.licenceState
})
const file: File = parseParams.file
const filename = file.name
if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
if (onParseStateChange)
onParseStateChange(`Loading ${filename} into the browser`)
let foundData = {
sheet: ''
return spreadSheetUtil.parseExcelFile(
parseParams,
this.promptExcelPassword,
onParseStateChange,
onTableFoundEvent
)
}
let fileType = filename.slice(
filename.lastIndexOf('.') + 1,
filename.lastIndexOf('.') + 4
)
/**
* Read the file minimally just to get the sheet names, not reading full file
* to help boost the performance
*
* @returns sheet names in string array
*/
public async parseExcelSheetNames(file: File): Promise<{
sheetNames: string[]
password?: string
}> {
return new Promise((resolve, reject) => {
const reader = new FileReader()
if (!file) {
console.warn('file is missing')
return resolve({ sheetNames: [] })
}
reader.onload = async (event: ProgressEvent<FileReader>) => {
if (!event?.target) {
console.warn('File reader event.target is missing')
return
}
if (fileType.toLowerCase() === 'xls') {
let reader: FileReader = new FileReader()
const self = this
reader.onload = async (theFile: any) => {
/* read workbook */
const bstr = this.toBstr(theFile.target.result)
let wb: XLSX.WorkBook | undefined = undefined
let fileUnlocking: boolean = false
const xlsxOptions: XLSX.ParsingOptions = {
type: 'binary',
cellDates: false,
cellFormula: true,
cellStyles: true,
cellNF: false,
cellText: false
}
let password: string | undefined
const data = event.target.result
try {
wb = XLSX.read(bstr, {
...xlsxOptions
wb = XLSX.read(data, {
// Load file minimally to parse sheets
bookSheets: true,
type: 'binary'
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
fileUnlocking = true
let passwordError = false
while (fileUnlocking) {
const password = await this.promptExcelPassword()
password = await this.promptExcelPassword({
error: passwordError
})
if (password) {
try {
wb = XLSX.read(bstr, {
...xlsxOptions,
wb = XLSX.read(data, {
// Load file minimally to parse sheets
bookSheets: true,
type: 'binary',
password: password
})
fileUnlocking = false
passwordError = false
} catch (err: any) {
this.excelPasswordModalService.open({
error: true
})
passwordError = true
if (!err.message.toLowerCase().includes('password')) {
fileUnlocking = false
}
}
if (!password)
return reject('Invalid password, failed to decrypt the file')
} else {
fileUnlocking = false
return reject('No password provided')
}
}
} else {
@ -169,738 +115,45 @@ export class SpreadsheetService {
}
}
if (!wb) {
return reject('No workbook found.')
}
if (!wb) return reject('Error parsing the workbook')
/* save data */
let isComplete: boolean = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
let csvArrayData: any[] = []
const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
wb,
parseParams
)
missingHeaders = rangeSheetRes.missingHeaders
if (rangeSheetRes.foundData) {
isComplete = true
csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
if (onParseStateChange)
onParseStateChange(
`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
)
let startAddress = ''
let endAddress = ''
for (
let row = rangeSheetRes.startRow;
row < rangeSheetRes.endRow;
++row
) {
const arr: any[] = []
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
const addr = XLSX.utils.encode_cell({
r: rangeSheetRes.rangeStartRow + row,
c: rangeSheetRes.rangeStartCol + col
})
let cell
if (!ws[addr]) {
cell = { v: '' }
} else {
cell = ws[addr]
}
if (startAddress === '' && ws[addr]) startAddress = addr
endAddress = addr
arr.push(cell)
})
// If we found at least one non empty value it means it is not empty row
// othervise, it is empty row
let arrNonEmptyValue = arr.find((x) => x.v !== '')
if (arrNonEmptyValue) csvArrayData.push(arr)
}
rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
if (onTableFoundEvent)
onTableFoundEvent(
`Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
)
} else {
missingHeaders = rangeSheetRes.missingHeaders
}
if (missingHeaders.length > 0) {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
let abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
uploader.queue.pop()
return reject(abortMsg)
}
// If first row is empty, that means no data has been found
if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
let abortMsg = 'No relevant data found in File !'
uploader.queue.pop()
return reject(abortMsg)
}
if (
parseParams.dateTimeHeaders.length > 0 ||
parseParams.dateHeaders.length > 0 ||
parseParams.timeHeaders.length > 0
) {
csvArrayData = this.updateDateTimeCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (parseParams.xlRules.length > 0) {
csvArrayData = this.updateXLRuleCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (!isComplete) {
let abortMsg = ''
if (missingHeaders.length === 0) {
abortMsg = 'No relevant data found in File !'
} else {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
}
// abort message is fired, return undefined
uploader.queue.pop()
return reject(abortMsg)
} else {
parseParams.headerShow = csvArrayHeaders
csvArrayData = csvArrayData.map((row: any) =>
row.map((col: any) => (col.t === 'n' ? col.v : col.w))
)
csvArrayData = csvArrayData.map((row: any) => {
return row.map((col: any, index: number) => {
if (!col && col !== 0) col = ''
/**
* Keeping this for the reference
* Code below used to convert JSON to CSV
* now the XLSX is converting to CSV
*/
// if (isNaN(col)) {
// // Match and replace the double quotes, ignore the first and last char
// // in case they are double quotes already
// col = col.replace(/(?<!^)"(?!$)/g, '""')
// if (col.search(/,/g) > -1 ||
// col.search(/\r|\n/g) > -1
// ) {
// // Missing quotes at the end
// if (col.search(/"$/g) < 0) {
// col = col + '"' // So we add them
// }
// // Missing quotes at the start
// if (col.search(/^"/g) < 0) {
// col = '"' + col // So we add them
// }
// }
// }
const colName = parseParams.headerShow[index]
const colRule = parseParams.dcValidator?.getRule(colName)
if (colRule?.type === 'numeric') {
if (isSpecialMissing(col) && !col.includes('.'))
col = '.' + col
}
return col
})
})
data = csvArrayData
// Apply licence rows limitation if exists, it is only affecting data
// which will be send to SAS
const strippedCsvArrayData = csvArrayData.slice(
0,
this.licenceState.value.submit_rows_limit
)
// To submit to sas service, we need clean version of CSV of file
// attached. XLSX will do the parsing and heavy lifting
// First we create worksheet of json (data we extracted)
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
skipHeader: true
})
// create CSV to be uploaded from worksheet
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
// Prepend headers
csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
// Blob from which CSV file will be created depending of the selected
// encoding
let blob: Blob
if (parseParams.encoding === 'WLATIN1') {
// WLATIN1
let encoded = iconv.decode(
Buffer.from(csvContentClean),
'CP-1252'
)
blob = new Blob([encoded], { type: 'application/csv' })
} else {
// UTF-8
blob = new Blob([csvContentClean], { type: 'application/csv' })
}
let newCSVFile: File = blobToFile(blob, filename + '.csv')
uploader.addToQueue([newCSVFile])
}
if (data.length === 0) {
return reject(
`Table in the file is empty. Data found on sheet: ${foundData.sheet}`
)
}
try {
const sheetNames = wb.SheetNames
return resolve({
uploader,
data,
rangeSheetRes,
headerShow: parseParams.headerShow
sheetNames: sheetNames,
password: password
})
} catch (e) {
console.error(e)
}
reader.readAsArrayBuffer(file)
} else if (fileType.toLowerCase() === 'csv') {
if (this.licenceState.value.submit_rows_limit !== Infinity) {
uploader.queue.pop()
return reject(
'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
)
}
if (parseParams.encoding === 'WLATIN1') {
let reader = new FileReader()
const self = this
// Closure to capture the file information.
reader.onload = (theFile: any) => {
let encoded = iconv.decode(
Buffer.from(theFile.target.result),
'CP-1252'
)
let blob = new Blob([encoded], { type: fileType })
let encodedFile: File = blobToFile(blob, filename)
uploader.queue.pop()
uploader.addToQueue([encodedFile])
reader.onerror = function (ex) {
console.log(ex)
}
resolve({
uploader
reader.readAsBinaryString(file)
})
}
reader.readAsArrayBuffer(file)
} else {
return resolve({
uploader
})
}
} else {
let abortMsg =
'Invalid file type "<b>' +
filename +
'</b>". Please upload csv or excel file.'
uploader.queue.pop()
return reject(abortMsg)
}
})
}
/**
* Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
* @param wb Excel workbook
* @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
*/
private getRangeAndSheet(
wb: XLSX.WorkBook,
parseParams: ParseParams
): SheetInfo {
let data = []
let rangeStartRow: number = 0
let rangeStartCol: number = 0
let startRow: number = -1
let endRow: number = -1
let sheetName: string = ''
let isComplete = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
wb.SheetNames.forEach((element: string) => {
// Checking for required data in each sheet in workbook/
if (isComplete) {
return
}
missingHeaders = []
sheetName = element
const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
data = <AOA>XLSX.utils.sheet_to_json(ws, {
header: 1,
blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
defval: ''
})
if (data.length <= 1) {
return
}
let tempArr: string[] = []
parseParams.headerArray.forEach(() => tempArr.push(''))
data.push(tempArr)
let foundHeaders = false
data.forEach((row: any, index: number) => {
if (isComplete) {
return
}
if (foundHeaders) {
let isDataEnd = true
let isPkNull = false
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
if (row[col] !== '' && row[col] !== undefined) {
isDataEnd = false
} else {
if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
isPkNull = true
}
}
})
if (isDataEnd || isPkNull) {
endRow = index
isComplete = true
} else {
if (startRow === -1) {
startRow = index
}
}
} else {
const rowLowerCase: string[] = row.map((x: any) =>
x.toString().toLowerCase()
)
// If in file there is no delete column, remove it from search of missing.
// This way delete column will be optional to provide in file
if (!rowLowerCase.includes('_____delete__this__record_____')) {
const deleteIndex = csvArrayHeadersLower.indexOf(
'_____delete__this__record_____'
)
if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
}
foundHeaders = true
csvArrayHeadersLower.forEach((x) => {
if (rowLowerCase.indexOf(x) === -1) {
foundHeaders = false
}
})
let result = []
result = this.findValidHeaders(
rowLowerCase,
csvArrayHeadersLower,
index,
sheetName,
parseParams
)
if (result[0] === false) {
foundHeaders = false
if (result[1].length > 0) {
result[1].forEach((data: string) => {
missingHeaders.push(data)
})
}
} else {
csvArrayHeadersMap = result[1]
}
}
})
if (isComplete) {
this.update_sheet_range(ws)
const worksheetSel = ws['!ref']
if (worksheetSel) {
const range = XLSX.utils.decode_range(ws['!ref'] || '')
rangeStartRow = range.s.r
rangeStartCol = range.s.c
}
}
})
// If start row is still -1 that means first row of found range is empty
if (startRow === -1) isComplete = false
const returnObj: SheetInfo = {
foundData: isComplete,
sheetName,
startRow,
endRow,
csvArrayHeadersMap,
missingHeaders,
rangeStartRow,
rangeStartCol
}
return returnObj
}
private findValidHeaders(
row: string[],
headers: string[],
rowNumber: number,
tabName: string,
parseParams: ParseParams
): Array<any> {
let headersFound = false
const missingErrorArray = []
let j = 0
while (j < row.length) {
if (headersFound) {
// return;
} else {
if (headers.indexOf(row[j]) !== -1) {
let breakIndex
let rowStart = 0
let rowEnd = 0
let arrStart = 0
let foundHeadersArray: string[] = []
let spaceBreak = false
for (let i = j; i < row.length; i++) {
if (
row[i] === '' ||
(foundHeadersArray.indexOf(row[i]) !== -1 &&
this.isColHeader(row[i], parseParams.headerArray))
) {
if (row[i] === '') {
spaceBreak = true
}
breakIndex = i
break
} else {
foundHeadersArray.push(row[i])
}
}
let tempArray: string[] = []
if (breakIndex !== undefined) {
tempArray = row.slice(j, breakIndex)
arrStart = j
rowEnd = breakIndex
if (spaceBreak) {
rowStart = j
j = breakIndex
} else {
rowStart = j
j = breakIndex - 1
}
} else {
tempArray = row.slice(j)
rowStart = j
arrStart = j
rowEnd = row.length
j = row.length
}
let foundHeaders = true
//We check if there are missing headers
headers.forEach((x) => {
if (tempArray.indexOf(x) === -1) {
foundHeaders = false
}
})
if (foundHeaders) {
headersFound = true
let mapHeaders: any[] = headers
let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
map[obj] = -1
return map
}, {})
let temp = row.slice(rowStart, rowEnd)
headers.forEach((x) => {
csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
})
return [true, csvArrayHeadersMap]
} else {
let missingHeaders = getMissingHeaders(tempArray, headers)
let missingMessage = '<b>TAB(' + tabName + ')</b>'
missingErrorArray.push([
missingMessage +
' - ' +
missingHeaders[1].join(',') +
' ( missing ' +
missingHeaders[0].join(',') +
' )',
missingHeaders[1].length
])
}
}
}
j++
}
return [false, missingErrorArray]
}
private isColHeader(col: string, headerArray: string[]) {
return headerArray.indexOf(col.toUpperCase()) > -1
}
/**
* Function that updates the !ref range value provided in official docs.
* @param ws worksheet to be updated
*/
private update_sheet_range(ws: XLSX.WorkSheet) {
const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
Object.keys(ws)
.filter(function (x) {
return x.charAt(0) != '!'
})
.map(XLSX.utils.decode_cell)
.forEach(function (x: any) {
range.s.c = Math.min(range.s.c, x.c)
range.s.r = Math.min(range.s.r, x.r)
range.e.c = Math.max(range.e.c, x.c)
range.e.r = Math.max(range.e.r, x.r)
})
ws['!ref'] = XLSX.utils.encode_range(range)
public bytesToMB(size: number): number {
return parseFloat((size / (1024 * 1024)).toFixed(2))
}
/**
* When excel is password protected we will display the password prompt for user to type password in.
* @returns Password user input or undefined if discarded by user
*/
private promptExcelPassword(): Promise<string | undefined> {
private promptExcelPassword = (
options?: OpenOptions
): Promise<string | undefined> => {
return new Promise((resolve, reject) => {
this.excelPasswordModalService.open().subscribe((result: Result) => {
this.excelPasswordModalService
.open(options)
.subscribe((result: Result) => {
resolve(result.password)
})
})
}
private updateDateTimeCols(
headers: any,
data: any,
parseParams: ParseParams
) {
if (parseParams.dateHeaders.length > 0) {
const dateCols: number[] = []
parseParams.dateHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
dateCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
} else {
if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date)
}
}
}
row[element] = obj
})
})
}
if (parseParams.timeHeaders.length > 0) {
let timeCols: number[] = []
parseParams.timeHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
timeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
timeCols.forEach((element) => {
const obj = row[element]
if (
isStringNumber(obj.v) ||
isStringDecimal(obj.v) ||
obj.v.includes('E-')
) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateToUtcTime(date)
}
row[element] = obj
})
})
}
if (parseParams.dateTimeHeaders.length > 0) {
let dateTimeCols: number[] = []
parseParams.dateTimeHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
dateTimeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateTimeCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
} else {
if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
let str = obj.v.substring(0, obj.v.indexOf(':'))
str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
obj.v = str
}
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date) + ' ' + dateToTime(date)
}
}
row[element] = obj
})
})
}
return data
}
private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
if (parseParams.xlRules.length > 0) {
const xlRuleCols: any = []
parseParams.xlRules.forEach((element: any) => {
if (headers.indexOf(element.XL_COLUMN) !== -1) {
element['index'] = headers.indexOf(element.XL_COLUMN)
xlRuleCols.push(element)
}
})
data.forEach((row: any[]) => {
xlRuleCols.forEach((element: any) => {
const obj = row[element.index]
if (element.XL_RULE === 'FORMULA') {
if ('f' in obj) {
if (obj['t'] === 'n') {
obj['v'] = '=' + obj['f']
} else {
obj['w'] = '=' + obj['f']
}
}
}
row[element] = obj
})
})
}
return data
}
private toBstr(res: any) {
let bytes = new Uint8Array(res)
let binary = ''
let length = bytes.byteLength
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i])
}
return binary
}
}

View File

@ -1,4 +1,4 @@
<ng-container *ngIf="options$ | async as options">
<div *ngIf="options$ | async as options" class="excel-password-root">
<clr-modal
[clrModalOpen]="options.open"
[clrModalSize]="'md'"
@ -11,6 +11,7 @@
<p class="m-0">Please enter password:</p>
<input
#filePasswordInput
[(ngModel)]="passwordInput"
data-lpignore="true"
autocomplete="off"
id="filePasswordInput"
@ -19,9 +20,13 @@
/>
</div>
<div class="modal-footer">
<div>
<p *ngIf="options.error" class="m-0 color-red">
Sorry that didn't work, try again.
</p>
</div>
<div class="buttons">
<button type="button" class="btn btn-sm btn-outline" (click)="close()">
Cancel
</button>
@ -34,5 +39,6 @@
Unlock
</button>
</div>
</div>
</clr-modal>
</ng-container>
</div>

View File

@ -0,0 +1,18 @@
.excel-password-root {
::ng-deep {
.modal {
z-index: 1060;
}
}
}
.modal-footer {
display: flex;
align-items: center;
justify-content: space-between;
.buttons {
display: flex;
gap: 5px;
}
}

View File

@ -1,24 +1,24 @@
import { Component } from '@angular/core'
import { Observable } from 'rxjs'
import {
ExcelPasswordModalService,
Options
} from './excel-password-modal.service'
import { ExcelPasswordModalService } from './excel-password-modal.service'
import { Options } from './models/options.interface'
@Component({
selector: 'app-excel-password-modal',
styleUrls: ['./excel-password-modal.component.scss'],
templateUrl: './excel-password-modal.component.html'
})
export class ExcelPasswordModalComponent {
options$: Observable<Options>
options$: Observable<Options> = this.excelPasswordModalService.optionsSubject$
fileUnlockError: boolean = false
constructor(private excelPasswordModalService: ExcelPasswordModalService) {
this.options$ = this.excelPasswordModalService.optionsSubject$
}
passwordInput: string = ''
constructor(private excelPasswordModalService: ExcelPasswordModalService) {}
close(password?: string) {
this.passwordInput = ''
this.excelPasswordModalService.close(password)
}
}

View File

@ -1,17 +1,7 @@
import { Injectable } from '@angular/core'
import { Subject, Observable } from 'rxjs'
export interface Options extends OpenOptions {
open: boolean
}
export interface Result {
password: string | undefined
}
export interface OpenOptions {
error?: boolean
}
import { OpenOptions, Options } from './models/options.interface'
import { Result } from './models/result.interface'
@Injectable({
providedIn: 'root'

View File

@ -0,0 +1,7 @@
export interface OpenOptions {
error?: boolean
}
export interface Options extends OpenOptions {
open: boolean
}

View File

@ -0,0 +1,3 @@
export interface Result {
password: string | undefined
}

View File

@ -2,7 +2,6 @@ $sidebarWidth: 272px;
.clr-vertical-nav .nav-link.active {
background-color: transparent;
padding-left: 5px;
}
clr-vertical-nav {

View File

@ -0,0 +1,3 @@
export class RandomClass {
constructor(jo: string) {}
}

View File

@ -0,0 +1,954 @@
import { isSpecialMissing } from '@sasjs/utils/input/validators'
import {
dateFormat,
dateToUtcTime,
dateToTime
} from 'src/app/editor/utils/date.utils'
import {
getMissingHeaders,
excelDateToJSDate
} from 'src/app/editor/utils/grid.utils'
import {
isStringNumber,
isStringDecimal
} from 'src/app/editor/utils/types.utils'
import { FileUploader } from 'src/app/models/FileUploader.class'
import SheetInfo from 'src/app/models/SheetInfo'
import { blobToFile } from 'src/app/xlmap/utils/file.utils'
import * as XLSX from '@sheet/crypto'
import { LicenceState } from 'src/app/models/LicenceState'
import { BehaviorSubject } from 'rxjs'
import { ParseParams } from 'src/app/models/ParseParams.interface'
import { ParseResult } from 'src/app/models/ParseResult.interface'
import { OpenOptions } from '../excel-password-modal/models/options.interface'
/**
* Used in combination with buffer
*/
import * as iconv from 'iconv-lite'
/**
* In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
* Which will be send to backend
*/
import { Buffer } from 'buffer'
type AOA = any[][]
export interface ConstructorParams {
licenceState: BehaviorSubject<LicenceState>
}
export class SpreadsheetUtil {
private licenceState: BehaviorSubject<LicenceState>
constructor(params: ConstructorParams) {
this.licenceState = params.licenceState
}
/**
* Parses attached file and searches fo the matching data
*
* @param parseParams params required for parsing the file
* @param onParseStateChange callback used to inform about parsing state
* so the user of the function can update the UI with latest info
* @param onTableFoundEvent callback fired when table range is found in the file
*
* @returns parsed list of files to upload and JSON data ready for HOT usage
*/
public parseExcelFile(
parseParams: ParseParams,
promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
onParseStateChange?: (uploadState: string) => void,
onTableFoundEvent?: (info: string) => void
): Promise<ParseResult | undefined> {
return new Promise((resolve, reject) => {
// If file size is bigger then 2 MB we need to load its bytes in chunks
const sizeInMB = this.bytesToMB(parseParams.file.size)
if (sizeInMB > 2) {
}
let data: any[] = []
const uploader: FileUploader = parseParams.uploader || new FileUploader()
const file: File = parseParams.file
const filename = file.name
if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
if (onParseStateChange)
onParseStateChange(`Loading ${filename} into the browser`)
let foundData = {
sheet: ''
}
let fileType = filename.slice(
filename.lastIndexOf('.') + 1,
filename.lastIndexOf('.') + 4
)
if (fileType.toLowerCase() === 'xls') {
let reader: FileReader = new FileReader()
const self = this
reader.onload = async (theFile: any) => {
/* read workbook */
const bstr = this.toBstr(theFile.target.result)
let wb: XLSX.WorkBook | undefined = undefined
let fileUnlocking: boolean = false
const xlsxOptions: XLSX.ParsingOptions = {
type: 'binary',
cellDates: false,
cellFormula: true,
cellStyles: true,
cellNF: false,
cellText: false,
password: parseParams.password
}
try {
wb = await this.xlsxRead(bstr, {
...xlsxOptions
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
fileUnlocking = true
let passwordError = false
while (fileUnlocking) {
const password = await promptExcelPassword({
error: passwordError
})
if (password) {
try {
wb = await this.xlsxRead(bstr, {
...xlsxOptions,
password: password
})
fileUnlocking = false
passwordError = false
} catch (err: any) {
passwordError = true
if (!err.message.toLowerCase().includes('password')) {
fileUnlocking = false
}
}
} else {
fileUnlocking = false
}
}
} else {
return reject('Error reading the file')
}
}
if (!wb) {
return reject('No workbook found.')
}
/* save data */
let isComplete: boolean = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
let csvArrayData: any[] = []
const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
wb,
parseParams
)
missingHeaders = rangeSheetRes.missingHeaders
if (rangeSheetRes.foundData) {
isComplete = true
csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
if (onParseStateChange)
onParseStateChange(
`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
)
let startAddress = ''
let endAddress = ''
for (
let row = rangeSheetRes.startRow;
row < rangeSheetRes.endRow;
++row
) {
const arr: any[] = []
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
const addr = XLSX.utils.encode_cell({
r: rangeSheetRes.rangeStartRow + row,
c: rangeSheetRes.rangeStartCol + col
})
let cell
if (!ws[addr]) {
cell = { v: '' }
} else {
cell = ws[addr]
}
if (startAddress === '' && ws[addr]) startAddress = addr
endAddress = addr
arr.push(cell)
})
// If we found at least one non empty value it means it is not empty row
// othervise, it is empty row
let arrNonEmptyValue = arr.find((x) => x.v !== '')
if (arrNonEmptyValue) csvArrayData.push(arr)
}
rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
if (onTableFoundEvent)
onTableFoundEvent(
`Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
)
} else {
missingHeaders = rangeSheetRes.missingHeaders
}
if (missingHeaders.length > 0) {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
let abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
uploader.queue.pop()
return reject(abortMsg)
}
// If first row is empty, that means no data has been found
if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
let abortMsg = 'No relevant data found in File !'
uploader.queue.pop()
return reject(abortMsg)
}
if (
parseParams.dateTimeHeaders.length > 0 ||
parseParams.dateHeaders.length > 0 ||
parseParams.timeHeaders.length > 0
) {
csvArrayData = this.updateDateTimeCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (parseParams.xlRules.length > 0) {
csvArrayData = this.updateXLRuleCols(
csvArrayHeaders,
csvArrayData,
parseParams
)
}
if (!isComplete) {
let abortMsg = ''
if (missingHeaders.length === 0) {
abortMsg = 'No relevant data found in File !'
} else {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
}
// abort message is fired, return undefined
uploader.queue.pop()
return reject(abortMsg)
} else {
parseParams.headerShow = csvArrayHeaders
csvArrayData = csvArrayData.map((row: any) =>
row.map((col: any) => (col.t === 'n' ? col.v : col.w))
)
csvArrayData = csvArrayData.map((row: any) => {
return row.map((col: any, index: number) => {
if (!col && col !== 0) col = ''
/**
* Keeping this for the reference
* Code below used to convert JSON to CSV
* now the XLSX is converting to CSV
*/
// if (isNaN(col)) {
// // Match and replace the double quotes, ignore the first and last char
// // in case they are double quotes already
// col = col.replace(/(?<!^)"(?!$)/g, '""')
// if (col.search(/,/g) > -1 ||
// col.search(/\r|\n/g) > -1
// ) {
// // Missing quotes at the end
// if (col.search(/"$/g) < 0) {
// col = col + '"' // So we add them
// }
// // Missing quotes at the start
// if (col.search(/^"/g) < 0) {
// col = '"' + col // So we add them
// }
// }
// }
const colName = parseParams.headerShow[index]
const colRule = parseParams.dcValidator?.getRule(colName)
if (colRule?.type === 'numeric') {
if (isSpecialMissing(col) && !col.includes('.'))
col = '.' + col
}
return col
})
})
data = csvArrayData
// Apply licence rows limitation if exists, it is only affecting data
// which will be send to SAS
const strippedCsvArrayData = csvArrayData.slice(
0,
this.licenceState.value.submit_rows_limit
)
// To submit to sas service, we need clean version of CSV of file
// attached. XLSX will do the parsing and heavy lifting
// First we create worksheet of json (data we extracted)
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
skipHeader: true
})
// create CSV to be uploaded from worksheet
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
// Prepend headers
csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
// Blob from which CSV file will be created depending of the selected
// encoding
let blob: Blob
if (parseParams.encoding === 'WLATIN1') {
// WLATIN1
let encoded = iconv.decode(
Buffer.from(csvContentClean),
'CP-1252'
)
blob = new Blob([encoded], { type: 'application/csv' })
} else {
// UTF-8
blob = new Blob([csvContentClean], { type: 'application/csv' })
}
let newCSVFile: File = blobToFile(blob, filename + '.csv')
uploader.addToQueue([newCSVFile])
}
if (data.length === 0) {
return reject(
`Table in the file is empty. Data found on sheet: ${foundData.sheet}`
)
}
return resolve({
uploader,
data,
rangeSheetRes,
headerShow: parseParams.headerShow
})
}
reader.readAsArrayBuffer(file)
} else if (fileType.toLowerCase() === 'csv') {
if (this.licenceState.value.submit_rows_limit !== Infinity) {
uploader.queue.pop()
return reject(
'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
)
}
if (parseParams.encoding === 'WLATIN1') {
let reader = new FileReader()
const self = this
// Closure to capture the file information.
reader.onload = (theFile: any) => {
let encoded = iconv.decode(
Buffer.from(theFile.target.result),
'CP-1252'
)
let blob = new Blob([encoded], { type: fileType })
let encodedFile: File = blobToFile(blob, filename)
uploader.queue.pop()
uploader.addToQueue([encodedFile])
resolve({
uploader
})
}
reader.readAsArrayBuffer(file)
} else {
return resolve({
uploader
})
}
} else {
let abortMsg =
'Invalid file type "<b>' +
filename +
'</b>". Please upload csv or excel file.'
uploader.queue.pop()
return reject(abortMsg)
}
})
}
public bytesToMB(size: number): number {
return parseFloat((size / (1024 * 1024)).toFixed(2))
}
/**
* XLSX Read wrapper which uses Web Worker to read the file and not block
* the UI while reading. It will allow reading bigger files.
* If worker fails, fallback is regular file read.
* @param data
* @param opts
* @returns
*/
private xlsxRead(
data: any,
opts?: XLSX.ParsingOptions | undefined
): Promise<XLSX.WorkBook> {
return new Promise((resolve, reject) => {
if (opts && opts.password) {
console.info('Not using worker to parse the XLSX - has password')
// At the moment worker can't use crypto version of SheetJS because of
// 'global not defined' issue
return resolve(XLSX.read(data, opts))
}
if (typeof Worker === 'undefined') {
console.info(
'Not using worker to parse the XLSX - no Worker available in this environment'
)
// Web workers are not supported in this environment.
// You should add a fallback so that your program still executes correctly.
return resolve(XLSX.read(data, opts))
}
// Ultimately use Web Worker to parse the excel
console.info('Using worker to parse the XLSX')
const worker = new Worker(
new URL('../../spreadsheet.worker', import.meta.url)
)
worker.onmessage = ({ data }) => {
if (data.event === 'reading_end') {
resolve(data.workbook)
} else if (data.error) {
reject(data.error)
} else {
console.info(
'Worker failed to parse the XLSX - fallback to non worker parsing'
)
// Fallback to reading without Worker
resolve(XLSX.read(data, opts))
}
}
worker.postMessage({
data,
opts
})
// Big timeout (10 minutes) in case Worker fails and no response
// and read the file with fallback method without worker
setTimeout(() => {
return resolve(XLSX.read(data, opts))
}, 600 * 1000) // 10 minutes
})
}
/**
* Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
* @param wb Excel workbook
* @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
*/
private getRangeAndSheet(
wb: XLSX.WorkBook,
parseParams: ParseParams
): SheetInfo {
let data = []
let rangeStartRow: number = 0
let rangeStartCol: number = 0
let startRow: number = -1
let endRow: number = -1
let sheetName: string = ''
let isComplete = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...parseParams.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
wb.SheetNames.forEach((element: string) => {
// Checking for required data in each sheet in workbook/
if (isComplete) {
return
}
missingHeaders = []
sheetName = element
const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
data = <AOA>XLSX.utils.sheet_to_json(ws, {
header: 1,
blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
defval: ''
})
if (data.length <= 1) {
return
}
let tempArr: string[] = []
parseParams.headerArray.forEach(() => tempArr.push(''))
data.push(tempArr)
let foundHeaders = false
data.forEach((row: any, index: number) => {
if (isComplete) {
return
}
if (foundHeaders) {
let isDataEnd = true
let isPkNull = false
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
if (row[col] !== '' && row[col] !== undefined) {
isDataEnd = false
} else {
if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
isPkNull = true
}
}
})
if (isDataEnd || isPkNull) {
endRow = index
isComplete = true
} else {
if (startRow === -1) {
startRow = index
}
}
} else {
const rowLowerCase: string[] = row.map((x: any) =>
x.toString().toLowerCase()
)
// If in file there is no delete column, remove it from search of missing.
// This way delete column will be optional to provide in file
if (!rowLowerCase.includes('_____delete__this__record_____')) {
const deleteIndex = csvArrayHeadersLower.indexOf(
'_____delete__this__record_____'
)
if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
}
foundHeaders = true
csvArrayHeadersLower.forEach((x) => {
if (rowLowerCase.indexOf(x) === -1) {
foundHeaders = false
}
})
let result = []
result = this.findValidHeaders(
rowLowerCase,
csvArrayHeadersLower,
index,
sheetName,
parseParams
)
if (result[0] === false) {
foundHeaders = false
if (result[1].length > 0) {
result[1].forEach((data: string) => {
missingHeaders.push(data)
})
}
} else {
csvArrayHeadersMap = result[1]
}
}
})
if (isComplete) {
this.update_sheet_range(ws)
const worksheetSel = ws['!ref']
if (worksheetSel) {
const range = XLSX.utils.decode_range(ws['!ref'] || '')
rangeStartRow = range.s.r
rangeStartCol = range.s.c
}
}
})
// If start row is still -1 that means first row of found range is empty
if (startRow === -1) isComplete = false
const returnObj: SheetInfo = {
foundData: isComplete,
sheetName,
startRow,
endRow,
csvArrayHeadersMap,
missingHeaders,
rangeStartRow,
rangeStartCol
}
return returnObj
}
private findValidHeaders(
row: string[],
headers: string[],
rowNumber: number,
tabName: string,
parseParams: ParseParams
): Array<any> {
let headersFound = false
const missingErrorArray = []
let j = 0
while (j < row.length) {
if (headersFound) {
// return;
} else {
if (headers.indexOf(row[j]) !== -1) {
let breakIndex
let rowStart = 0
let rowEnd = 0
let arrStart = 0
let foundHeadersArray: string[] = []
let spaceBreak = false
for (let i = j; i < row.length; i++) {
if (
row[i] === '' ||
(foundHeadersArray.indexOf(row[i]) !== -1 &&
this.isColHeader(row[i], parseParams.headerArray))
) {
if (row[i] === '') {
spaceBreak = true
}
breakIndex = i
break
} else {
foundHeadersArray.push(row[i])
}
}
let tempArray: string[] = []
if (breakIndex !== undefined) {
tempArray = row.slice(j, breakIndex)
arrStart = j
rowEnd = breakIndex
if (spaceBreak) {
rowStart = j
j = breakIndex
} else {
rowStart = j
j = breakIndex - 1
}
} else {
tempArray = row.slice(j)
rowStart = j
arrStart = j
rowEnd = row.length
j = row.length
}
let foundHeaders = true
//We check if there are missing headers
headers.forEach((x) => {
if (tempArray.indexOf(x) === -1) {
foundHeaders = false
}
})
if (foundHeaders) {
headersFound = true
let mapHeaders: any[] = headers
let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
map[obj] = -1
return map
}, {})
let temp = row.slice(rowStart, rowEnd)
headers.forEach((x) => {
csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
})
return [true, csvArrayHeadersMap]
} else {
let missingHeaders = getMissingHeaders(tempArray, headers)
let missingMessage = '<b>TAB(' + tabName + ')</b>'
missingErrorArray.push([
missingMessage +
' - ' +
missingHeaders[1].join(',') +
' ( missing ' +
missingHeaders[0].join(',') +
' )',
missingHeaders[1].length
])
}
}
}
j++
}
return [false, missingErrorArray]
}
private isColHeader(col: string, headerArray: string[]) {
return headerArray.indexOf(col.toUpperCase()) > -1
}
/**
* Function that updates the !ref range value provided in official docs.
* @param ws worksheet to be updated
*/
private update_sheet_range(ws: XLSX.WorkSheet) {
const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
Object.keys(ws)
.filter(function (x) {
return x.charAt(0) != '!'
})
.map(XLSX.utils.decode_cell)
.forEach(function (x: any) {
range.s.c = Math.min(range.s.c, x.c)
range.s.r = Math.min(range.s.r, x.r)
range.e.c = Math.max(range.e.c, x.c)
range.e.r = Math.max(range.e.r, x.r)
})
ws['!ref'] = XLSX.utils.encode_range(range)
}
/**
* When excel is password protected we will display the password prompt for user to type password in.
* @returns Password user input or undefined if discarded by user
*/
// private promptExcelPassword(): Promise<string | undefined> {
// return new Promise((resolve, reject) => {
// this.excelPasswordModalService.open().subscribe((result: Result) => {
// resolve(result.password)
// })
// })
// }
private updateDateTimeCols(
headers: any,
data: any,
parseParams: ParseParams
) {
if (parseParams.dateHeaders.length > 0) {
const dateCols: number[] = []
parseParams.dateHeaders.forEach((element: any) => {
if (headers.indexOf(element) !== -1) {
dateCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
} else {
if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date)
}
}
}
row[element] = obj
})
})
}
if (parseParams.timeHeaders.length > 0) {
let timeCols: number[] = []
parseParams.timeHeaders.forEach((element: any) => {
if (headers.indexOf(element) !== -1) {
timeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
timeCols.forEach((element) => {
const obj = row[element]
if (
isStringNumber(obj.v) ||
isStringDecimal(obj.v) ||
obj.v.includes('E-')
) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateToUtcTime(date)
}
row[element] = obj
})
})
}
if (parseParams.dateTimeHeaders.length > 0) {
let dateTimeCols: number[] = []
parseParams.dateTimeHeaders.forEach((element: any) => {
if (headers.indexOf(element) !== -1) {
dateTimeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateTimeCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
} else {
if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
let str = obj.v.substring(0, obj.v.indexOf(':'))
str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
obj.v = str
}
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date) + ' ' + dateToTime(date)
}
}
row[element] = obj
})
})
}
return data
}
private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
if (parseParams.xlRules.length > 0) {
const xlRuleCols: any = []
parseParams.xlRules.forEach((element: any) => {
if (headers.indexOf(element.XL_COLUMN) !== -1) {
element['index'] = headers.indexOf(element.XL_COLUMN)
xlRuleCols.push(element)
}
})
data.forEach((row: any[]) => {
xlRuleCols.forEach((element: any) => {
const obj = row[element.index]
if (element.XL_RULE === 'FORMULA') {
if ('f' in obj) {
if (obj['t'] === 'n') {
obj['v'] = '=' + obj['f']
} else {
obj['w'] = '=' + obj['f']
}
}
}
row[element] = obj
})
})
}
return data
}
private toBstr(res: any) {
let bytes = new Uint8Array(res)
let binary = ''
let length = bytes.byteLength
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i])
}
return binary
}
}

View File

@ -8,6 +8,7 @@ import {
import { SasService } from '../../services/sas.service'
import * as marked from 'marked'
import { EULA } from 'src/environments/_eula'
import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-terms',
@ -51,8 +52,12 @@ export class TermsComponent implements OnInit, AfterViewInit {
this.sasService
.request(`public/registeruser`, table)
.then((res: any) => {
if (res.return && res.return[0] && res.return[0].MSG === 'SUCCESS') {
.then((res: RequestWrapperResponse) => {
if (
res.adapterResponse.return &&
res.adapterResponse.return[0] &&
res.adapterResponse.return[0].MSG === 'SUCCESS'
) {
location.reload()
}
})

View File

@ -0,0 +1,30 @@
/// <reference lib="webworker" />
/**
* We use normal version of the XLSX (SheetJS)
* Because at the moment "@sheet/crypto" can't work in the Web Worker environment
* Because of the missing "global" variable.
*/
import * as XLSX from 'xlsx'
addEventListener('message', ({ data }) => {
const { data: xldata, opts: xlopts } = data as {
data: any
opts?: any
}
try {
const workbook = XLSX.read(xldata, xlopts)
postMessage({
event: 'reading_end',
workbook: workbook
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
postMessage({
error: err
})
}
}
})

View File

@ -8,6 +8,7 @@ import { HotTableInterface } from '../models/HotTable.interface'
import { LicenceService } from '../services/licence.service'
import { globals } from '../_globals'
import { EditorsRestoreServiceResponse } from '../models/sas/editors-restore.model'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-stage',
@ -180,10 +181,12 @@ export class StageComponent implements OnInit {
this.sasService
.request('editors/restore', data)
.then((res: EditorsRestoreServiceResponse) => {
if (res.restore_out) {
.then((res: RequestWrapperResponse<EditorsRestoreServiceResponse>) => {
if (res.adapterResponse.restore_out) {
this.route.navigate([`/stage`]).then(() => {
this.route.navigate([`/stage/${res.restore_out[0].LOADREF}`])
this.route.navigate([
`/stage/${res.adapterResponse.restore_out[0].LOADREF}`
])
})
}
})

View File

@ -1,15 +1,16 @@
export interface EnvironmentInfo {
SYSSITE: string
SYSSCPL: string
SYSTCPIPHOSTNAME: string
SYSVLONG: string
MEMSIZE: string
SYSPROCESSMODE: string
SYSHOSTNAME: string
SYSUSERID: string
SYSHOSTINFOLONG: string
SYSENCODING: string
AUTOEXEC: string
ISADMIN: number
DC_ADMIN_GROUP: string
SYSSITE?: string
SYSSCPL?: string
SYSTCPIPHOSTNAME?: string
SYSVLONG?: string
MEMSIZE?: string
SYSPROCESSMODE?: string
SYSHOSTNAME?: string
SYSUSERID?: string
SYSHOSTINFOLONG?: string
SYSENCODING?: string
AUTOEXEC?: string
ISADMIN?: number
DC_ADMIN_GROUP?: string
APP_LOC?: string
}

View File

@ -77,6 +77,10 @@
DC Admin Group:
<span class="dark">{{ environmentInfo?.DC_ADMIN_GROUP }}</span>
</p>
<p cds-text="label" class="m-0">
App Location:
<span class="dark">{{ environmentInfo?.APP_LOC }}</span>
</p>
</div>
</div>

View File

@ -9,6 +9,7 @@ import { AppInfo } from './models/app-info.model'
import { EnvironmentInfo } from './models/environment-info.model'
import { AppSettingsService } from '../services/app-settings.service'
import { AppSettings } from '../models/AppSettings'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-system',
@ -51,7 +52,7 @@ export class SystemComponent implements OnInit {
this.environmentInfo = this.appService.getEnvironmentInfo()
this.settings = this.appSettingsService.settings.value
if (this.environmentInfo) {
if (this.environmentInfo.AUTOEXEC) {
this.environmentInfo.AUTOEXEC = decodeURIComponent(
this.environmentInfo.AUTOEXEC
)
@ -83,12 +84,12 @@ export class SystemComponent implements OnInit {
this.sasService
.request('admin/refreshcatalog', null)
.then((res: any) => {
this.response = this.parseResponse(res)
.then((res: RequestWrapperResponse) => {
this.response = this.parseResponse(res.adapterResponse)
this.responseModal = true
})
.catch((err: any) => {
this.response = this.parseResponse(err)
this.response = this.parseResponse(err.adapterResponse)
this.responseModal = true
})
.finally(() => {
@ -101,12 +102,12 @@ export class SystemComponent implements OnInit {
this.sasService
.request('admin/refreshtablelineage', null)
.then((res: any) => {
this.response = this.parseResponse(res)
.then((res: RequestWrapperResponse) => {
this.response = this.parseResponse(res.adapterResponse)
this.responseModal = true
})
.catch((err: any) => {
this.response = this.parseResponse(err)
this.response = this.parseResponse(err.adapterResponse)
this.responseModal = true
})
.finally(() => {

View File

@ -6,6 +6,7 @@ import { Location } from '@angular/common'
import { SasService } from '../services/sas.service'
import { SASjsConfig } from '@sasjs/adapter'
import { ServerType } from '@sasjs/utils/types/serverType'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-user',
@ -89,10 +90,10 @@ export class UserComponent implements OnInit {
} else {
this.sasService
.request('usernav/usermembers', null)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
this.users = res.users
globals.usernav.userList = res.users
this.users = res.adapterResponse.users
globals.usernav.userList = res.adapterResponse.users
})
}
} else {
@ -141,18 +142,18 @@ export class UserComponent implements OnInit {
this.sasService
.request('usernav/usergroupsbymember', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
switch (this.serverType) {
case ServerType.Sas9: {
this.userInfo = res.info[0]
this.userEmails = res.emails
this.userEmailsCount = res.emails.length
this.userRoles = res.roles
this.userRolesCount = res.roles.length
this.userLogins = res.logins
this.userLoginsCount = res.logins.length
this.userInfo = res.adapterResponse.info[0]
this.userEmails = res.adapterResponse.emails
this.userEmailsCount = res.adapterResponse.emails.length
this.userRoles = res.adapterResponse.roles
this.userRolesCount = res.adapterResponse.roles.length
this.userLogins = res.adapterResponse.logins
this.userLoginsCount = res.adapterResponse.logins.length
break
}
@ -162,7 +163,7 @@ export class UserComponent implements OnInit {
(userAr) => userAr.URI === uri
)
} else {
const group = res.groups[0]
const group = res.adapterResponse.groups[0]
this.userInfo = {
URI: group.ID,
@ -175,9 +176,9 @@ export class UserComponent implements OnInit {
}
}
this.userData = res
this.userGroups = res.groups
this.userGroupsCount = res.groups.length
this.userData = res.adapterResponse
this.userGroups = res.adapterResponse.groups
this.userGroupsCount = res.adapterResponse.groups.length
})
}
}
@ -238,18 +239,18 @@ export class UserComponent implements OnInit {
let data = { iwant: [{ uri: user.URI }] }
this.sasService
.request('usernav/usergroupsbymember', data)
.then((res: any) => {
.then((res: RequestWrapperResponse) => {
this.loading = false
switch (this.serverType) {
case ServerType.Sas9: {
this.userInfo = res.info[0]
this.userEmails = res.emails
this.userEmailsCount = res.emails.length
this.userRoles = res.roles
this.userRolesCount = res.roles.length
this.userLogins = res.logins
this.userLoginsCount = res.logins.length
this.userInfo = res.adapterResponse.info[0]
this.userEmails = res.adapterResponse.emails
this.userEmailsCount = res.adapterResponse.emails.length
this.userRoles = res.adapterResponse.roles
this.userRolesCount = res.adapterResponse.roles.length
this.userLogins = res.adapterResponse.logins
this.userLoginsCount = res.adapterResponse.logins.length
break
}
@ -259,7 +260,7 @@ export class UserComponent implements OnInit {
(userAr) => userAr.URI === user.URI
)
} else {
const group = res.groups[0]
const group = res.adapterResponse.groups[0]
this.userInfo = {
URI: group.ID,
@ -272,9 +273,9 @@ export class UserComponent implements OnInit {
}
}
this.userData = res
this.userGroups = res.groups
this.userGroupsCount = res.groups.length
this.userData = res.adapterResponse
this.userGroups = res.adapterResponse.groups
this.userGroupsCount = res.adapterResponse.groups.length
})
}
}

View File

@ -37,6 +37,7 @@ import { DataFormat } from '../models/sas/common/DateFormat'
import { Libinfo } from '../models/sas/common/Libinfo'
import { LicenceService } from '../services/licence.service'
import { Location } from '@angular/common'
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
@Component({
selector: 'app-viewer',

View File

@ -22,6 +22,7 @@ import {
} from '../services'
import { getCellAddress, getFinishingCell } from './utils/xl.utils'
import { blobToFile, byteArrayToBinaryString } from './utils/file.utils'
import { UploadFileResponse } from '../models/UploadFile'
interface XLMapRule {
XLMAP_ID: string
@ -340,9 +341,9 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
.uploadFile(uploadUrl, filesToUpload, {
table: this.selectedXLMap.targetDS
})
.then((res: any) => {
if (res.sasjsAbort) {
const abortRes = res
.then((res: UploadFileResponse) => {
if (res.adapterResponse.sasjsAbort) {
const abortRes = res.adapterResponse
const abortMsg = abortRes.sasjsAbort[0].MSG
const macMsg = abortRes.sasjsAbort[0].MAC
@ -351,14 +352,14 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
SYSERRORTEXT: abortRes.SYSERRORTEXT,
MAC: macMsg
})
} else if (res.sasparams) {
const params = res.sasparams[0]
} else if (res.adapterResponse.sasparams) {
const params = res.adapterResponse.sasparams[0]
const tableId = params.DSID
this.router.navigateByUrl('/stage/' + tableId)
}
})
.catch((err: any) => {
this.eventService.catchResponseError('file upload', err)
this.eventService.catchResponseError('file upload', err.response)
})
.finally(() => {
this.status = Status.ReadyToSubmit

View File

@ -0,0 +1,15 @@
/* To learn more about this file see: https://angular.io/config/tsconfig. */
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "./out-tsc/worker",
"lib": [
"es2018",
"webworker"
],
"types": []
},
"include": [
"src/**/*.worker.ts"
]
}