dc/dc
dc/dc
3
1
Fork 0
Code Issues 16 Pull Requests 1 Actions Packages Projects Releases 35 Wiki Activity

feat(multi load): refactored range find function, unlocking excel with password is reusable
Some checks failed
Build / Build-and-ng-test (pull_request) Failing after 52s
Details

Browse Source
This commit is contained in:
Mihajlo Medjedovic 2024-06-13 12:59:26 +02:00
parent 3c62ff6913
commit eb7c44333c
29 changed files with 1573 additions and 905 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
client/package-lock.json generated
View File

@ -3428,9 +3428,9 @@
"optional": true
},
"node_modules/@cds/core": {
"version": "6.11.0",
"resolved": "https://registry.npmjs.org/@cds/core/-/core-6.11.0.tgz",
"integrity": "sha512-7tNZeLEfpcNdDN85jV9137zxy7euqBCtG8SRNX0E8XGMMODs7JpxvORahhwRGwd7geSbZpA70urNo+h5Y2Mqrg==",
"version": "6.12.0",
"resolved": "https://registry.npmjs.org/@cds/core/-/core-6.12.0.tgz",
"integrity": "sha512-eJpddpt4vx4s0EFi2kAvnRZz1h/JVZPqjkNXpDalBM61IqQp9g2ETGoa594t6BWe3cwGDrkiKuEB+FbAx+IEhw==",
"dependencies": {
"lit": "^2.1.3",
"ramda": "^0.29.0",
@ -4531,9 +4531,9 @@
"dev": true
},
"node_modules/@handsontable/angular": {
"version": "14.3.0",
"resolved": "https://registry.npmjs.org/@handsontable/angular/-/angular-14.3.0.tgz",
"integrity": "sha512-Tdea1fKgQY1DG1upr2I8AlzbYg+kvI4sw4FU/dwWCgKKUkArAT8bW2aOhZMh55CYLJksLFTgCtPSQQMuSeR39g==",
"version": "14.4.0",
"resolved": "https://registry.npmjs.org/@handsontable/angular/-/angular-14.4.0.tgz",
"integrity": "sha512-r984kb/tssJ0nR/uB2bimJWoi+PxoJb4A3RDg1PwAZdquYCPs1j1Yr66N2Z46Ua1Pc1IOnaruLg/kkOuTnPVGg==",
"optionalDependencies": {
"tslib": "^2.3.0"
},
@ -4584,6 +4584,7 @@
"version": "0.11.14",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz",
"integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==",
"deprecated": "Use @eslint/config-array instead",
"dev": true,
"dependencies": {
"@humanwhocodes/object-schema": "^2.0.2",
@ -4611,6 +4612,7 @@
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz",
"integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==",
"deprecated": "Use @eslint/object-schema instead",
"dev": true
},
"node_modules/@isaacs/cliui": {
@ -6238,9 +6240,9 @@
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="
},
"node_modules/@types/lodash": {
"version": "4.17.4",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.4.tgz",
"integrity": "sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ==",
"version": "4.17.5",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.5.tgz",
"integrity": "sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==",
"dev": true
},
"node_modules/@types/lodash-es": {
@ -7071,6 +7073,15 @@
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz",
"integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==",
"dev": true,
"peerDependencies": {
"acorn": "^8"
}
},
"node_modules/acorn-import-attributes": {
"version": "1.9.5",
"resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
"integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
"peerDependencies": {
"acorn": "^8"
}
@ -8050,9 +8061,9 @@
}
},
"node_modules/browserslist": {
"version": "4.23.0",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz",
"integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==",
"version": "4.23.1",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.1.tgz",
"integrity": "sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw==",
"funding": [
{
"type": "opencollective",
@ -8068,10 +8079,10 @@
}
],
"dependencies": {
"caniuse-lite": "^1.0.30001587",
"electron-to-chromium": "^1.4.668",
"caniuse-lite": "^1.0.30001629",
"electron-to-chromium": "^1.4.796",
"node-releases": "^2.0.14",
"update-browserslist-db": "^1.0.13"
"update-browserslist-db": "^1.0.16"
},
"bin": {
"browserslist": "cli.js"
@ -8224,9 +8235,9 @@
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001629",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001629.tgz",
"integrity": "sha512-c3dl911slnQhmxUIT4HhYzT7wnBK/XYpGnYLOj4nJBaRiw52Ibe7YxlDaAeRECvA786zCuExhxIUJ2K7nHMrBw==",
"version": "1.0.30001633",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001633.tgz",
"integrity": "sha512-6sT0yf/z5jqf8tISAgpJDrmwOpLsrpnyCdD/lOZKvKkkJK4Dn0X5i7KF7THEZhOq+30bmhwBlNEaqPUiHiKtZg==",
"funding": [
{
"type": "opencollective",
@ -10137,9 +10148,9 @@
}
},
"node_modules/electron-to-chromium": {
"version": "1.4.795",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.795.tgz",
"integrity": "sha512-hHo4lK/8wb4NUa+NJYSFyJ0xedNHiR6ylilDtb8NUW9d4dmBFmGiecYEKCEbti1wTNzbKXLfl4hPWEkAFbHYlw=="
"version": "1.4.802",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.802.tgz",
"integrity": "sha512-TnTMUATbgNdPXVSHsxvNVSG0uEd6cSZsANjm8c9HbvflZVVn1yTRcmVXYT1Ma95/ssB/Dcd30AHweH2TE+dNpA=="
},
"node_modules/elliptic": {
"version": "6.5.5",
@ -11509,9 +11520,9 @@
}
},
"node_modules/foreground-child": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
"integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.0.tgz",
"integrity": "sha512-CrWQNaEl1/6WeZoarcM9LHupTo3RpZO2Pdk1vktwzPiQTsJnAKJmm3TACKeG5UZbWDfaH2AbvYxzP96y0MT7fA==",
"dev": true,
"dependencies": {
"cross-spawn": "^7.0.0",
@ -12015,15 +12026,15 @@
}
},
"node_modules/handsontable": {
"version": "14.3.0",
"resolved": "https://registry.npmjs.org/handsontable/-/handsontable-14.3.0.tgz",
"integrity": "sha512-XRKivPK+DkxWTYr+H6ywwLhSn/B5WMj3F3whF7O+Mo6Edm35rF8ydOm6oxVuVWnkvvy4FLqch7wgQk2w6hC8VA==",
"version": "14.4.0",
"resolved": "https://registry.npmjs.org/handsontable/-/handsontable-14.4.0.tgz",
"integrity": "sha512-uTIIx5UgG+2I0flUDj8lxftJAXbBpbkIAFIKugvMW2bNHYNxaNMGAGZ/QFOeCXiYwxeztBATKxMkLpg7N+HrOg==",
"dependencies": {
"@handsontable/pikaday": "^1.0.0",
"@types/pikaday": "1.7.4",
"core-js": "^3.31.1",
"core-js": "^3.37.0",
"dompurify": "^2.1.1",
"moment": "2.29.4",
"moment": "2.30.1",
"numbro": "2.1.2",
"pikaday": "1.8.2"
},
@ -12041,14 +12052,6 @@
"url": "https://opencollective.com/core-js"
}
},
"node_modules/handsontable/node_modules/moment": {
"version": "2.29.4",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
"integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
"engines": {
"node": "*"
}
},
"node_modules/has-bigints": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz",
@ -13532,9 +13535,9 @@
}
},
"node_modules/jiti": {
"version": "1.21.3",
"resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.3.tgz",
"integrity": "sha512-uy2bNX5zQ+tESe+TiC7ilGRz8AtRGmnJH55NC5S0nSUjvvvM2hJHmefHErugGXN4pNv4Qx7vLsnNw9qJ9mtIsw==",
"version": "1.21.6",
"resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz",
"integrity": "sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==",
"dev": true,
"bin": {
"jiti": "bin/jiti.js"
@ -18348,9 +18351,9 @@
}
},
"node_modules/rfdc": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.1.tgz",
"integrity": "sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==",
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz",
"integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==",
"dev": true
},
"node_modules/rimraf": {
@ -20508,9 +20511,9 @@
}
},
"node_modules/uglify-js": {
"version": "3.17.4",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz",
"integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==",
"version": "3.18.0",
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.18.0.tgz",
"integrity": "sha512-SyVVbcNBCk0dzr9XL/R/ySrmYf0s372K6/hFklzgcp2lBFyXtw4I7BOdDjlLhE1aVqaI/SHWXWmYdlZxuyF38A==",
"dev": true,
"optional": true,
"bin": {
@ -20925,9 +20928,9 @@
}
},
"node_modules/webpack": {
"version": "5.91.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.91.0.tgz",
"integrity": "sha512-rzVwlLeBWHJbmgTC/8TvAcu5vpJNII+MelQpylD4jNERPwpBJOE2lEcko1zJX3QJeLjTTAnQxn/OJ8bjDzVQaw==",
"version": "5.92.0",
"resolved": "https://registry.npmjs.org/webpack/-/webpack-5.92.0.tgz",
"integrity": "sha512-Bsw2X39MYIgxouNATyVpCNVWBCuUwDgWtN78g6lSdPJRLaQ/PUVm/oXcaRAyY/sMFoKFQrsPeqvTizWtq7QPCA==",
"dependencies": {
"@types/eslint-scope": "^3.7.3",
"@types/estree": "^1.0.5",
@ -20935,10 +20938,10 @@
"@webassemblyjs/wasm-edit": "^1.12.1",
"@webassemblyjs/wasm-parser": "^1.12.1",
"acorn": "^8.7.1",
"acorn-import-assertions": "^1.9.0",
"acorn-import-attributes": "^1.9.5",
"browserslist": "^4.21.10",
"chrome-trace-event": "^1.0.2",
"enhanced-resolve": "^5.16.0",
"enhanced-resolve": "^5.17.0",
"es-module-lexer": "^1.2.1",
"eslint-scope": "5.1.1",
"events": "^3.2.0",

1
client/package.json
View File

@ -53,7 +53,6 @@
"@sasjs/utils": "^3.4.0",
"@sheet/crypto": "file:libraries/sheet-crypto.tgz",
"@types/d3-graphviz": "^2.6.7",
"@sheet/crypto": "file:libraries/sheet-crypto.tgz",
"@types/text-encoding": "0.0.35",
"base64-arraybuffer": "^0.2.0",
"buffer": "^5.4.3",

1
client/src/app/app.component.html
View File

@ -245,6 +245,7 @@
<app-alerts *ngIf="!errTop"></app-alerts>
<app-requests-modal [(opened)]="requestsModal"></app-requests-modal>
<app-excel-password-modal></app-excel-password-modal>
<!-- <app-terms *ngIf="showRegistration"></app-terms> -->

6
client/src/app/app.component.scss
View File

@ -173,6 +173,12 @@ header {
}
}
.btn-primary .btn, .btn.btn-primary {
&:disabled {
opacity: 0.65;
}
}
.btn {
cursor: pointer;
display: inline-block;

3
client/src/app/app.component.ts
View File

@ -17,11 +17,12 @@ import '@cds/core/icon/register.js'
import {
ClarityIcons,
exclamationTriangleIcon,
fileIcon,
moonIcon,
sunIcon
} from '@cds/core/icon'
ClarityIcons.addIcons(moonIcon, sunIcon, exclamationTriangleIcon)
ClarityIcons.addIcons(moonIcon, sunIcon, exclamationTriangleIcon, fileIcon)
@Component({
selector: 'my-app',

8
client/src/app/editor/editor.component.html
View File

@ -3,7 +3,7 @@
appFileDrop
(fileOver)="fileOverBase($event)"
[uploader]="uploader"
(fileDrop)="getFileDesc($event, true)"
(fileDrop)="attachFile($event, true)"
[clrModalSize]="'xl'"
[clrModalStaticBackdrop]="false"
[clrModalClosable]="excelUploadState === 'Validating-DQ'"
@ -81,7 +81,7 @@
type="file"
appFileSelect
[uploader]="uploader"
(change)="getFileDesc($event)"
(change)="attachFile($event)"
/>
</div>
@ -92,7 +92,7 @@
<button
[disabled]="true"
class="btnView btn btn-sm btn-success profile-buttons w-100"
(click)="getFile()"
(click)="uploadParsedFiles()"
>
Upload
</button>
@ -576,7 +576,7 @@
<button
type="button"
class="btn btn-sm btn-primary"
(click)="getFile(); submitLimitNotice = false"
(click)="uploadParsedFiles(); submitLimitNotice = false"
>
Submit
</button>

5
client/src/app/editor/editor.component.scss
View File

@ -201,6 +201,7 @@ hot-table {
display: flex;
justify-content: center;
align-items: flex-start;
margin: 1px;
@ -211,7 +212,9 @@ hot-table {
span {
font-size: 20px;
margin-top: 20px;
color: #fff;
padding: 10px;
background: #dbdbdb;
border-radius: 5px;
}
}

878
client/src/app/editor/editor.component.ts
View File

@ -59,10 +59,7 @@ import { UploadStaterComponent } from './components/upload-stater/upload-stater.
import { DynamicExtendedCellValidation } from './models/dynamicExtendedCellValidation'
import { EditRecordInputFocusedEvent } from './models/edit-record/edit-record-events'
import { EditorRestrictions } from './models/editor-restrictions.model'
import { dateFormat, dateToTime, dateToUtcTime } from './utils/date.utils'
import {
excelDateToJSDate,
getMissingHeaders,
parseTableColumns
} from './utils/grid.utils'
import {
@ -70,10 +67,11 @@ import {
noSpinnerRenderer,
spinnerRenderer
} from './utils/renderers.utils'
import { isStringDecimal, isStringNumber } from './utils/types.utils'
import { LicenceService } from '../services/licence.service'
import * as numbro from 'numbro'
import * as languages from 'numbro/dist/languages.min'
import { FileUploadEncoding } from '../models/FileUploadEncoding'
import { ParseResult, SpreadsheetService } from '../services/spreadsheet.service'
@Component({
selector: 'app-editor',
@ -286,7 +284,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
public dateHeaders: string[] = []
public xlRules: ExcelRule[] = []
public encoding: string = 'UTF-8'
public encoding: FileUploadEncoding = 'UTF-8'
// header column names
headerColumns: Array<any> = []
@ -373,7 +371,8 @@ export class EditorComponent implements OnInit, AfterViewInit {
private route: ActivatedRoute,
private sasService: SasService,
private cdf: ChangeDetectorRef,
private hotRegisterer: HotTableRegisterer
private hotRegisterer: HotTableRegisterer,
private spreadsheetService: SpreadsheetService
) {
const lang = languages[window.navigator.language]
if (lang)
@ -475,609 +474,49 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.hasBaseDropZoneOver = e
}
/**
* Function that updates the !ref range value provided in official docs.
* @param ws worksheet to be updated
*/
private update_sheet_range(ws: XLSX.WorkSheet) {
var range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
Object.keys(ws)
.filter(function (x) {
return x.charAt(0) != '!'
})
.map(XLSX.utils.decode_cell)
.forEach(function (x: any) {
range.s.c = Math.min(range.s.c, x.c)
range.s.r = Math.min(range.s.r, x.r)
range.e.c = Math.max(range.e.c, x.c)
range.e.r = Math.max(range.e.r, x.r)
})
public attachFile(event: any, dropped: boolean = false) {
let file: File = dropped ? event[0] : event.target.files[0]
ws['!ref'] = XLSX.utils.encode_range(range)
}
/**
* Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
* @param wb Excel workbook
* @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
*/
public getRangeAndSheet(wb: XLSX.WorkBook): SheetInfo {
let rangeStartRow: number = 0
let rangeStartCol: number = 0
let startRow: number = -1
let endRow: number = -1
let sheetName: string = ''
let isComplete = false
let missingHeaders: string[] = []
let csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...this.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
wb.SheetNames.forEach((element: string) => {
// Checking for required data in each sheet in workbook/
if (isComplete) {
return
}
missingHeaders = []
sheetName = element
const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
this.data = <AOA>XLSX.utils.sheet_to_json(ws, {
header: 1,
blankrows: false,
defval: ''
})
if (this.data.length <= 1) {
return
}
let tempArr: string[] = []
this.headerArray.forEach(() => tempArr.push(''))
this.data.push(tempArr)
let foundHeaders = false
this.data.forEach((row: any, index: number) => {
if (isComplete) {
return
}
if (foundHeaders) {
let isDataEnd = true
let isPkNull = false
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
if (row[col] !== '' && row[col] !== undefined) {
isDataEnd = false
} else {
if (this.headerPks.indexOf(x.toUpperCase()) !== -1) {
isPkNull = true
}
}
})
if (isDataEnd || isPkNull) {
endRow = index
isComplete = true
} else {
if (startRow === -1) {
startRow = index
}
}
} else {
const rowLowerCase: string[] = row.map((x: any) =>
x.toString().toLowerCase()
)
// If in file there is no delete column, remove it from search of missing.
// This way delete column will be optional to provide in file
if (!rowLowerCase.includes('_____delete__this__record_____')) {
const deleteIndex = csvArrayHeadersLower.indexOf(
'_____delete__this__record_____'
)
if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
}
foundHeaders = true
csvArrayHeadersLower.forEach((x) => {
if (rowLowerCase.indexOf(x) === -1) {
foundHeaders = false
}
})
let result = []
result = this.findValidHeaders(
rowLowerCase,
csvArrayHeadersLower,
index,
sheetName
)
if (result[0] === false) {
foundHeaders = false
if (result[1].length > 0) {
result[1].forEach((data: string) => {
missingHeaders.push(data)
})
}
} else {
csvArrayHeadersMap = result[1]
}
}
})
if (isComplete) {
this.update_sheet_range(ws)
const worksheetSel = ws['!ref']
if (worksheetSel) {
const range = XLSX.utils.decode_range(ws['!ref'] || '')
rangeStartRow = range.s.r
rangeStartCol = range.s.c
}
}
})
// If start row is still -1 that means first row of found range is empty
if (startRow === -1) isComplete = false
const returnObj: SheetInfo = {
foundData: isComplete,
sheetName,
startRow,
endRow,
csvArrayHeadersMap,
missingHeaders,
rangeStartRow,
rangeStartCol
}
return returnObj
}
/**
* When excel is password protected we will display the password promppt for user to type password in.
* @returns Password user input or undefined if discarded by user
*/
public promptExcelPassword(): Promise<string | undefined> {
return new Promise((resolve, reject) => {
this.filePasswordModal = true
setTimeout(() => {
const filePasswordInputElement: any =
document.querySelector('#filePasswordInput')
if (filePasswordInputElement) {
filePasswordInputElement.focus()
filePasswordInputElement.value = ''
}
}, 100)
this.filePasswordSubject.subscribe((password: string | undefined) => {
this.fileUnlockError = false
if (password) {
resolve(password)
} else {
resolve(undefined)
}
})
})
}
/**
* Parses attached file, to be uploaded
* If attached file is CSV it will be send to backend straight away
* If attached file is EXCEL it will be displayed in the table, in preview mode
* @param event file drop event
* @param dropped whether it's dropped or added by browse button
*/
public getFileDesc(event: any, dropped: boolean = false) {
this.excelUploadState = 'Loading'
this.excelFileParsing = true
let file
if (dropped) {
file = event[0]
} else {
file = event.target.files[0]
}
this.excelFileReady = false
this.filename = ''
let filename = file.name
this.filename = filename
this.appendUploadState(`Loading ${filename} into the browser`)
let foundData = {
sheet: ''
}
let fileType = filename.slice(
filename.lastIndexOf('.') + 1,
filename.lastIndexOf('.') + 4
)
if (fileType.toLowerCase() === 'xls') {
let reader: FileReader = new FileReader()
const self = this
reader.onload = async (theFile: any) => {
/* read workbook */
const bstr = this.toBstr(theFile.target.result)
let wb: XLSX.WorkBook | undefined = undefined
let fileUnlocking: boolean = false
const xlsxOptions: XLSX.ParsingOptions = {
type: 'binary',
cellDates: false,
cellFormula: true,
cellStyles: true,
cellNF: false,
cellText: false
}
try {
wb = XLSX.read(bstr, {
...xlsxOptions
})
} catch (err: any) {
if (err.message.toLowerCase().includes('password')) {
fileUnlocking = true
while (fileUnlocking) {
const password = await this.promptExcelPassword()
if (password) {
try {
wb = XLSX.read(bstr, {
...xlsxOptions,
password: password
})
fileUnlocking = false
this.fileUnlockError = false
} catch (err: any) {
this.fileUnlockError = true
if (!err.message.toLowerCase().includes('password')) {
fileUnlocking = false
}
}
} else {
fileUnlocking = false
}
}
} else {
this.eventService.showAbortModal(
null,
err,
undefined,
'Error reading file'
)
}
}
if (!wb) {
this.excelFileParsing = false
this.showUploadModal = false
return
}
/* save data */
let isComplete: boolean = false
let missingHeaders: string[] = []
const csvArrayHeaders: string[] = [
'_____DELETE__THIS__RECORD_____',
...this.headerArray
]
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
(map: any, obj: string) => {
map[obj] = -1
return map
},
{}
)
let csvArrayData: any[] = []
const rangeSheetRes: SheetInfo = this.getRangeAndSheet(wb)
missingHeaders = rangeSheetRes.missingHeaders
if (rangeSheetRes.foundData) {
isComplete = true
csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
this.appendUploadState(
`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
)
let startAddress = ''
let endAddress = ''
for (
let row = rangeSheetRes.startRow;
row < rangeSheetRes.endRow;
++row
) {
const arr: any[] = []
csvArrayHeadersLower.forEach((x) => {
const col = csvArrayHeadersMap[x]
const addr = XLSX.utils.encode_cell({
r: rangeSheetRes.rangeStartRow + row,
c: rangeSheetRes.rangeStartCol + col
})
if (startAddress === '') startAddress = addr
endAddress = addr
let cell
if (!ws[addr]) {
cell = { v: '' }
} else {
cell = ws[addr]
}
arr.push(cell)
})
// If we found at least one non empty value it means it is not empty row
// othervise, it is empty row
let arrNonEmptyValue = arr.find((x) => x.v !== '')
if (arrNonEmptyValue) csvArrayData.push(arr)
}
this.eventService.showInfoModal(
'Table Found',
`Sheet: ${rangeSheetRes.sheetName}\nRange: ${startAddress}:${endAddress}`
)
} else {
missingHeaders = rangeSheetRes.missingHeaders
}
if (missingHeaders.length > 0) {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
let abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
this.eventService.showAbortModal(null, abortMsg)
setTimeout(() => {
this.filename = ''
})
this.excelFileParsing = false
this.uploader.queue.pop()
return
}
// If first row is empty, that means no data has been found
if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
let abortMsg = 'No relevant data found in File !'
this.eventService.showAbortModal(null, abortMsg)
setTimeout(() => {
this.filename = ''
})
this.excelFileParsing = false
this.uploader.queue.pop()
return
}
if (
this.dateTimeHeaders.length > 0 ||
this.dateHeaders.length > 0 ||
this.timeHeaders.length > 0
) {
csvArrayData = this.updateDateTimeCols(csvArrayHeaders, csvArrayData)
}
if (this.xlRules.length > 0) {
csvArrayData = this.updateXLRuleCols(csvArrayHeaders, csvArrayData)
}
if (!isComplete) {
if (missingHeaders.length === 0) {
let abortMsg = 'No relevant data found in File !'
this.eventService.showAbortModal(null, abortMsg)
setTimeout(() => {
this.filename = ''
})
} else {
missingHeaders.sort(function compareSecondColumn(a, b) {
if (a[1] === b[1]) {
return 0
} else {
return a[1] > b[1] ? -1 : 1
}
})
let abortMsg = missingHeaders
.map((x) => x[0])
.slice(0, 5)
.join('\n')
this.eventService.showAbortModal(null, abortMsg)
}
this.excelFileParsing = false
this.uploader.queue.pop()
return
} else {
this.headerShow = csvArrayHeaders
csvArrayData = csvArrayData.map((row: any) =>
row.map((col: any) => (col.t === 'n' ? col.v : col.w))
)
csvArrayData = csvArrayData.map((row: any) => {
return row.map((col: any, index: number) => {
if (!col && col !== 0) col = ''
/**
* Keeping this for the reference
* Code below used to convert JSON to CSV
* now the XLSX is converting to CSV
*/
// if (isNaN(col)) {
// // Match and replace the double quotes, ignore the first and last char
// // in case they are double quotes already
// col = col.replace(/(?<!^)"(?!$)/g, '""')
// if (col.search(/,/g) > -1 ||
// col.search(/\r|\n/g) > -1
// ) {
// // Missing quotes at the end
// if (col.search(/"$/g) < 0) {
// col = col + '"' // So we add them
// }
// // Missing quotes at the start
// if (col.search(/^"/g) < 0) {
// col = '"' + col // So we add them
// }
// }
// }
const colName = this.headerShow[index]
const colRule = this.dcValidator?.getRule(colName)
if (colRule?.type === 'numeric') {
if (isSpecialMissing(col) && !col.includes('.')) col = '.' + col
}
return col
})
})
this.data = csvArrayData
// Apply licence rows limitation if exists, it is only affecting data
// which will be send to SAS
const strippedCsvArrayData = csvArrayData.slice(
0,
this.licenceState.value.submit_rows_limit
)
// To submit to sas service, we need clean version of CSV of file
// attached. XLSX will do the parsing and heavy lifting
// First we create worksheet of json (data we extracted)
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
skipHeader: true
})
// create CSV to be uploaded from worksheet
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
// Prepend headers
csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
if (this.encoding === 'WLATIN1') {
let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
let blob = new Blob([encoded], { type: 'application/csv' })
let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
this.uploader.addToQueue([newCSVFile])
} else {
let blob = new Blob([csvContentClean], { type: 'application/csv' })
let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
this.uploader.addToQueue([newCSVFile])
}
this.excelFileReady = true
}
if (this.data.length === 0) {
this.showUploadModal = false
this.uploadPreview = false
this.excelFileParsing = false
this.eventService.showAbortModal(
null,
`Table in the file is empty. Data found on sheet: ${foundData.sheet}`
)
return
}
this.filename = file.name
this.spreadsheetService.parseExcelFile({
file: file,
dcValidator: this.dcValidator!,
headerPks: this.headerPks,
headerArray: this.headerArray,
headerShow: this.headerShow,
timeHeaders: this.timeHeaders,
dateHeaders: this.dateHeaders,
dateTimeHeaders: this.dateTimeHeaders,
xlRules: this.xlRules,
encoding: this.encoding
}, (uploadState: string) => {
this.appendUploadState(uploadState)
}).then((parseResult: ParseResult | undefined) => {
if (parseResult) {
this.excelFileReady = true
this.data = parseResult.data
this.uploader = parseResult.uploader
this.getPendingExcelPreview()
return
}
reader.readAsArrayBuffer(file)
} else if (fileType.toLowerCase() === 'csv') {
if (this.licenceState.value.submit_rows_limit !== Infinity) {
this.eventService.showInfoModal(
'Notice',
'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
)
this.excelFileReady = true
this.excelFileParsing = false
this.uploader.queue.pop()
}).catch((error: string) => {
this.eventService.showInfoModal('Error', error)
return
}
this.showUploadModal = false
this.uploadPreview = false
if (this.encoding === 'WLATIN1') {
let reader = new FileReader()
const self = this
// Closure to capture the file information.
reader.onload = (theFile: any) => {
let encoded = iconv.decode(
Buffer.from(theFile.target.result),
'CP-1252'
)
let blob = new Blob([encoded], { type: fileType })
let encodedFile: File = this.blobToFile(blob, this.filename)
this.uploader.queue.pop()
this.uploader.addToQueue([encodedFile])
this.excelFileReady = true
}
this.excelFileReady = true
this.excelFileParsing = false
reader.readAsArrayBuffer(file)
this.getFile()
} else {
this.excelFileReady = true
this.excelFileParsing = false
this.getFile()
}
} else {
let abortMsg =
'Invalid file type "<b>' +
this.filename +
'</b>". Please upload csv or excel file.'
this.eventService.showAbortModal(null, abortMsg)
this.excelFileReady = true
setTimeout(() => {
this.filename = ''
})
}).finally(() => {
this.excelFileParsing = false
this.uploader.queue.pop()
}
})
}
/**
@ -1089,13 +528,14 @@ export class EditorComponent implements OnInit, AfterViewInit {
return
}
this.getFile()
this.uploadParsedFiles()
}
/**
* This method will run validations and upload all of the pending files that are in the uploader queue
* This method will run validations and upload all of the pending files
* that are in the uploader queue.
*/
public getFile() {
public uploadParsedFiles() {
if (this.checkInvalid()) {
this.eventService.showAbortModal(null, 'Invalid values are present.')
return
@ -1272,114 +712,6 @@ export class EditorComponent implements OnInit, AfterViewInit {
}
}
findValidHeaders(
row: string[],
headers: string[],
rowNumber: number,
tabName: string
): Array<any> {
let headersFound = false
let missingErrorArray = []
let j = 0
while (j < row.length) {
if (headersFound) {
// return;
} else {
if (headers.indexOf(row[j]) !== -1) {
let breakIndex
let rowStart = 0
let rowEnd = 0
let arrStart = 0
let foundHeadersArray: string[] = []
let spaceBreak = false
for (let i = j; i < row.length; i++) {
if (
row[i] === '' ||
(foundHeadersArray.indexOf(row[i]) !== -1 &&
this.isColHeader(row[i]))
) {
if (row[i] === '') {
spaceBreak = true
}
breakIndex = i
break
} else {
foundHeadersArray.push(row[i])
}
}
let tempArray: string[] = []
if (breakIndex !== undefined) {
tempArray = row.slice(j, breakIndex)
arrStart = j
rowEnd = breakIndex
if (spaceBreak) {
rowStart = j
j = breakIndex
} else {
rowStart = j
j = breakIndex - 1
}
} else {
tempArray = row.slice(j)
rowStart = j
arrStart = j
rowEnd = row.length
j = row.length
}
let foundHeaders = true
//We check if there are missing headers
headers.forEach((x) => {
if (tempArray.indexOf(x) === -1) {
foundHeaders = false
}
})
if (foundHeaders) {
headersFound = true
let mapHeaders: any[] = headers
let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
map[obj] = -1
return map
}, {})
let temp = row.slice(rowStart, rowEnd)
headers.forEach((x) => {
csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
})
return [true, csvArrayHeadersMap]
} else {
let missingHeaders = getMissingHeaders(tempArray, headers)
let missingMessage = '<b>TAB(' + tabName + ')</b>'
missingErrorArray.push([
missingMessage +
' - ' +
missingHeaders[1].join(',') +
' ( missing ' +
missingHeaders[0].join(',') +
' )',
missingHeaders[1].length
])
}
}
}
j++
}
return [false, missingErrorArray]
}
isColPk(col: string) {
return this.headerPks.indexOf(col) > -1
}
@ -1398,136 +730,6 @@ export class EditorComponent implements OnInit, AfterViewInit {
this.sasStoreService.removeClause()
}
updateDateTimeCols(headers: any, data: any) {
if (this.dateHeaders.length > 0) {
let dateCols: number[] = []
this.dateHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
dateCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v)) {
const date = excelDateToJSDate(Number(obj.v))
obj.v =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
} else {
if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
const date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date)
}
}
}
row[element] = obj
})
})
}
if (this.timeHeaders.length > 0) {
let timeCols: number[] = []
this.timeHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
timeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
timeCols.forEach((element) => {
const obj = row[element]
if (
isStringNumber(obj.v) ||
isStringDecimal(obj.v) ||
obj.v.includes('E-')
) {
const date = excelDateToJSDate(Number(obj.v))
obj.v = dateToUtcTime(date)
}
row[element] = obj
})
})
}
if (this.dateTimeHeaders.length > 0) {
let dateTimeCols: number[] = []
this.dateTimeHeaders.forEach((element) => {
if (headers.indexOf(element) !== -1) {
dateTimeCols.push(headers.indexOf(element))
}
})
data.forEach((row: any[]) => {
dateTimeCols.forEach((element) => {
const obj = row[element]
if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
let date = excelDateToJSDate(Number(obj.v))
obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
} else {
if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
let str = obj.v.substring(0, obj.v.indexOf(':'))
str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
obj.v = str
}
let date = new Date(obj.v)
if (date.toUTCString() !== 'Invalid Date') {
obj.v = dateFormat(date) + ' ' + dateToTime(date)
}
}
row[element] = obj
})
})
}
return data
}
updateXLRuleCols(headers: any, data: any) {
if (this.xlRules.length > 0) {
const xlRuleCols: any = []
this.xlRules.forEach((element: any) => {
if (headers.indexOf(element.XL_COLUMN) !== -1) {
element['index'] = headers.indexOf(element.XL_COLUMN)
xlRuleCols.push(element)
}
})
data.forEach((row: any[]) => {
xlRuleCols.forEach((element: any) => {
const obj = row[element.index]
if (element.XL_RULE === 'FORMULA') {
if ('f' in obj) {
if (obj['t'] === 'n') {
obj['v'] = '=' + obj['f']
} else {
obj['w'] = '=' + obj['f']
}
}
}
row[element] = obj
})
})
}
return data
}
private blobToFile(theBlob: Blob, fileName: string): File {
const b: any = theBlob
b.lastModifiedDate = new Date()
b.name = fileName
return b as File
}
public toBstr(res: any) {
let bytes = new Uint8Array(res)
let binary = ''
let length = bytes.byteLength
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i])
}
return binary
}
async sendClause() {
this.submitLoading = true

4
client/src/app/home/home-routing.module.ts
View File

@ -3,6 +3,7 @@ import { RouterModule, Routes } from '@angular/router'
import { HomeRouteComponent } from '../routes/home-route/home-route.component'
import { HomeComponent } from './home.component'
import { XLMapModule } from '../xlmap/xlmap.module'
import { MultiDatasetModule } from '../multi-dataset/multi-dataset.module'
const routes: Routes = [
{
@ -11,7 +12,8 @@ const routes: Routes = [
children: [
{ path: '', pathMatch: 'full', redirectTo: 'tables' },
{ path: 'tables', component: HomeComponent },
{ path: 'files', loadChildren: () => XLMapModule }
{ path: 'excel-maps', loadChildren: () => XLMapModule },
{ path: 'multi-load', loadChildren: () => MultiDatasetModule}
]
}
]

1
client/src/app/models/FileUploadEncoding.ts Normal file
View File

@ -0,0 +1 @@
export type FileUploadEncoding = 'UTF-8' | 'WLATIN1'

21
client/src/app/multi-dataset/multi-dataset-routing.module.ts Normal file
View File

@ -0,0 +1,21 @@
import { NgModule } from '@angular/core'
import { RouterModule, Routes } from '@angular/router'
import { MultiDatasetRouteComponent } from '../routes/multi-dataset-route/multi-dataset-route.component'
import { MultiDatasetComponent } from './multi-dataset.component'
const routes: Routes = [
{
path: '',
component: MultiDatasetRouteComponent,
children: [
{ path: '', component: MultiDatasetComponent }
]
}
]
@NgModule({
imports: [RouterModule.forChild(routes)],
exports: [RouterModule]
})
export class MultiDatasetRoutingModule {}

132
client/src/app/multi-dataset/multi-dataset.component.html Normal file
View File

@ -0,0 +1,132 @@
<app-sidebar>
<div *ngIf="datasetsLoading" class="my-10-mx-auto text-center">
<clr-spinner clrMedium></clr-spinner>
</div>
<div class="text-center mb-10">
<button (click)="fileUploadInput.click()" class="btn btn-primary btn-sm" [disabled]="selectedFile !== null">Browse file</button>
<input
hidden
#fileUploadInput
id="file-upload"
type="file"
(change)="onFileChange($event)"
appFileSelect
/>
</div>
<ng-container *ngIf="datasets.length > 0">
<p cds-text="caption" class="ml-10">Found tables:</p>
<clr-tree>
<clr-tree-node *ngFor="let dataset of datasets">
<button
class="clr-treenode-link"
[class.active]="dataset.active"
>
<cds-icon *ngIf="dataset.status === 'error'" status</