diff --git a/client/package-lock.json b/client/package-lock.json index ec4a68d..d9815cb 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -3428,9 +3428,9 @@ "optional": true }, "node_modules/@cds/core": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@cds/core/-/core-6.11.0.tgz", - "integrity": "sha512-7tNZeLEfpcNdDN85jV9137zxy7euqBCtG8SRNX0E8XGMMODs7JpxvORahhwRGwd7geSbZpA70urNo+h5Y2Mqrg==", + "version": "6.12.0", + "resolved": "https://registry.npmjs.org/@cds/core/-/core-6.12.0.tgz", + "integrity": "sha512-eJpddpt4vx4s0EFi2kAvnRZz1h/JVZPqjkNXpDalBM61IqQp9g2ETGoa594t6BWe3cwGDrkiKuEB+FbAx+IEhw==", "dependencies": { "lit": "^2.1.3", "ramda": "^0.29.0", @@ -4531,9 +4531,9 @@ "dev": true }, "node_modules/@handsontable/angular": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/@handsontable/angular/-/angular-14.3.0.tgz", - "integrity": "sha512-Tdea1fKgQY1DG1upr2I8AlzbYg+kvI4sw4FU/dwWCgKKUkArAT8bW2aOhZMh55CYLJksLFTgCtPSQQMuSeR39g==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/@handsontable/angular/-/angular-14.4.0.tgz", + "integrity": "sha512-r984kb/tssJ0nR/uB2bimJWoi+PxoJb4A3RDg1PwAZdquYCPs1j1Yr66N2Z46Ua1Pc1IOnaruLg/kkOuTnPVGg==", "optionalDependencies": { "tslib": "^2.3.0" }, @@ -4584,6 +4584,7 @@ "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "deprecated": "Use @eslint/config-array instead", "dev": true, "dependencies": { "@humanwhocodes/object-schema": "^2.0.2", @@ -4611,6 +4612,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", "dev": true }, "node_modules/@isaacs/cliui": { @@ -6238,9 +6240,9 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" }, "node_modules/@types/lodash": { - "version": "4.17.4", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.4.tgz", - "integrity": "sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ==", + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==", "dev": true }, "node_modules/@types/lodash-es": { @@ -7071,6 +7073,15 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", + "dev": true, + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", "peerDependencies": { "acorn": "^8" } @@ -8050,9 +8061,9 @@ } }, "node_modules/browserslist": { - "version": "4.23.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", - "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", + "version": "4.23.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.1.tgz", + "integrity": "sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw==", "funding": [ { "type": "opencollective", @@ -8068,10 +8079,10 @@ } ], "dependencies": { - "caniuse-lite": "^1.0.30001587", - "electron-to-chromium": "^1.4.668", + "caniuse-lite": "^1.0.30001629", + "electron-to-chromium": "^1.4.796", "node-releases": "^2.0.14", - "update-browserslist-db": "^1.0.13" + "update-browserslist-db": "^1.0.16" }, "bin": { "browserslist": "cli.js" @@ -8224,9 +8235,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001629", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001629.tgz", - "integrity": "sha512-c3dl911slnQhmxUIT4HhYzT7wnBK/XYpGnYLOj4nJBaRiw52Ibe7YxlDaAeRECvA786zCuExhxIUJ2K7nHMrBw==", + "version": "1.0.30001633", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001633.tgz", + "integrity": "sha512-6sT0yf/z5jqf8tISAgpJDrmwOpLsrpnyCdD/lOZKvKkkJK4Dn0X5i7KF7THEZhOq+30bmhwBlNEaqPUiHiKtZg==", "funding": [ { "type": "opencollective", @@ -10137,9 +10148,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.795", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.795.tgz", - "integrity": "sha512-hHo4lK/8wb4NUa+NJYSFyJ0xedNHiR6ylilDtb8NUW9d4dmBFmGiecYEKCEbti1wTNzbKXLfl4hPWEkAFbHYlw==" + "version": "1.4.802", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.802.tgz", + "integrity": "sha512-TnTMUATbgNdPXVSHsxvNVSG0uEd6cSZsANjm8c9HbvflZVVn1yTRcmVXYT1Ma95/ssB/Dcd30AHweH2TE+dNpA==" }, "node_modules/elliptic": { "version": "6.5.5", @@ -11509,9 +11520,9 @@ } }, "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.0.tgz", + "integrity": "sha512-CrWQNaEl1/6WeZoarcM9LHupTo3RpZO2Pdk1vktwzPiQTsJnAKJmm3TACKeG5UZbWDfaH2AbvYxzP96y0MT7fA==", "dev": true, "dependencies": { "cross-spawn": "^7.0.0", @@ -12015,15 +12026,15 @@ } }, "node_modules/handsontable": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/handsontable/-/handsontable-14.3.0.tgz", - "integrity": "sha512-XRKivPK+DkxWTYr+H6ywwLhSn/B5WMj3F3whF7O+Mo6Edm35rF8ydOm6oxVuVWnkvvy4FLqch7wgQk2w6hC8VA==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/handsontable/-/handsontable-14.4.0.tgz", + "integrity": "sha512-uTIIx5UgG+2I0flUDj8lxftJAXbBpbkIAFIKugvMW2bNHYNxaNMGAGZ/QFOeCXiYwxeztBATKxMkLpg7N+HrOg==", "dependencies": { "@handsontable/pikaday": "^1.0.0", "@types/pikaday": "1.7.4", - "core-js": "^3.31.1", + "core-js": "^3.37.0", "dompurify": "^2.1.1", - "moment": "2.29.4", + "moment": "2.30.1", "numbro": "2.1.2", "pikaday": "1.8.2" }, @@ -12041,14 +12052,6 @@ "url": "https://opencollective.com/core-js" } }, - "node_modules/handsontable/node_modules/moment": { - "version": "2.29.4", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", - "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", - "engines": { - "node": "*" - } - }, "node_modules/has-bigints": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", @@ -13532,9 +13535,9 @@ } }, "node_modules/jiti": { - "version": "1.21.3", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.3.tgz", - "integrity": "sha512-uy2bNX5zQ+tESe+TiC7ilGRz8AtRGmnJH55NC5S0nSUjvvvM2hJHmefHErugGXN4pNv4Qx7vLsnNw9qJ9mtIsw==", + "version": "1.21.6", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz", + "integrity": "sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==", "dev": true, "bin": { "jiti": "bin/jiti.js" @@ -18348,9 +18351,9 @@ } }, "node_modules/rfdc": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.1.tgz", - "integrity": "sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", "dev": true }, "node_modules/rimraf": { @@ -20508,9 +20511,9 @@ } }, "node_modules/uglify-js": { - "version": "3.17.4", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", - "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.18.0.tgz", + "integrity": "sha512-SyVVbcNBCk0dzr9XL/R/ySrmYf0s372K6/hFklzgcp2lBFyXtw4I7BOdDjlLhE1aVqaI/SHWXWmYdlZxuyF38A==", "dev": true, "optional": true, "bin": { @@ -20925,9 +20928,9 @@ } }, "node_modules/webpack": { - "version": "5.91.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.91.0.tgz", - "integrity": "sha512-rzVwlLeBWHJbmgTC/8TvAcu5vpJNII+MelQpylD4jNERPwpBJOE2lEcko1zJX3QJeLjTTAnQxn/OJ8bjDzVQaw==", + "version": "5.92.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.92.0.tgz", + "integrity": "sha512-Bsw2X39MYIgxouNATyVpCNVWBCuUwDgWtN78g6lSdPJRLaQ/PUVm/oXcaRAyY/sMFoKFQrsPeqvTizWtq7QPCA==", "dependencies": { "@types/eslint-scope": "^3.7.3", "@types/estree": "^1.0.5", @@ -20935,10 +20938,10 @@ "@webassemblyjs/wasm-edit": "^1.12.1", "@webassemblyjs/wasm-parser": "^1.12.1", "acorn": "^8.7.1", - "acorn-import-assertions": "^1.9.0", + "acorn-import-attributes": "^1.9.5", "browserslist": "^4.21.10", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.16.0", + "enhanced-resolve": "^5.17.0", "es-module-lexer": "^1.2.1", "eslint-scope": "5.1.1", "events": "^3.2.0", diff --git a/client/package.json b/client/package.json index 6e5e50f..4f6c77e 100644 --- a/client/package.json +++ b/client/package.json @@ -53,7 +53,6 @@ "@sasjs/utils": "^3.4.0", "@sheet/crypto": "file:libraries/sheet-crypto.tgz", "@types/d3-graphviz": "^2.6.7", - "@sheet/crypto": "file:libraries/sheet-crypto.tgz", "@types/text-encoding": "0.0.35", "base64-arraybuffer": "^0.2.0", "buffer": "^5.4.3", diff --git a/client/src/app/app.component.html b/client/src/app/app.component.html index 86442ed..828eebf 100644 --- a/client/src/app/app.component.html +++ b/client/src/app/app.component.html @@ -245,6 +245,7 @@ + diff --git a/client/src/app/app.component.scss b/client/src/app/app.component.scss index 294a9d5..9b80967 100644 --- a/client/src/app/app.component.scss +++ b/client/src/app/app.component.scss @@ -173,6 +173,12 @@ header { } } + .btn-primary .btn, .btn.btn-primary { + &:disabled { + opacity: 0.65; + } + } + .btn { cursor: pointer; display: inline-block; diff --git a/client/src/app/app.component.ts b/client/src/app/app.component.ts index 91c0002..9a4b7e1 100644 --- a/client/src/app/app.component.ts +++ b/client/src/app/app.component.ts @@ -17,11 +17,12 @@ import '@cds/core/icon/register.js' import { ClarityIcons, exclamationTriangleIcon, + fileIcon, moonIcon, sunIcon } from '@cds/core/icon' -ClarityIcons.addIcons(moonIcon, sunIcon, exclamationTriangleIcon) +ClarityIcons.addIcons(moonIcon, sunIcon, exclamationTriangleIcon, fileIcon) @Component({ selector: 'my-app', diff --git a/client/src/app/editor/editor.component.html b/client/src/app/editor/editor.component.html index 77a505a..c5c01c3 100644 --- a/client/src/app/editor/editor.component.html +++ b/client/src/app/editor/editor.component.html @@ -3,7 +3,7 @@ appFileDrop (fileOver)="fileOverBase($event)" [uploader]="uploader" - (fileDrop)="getFileDesc($event, true)" + (fileDrop)="attachFile($event, true)" [clrModalSize]="'xl'" [clrModalStaticBackdrop]="false" [clrModalClosable]="excelUploadState === 'Validating-DQ'" @@ -81,7 +81,7 @@ type="file" appFileSelect [uploader]="uploader" - (change)="getFileDesc($event)" + (change)="attachFile($event)" /> @@ -92,7 +92,7 @@ Upload @@ -576,7 +576,7 @@ Submit diff --git a/client/src/app/editor/editor.component.scss b/client/src/app/editor/editor.component.scss index b787202..cea5959 100644 --- a/client/src/app/editor/editor.component.scss +++ b/client/src/app/editor/editor.component.scss @@ -201,6 +201,7 @@ hot-table { display: flex; justify-content: center; + align-items: flex-start; margin: 1px; @@ -211,7 +212,9 @@ hot-table { span { font-size: 20px; margin-top: 20px; - color: #fff; + padding: 10px; + background: #dbdbdb; + border-radius: 5px; } } diff --git a/client/src/app/editor/editor.component.ts b/client/src/app/editor/editor.component.ts index 0246f0b..a3941e2 100644 --- a/client/src/app/editor/editor.component.ts +++ b/client/src/app/editor/editor.component.ts @@ -59,10 +59,7 @@ import { UploadStaterComponent } from './components/upload-stater/upload-stater. import { DynamicExtendedCellValidation } from './models/dynamicExtendedCellValidation' import { EditRecordInputFocusedEvent } from './models/edit-record/edit-record-events' import { EditorRestrictions } from './models/editor-restrictions.model' -import { dateFormat, dateToTime, dateToUtcTime } from './utils/date.utils' import { - excelDateToJSDate, - getMissingHeaders, parseTableColumns } from './utils/grid.utils' import { @@ -70,10 +67,11 @@ import { noSpinnerRenderer, spinnerRenderer } from './utils/renderers.utils' -import { isStringDecimal, isStringNumber } from './utils/types.utils' import { LicenceService } from '../services/licence.service' import * as numbro from 'numbro' import * as languages from 'numbro/dist/languages.min' +import { FileUploadEncoding } from '../models/FileUploadEncoding' +import { ParseResult, SpreadsheetService } from '../services/spreadsheet.service' @Component({ selector: 'app-editor', @@ -286,7 +284,7 @@ export class EditorComponent implements OnInit, AfterViewInit { public dateHeaders: string[] = [] public xlRules: ExcelRule[] = [] - public encoding: string = 'UTF-8' + public encoding: FileUploadEncoding = 'UTF-8' // header column names headerColumns: Array = [] @@ -373,7 +371,8 @@ export class EditorComponent implements OnInit, AfterViewInit { private route: ActivatedRoute, private sasService: SasService, private cdf: ChangeDetectorRef, - private hotRegisterer: HotTableRegisterer + private hotRegisterer: HotTableRegisterer, + private spreadsheetService: SpreadsheetService ) { const lang = languages[window.navigator.language] if (lang) @@ -475,609 +474,49 @@ export class EditorComponent implements OnInit, AfterViewInit { this.hasBaseDropZoneOver = e } - /** - * Function that updates the !ref range value provided in official docs. - * @param ws worksheet to be updated - */ - private update_sheet_range(ws: XLSX.WorkSheet) { - var range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } } - Object.keys(ws) - .filter(function (x) { - return x.charAt(0) != '!' - }) - .map(XLSX.utils.decode_cell) - .forEach(function (x: any) { - range.s.c = Math.min(range.s.c, x.c) - range.s.r = Math.min(range.s.r, x.r) - range.e.c = Math.max(range.e.c, x.c) - range.e.r = Math.max(range.e.r, x.r) - }) + public attachFile(event: any, dropped: boolean = false) { + let file: File = dropped ? event[0] : event.target.files[0] - ws['!ref'] = XLSX.utils.encode_range(range) - } - - /** - * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers - * @param wb Excel workbook - * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range - */ - public getRangeAndSheet(wb: XLSX.WorkBook): SheetInfo { - let rangeStartRow: number = 0 - let rangeStartCol: number = 0 - let startRow: number = -1 - let endRow: number = -1 - let sheetName: string = '' - let isComplete = false - let missingHeaders: string[] = [] - let csvArrayHeaders: string[] = [ - '_____DELETE__THIS__RECORD_____', - ...this.headerArray - ] - let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) - let csvArrayHeadersMap = csvArrayHeadersLower.reduce( - (map: any, obj: string) => { - map[obj] = -1 - return map - }, - {} - ) - - wb.SheetNames.forEach((element: string) => { - // Checking for required data in each sheet in workbook/ - if (isComplete) { - return - } - - missingHeaders = [] - sheetName = element - const ws: XLSX.WorkSheet = wb.Sheets[sheetName] - - this.data = XLSX.utils.sheet_to_json(ws, { - header: 1, - blankrows: false, - defval: '' - }) - - if (this.data.length <= 1) { - return - } - - let tempArr: string[] = [] - this.headerArray.forEach(() => tempArr.push('')) - this.data.push(tempArr) - - let foundHeaders = false - - this.data.forEach((row: any, index: number) => { - if (isComplete) { - return - } - - if (foundHeaders) { - let isDataEnd = true - let isPkNull = false - - csvArrayHeadersLower.forEach((x) => { - const col = csvArrayHeadersMap[x] - - if (row[col] !== '' && row[col] !== undefined) { - isDataEnd = false - } else { - if (this.headerPks.indexOf(x.toUpperCase()) !== -1) { - isPkNull = true - } - } - }) - - if (isDataEnd || isPkNull) { - endRow = index - isComplete = true - } else { - if (startRow === -1) { - startRow = index - } - } - } else { - const rowLowerCase: string[] = row.map((x: any) => - x.toString().toLowerCase() - ) - - // If in file there is no delete column, remove it from search of missing. - // This way delete column will be optional to provide in file - if (!rowLowerCase.includes('_____delete__this__record_____')) { - const deleteIndex = csvArrayHeadersLower.indexOf( - '_____delete__this__record_____' - ) - - if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1) - } - - foundHeaders = true - - csvArrayHeadersLower.forEach((x) => { - if (rowLowerCase.indexOf(x) === -1) { - foundHeaders = false - } - }) - - let result = [] - - result = this.findValidHeaders( - rowLowerCase, - csvArrayHeadersLower, - index, - sheetName - ) - - if (result[0] === false) { - foundHeaders = false - - if (result[1].length > 0) { - result[1].forEach((data: string) => { - missingHeaders.push(data) - }) - } - } else { - csvArrayHeadersMap = result[1] - } - } - }) - - if (isComplete) { - this.update_sheet_range(ws) - const worksheetSel = ws['!ref'] - - if (worksheetSel) { - const range = XLSX.utils.decode_range(ws['!ref'] || '') - rangeStartRow = range.s.r - rangeStartCol = range.s.c - } - } - }) - - // If start row is still -1 that means first row of found range is empty - if (startRow === -1) isComplete = false - - const returnObj: SheetInfo = { - foundData: isComplete, - sheetName, - startRow, - endRow, - csvArrayHeadersMap, - missingHeaders, - rangeStartRow, - rangeStartCol - } - - return returnObj - } - - /** - * When excel is password protected we will display the password promppt for user to type password in. - * @returns Password user input or undefined if discarded by user - */ - public promptExcelPassword(): Promise { - return new Promise((resolve, reject) => { - this.filePasswordModal = true - - setTimeout(() => { - const filePasswordInputElement: any = - document.querySelector('#filePasswordInput') - if (filePasswordInputElement) { - filePasswordInputElement.focus() - filePasswordInputElement.value = '' - } - }, 100) - - this.filePasswordSubject.subscribe((password: string | undefined) => { - this.fileUnlockError = false - - if (password) { - resolve(password) - } else { - resolve(undefined) - } - }) - }) - } - - /** - * Parses attached file, to be uploaded - * If attached file is CSV it will be send to backend straight away - * If attached file is EXCEL it will be displayed in the table, in preview mode - * @param event file drop event - * @param dropped whether it's dropped or added by browse button - */ - public getFileDesc(event: any, dropped: boolean = false) { this.excelUploadState = 'Loading' this.excelFileParsing = true - - let file - if (dropped) { - file = event[0] - } else { - file = event.target.files[0] - } - this.excelFileReady = false - this.filename = '' - let filename = file.name - this.filename = filename - - this.appendUploadState(`Loading ${filename} into the browser`) - - let foundData = { - sheet: '' - } - - let fileType = filename.slice( - filename.lastIndexOf('.') + 1, - filename.lastIndexOf('.') + 4 - ) - - if (fileType.toLowerCase() === 'xls') { - let reader: FileReader = new FileReader() - const self = this - reader.onload = async (theFile: any) => { - /* read workbook */ - const bstr = this.toBstr(theFile.target.result) - let wb: XLSX.WorkBook | undefined = undefined - let fileUnlocking: boolean = false - - const xlsxOptions: XLSX.ParsingOptions = { - type: 'binary', - cellDates: false, - cellFormula: true, - cellStyles: true, - cellNF: false, - cellText: false - } - - try { - wb = XLSX.read(bstr, { - ...xlsxOptions - }) - } catch (err: any) { - if (err.message.toLowerCase().includes('password')) { - fileUnlocking = true - - while (fileUnlocking) { - const password = await this.promptExcelPassword() - - if (password) { - try { - wb = XLSX.read(bstr, { - ...xlsxOptions, - password: password - }) - - fileUnlocking = false - this.fileUnlockError = false - } catch (err: any) { - this.fileUnlockError = true - - if (!err.message.toLowerCase().includes('password')) { - fileUnlocking = false - } - } - } else { - fileUnlocking = false - } - } - } else { - this.eventService.showAbortModal( - null, - err, - undefined, - 'Error reading file' - ) - } - } - - if (!wb) { - this.excelFileParsing = false - this.showUploadModal = false - return - } - - /* save data */ - let isComplete: boolean = false - let missingHeaders: string[] = [] - - const csvArrayHeaders: string[] = [ - '_____DELETE__THIS__RECORD_____', - ...this.headerArray - ] - let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) - let csvArrayHeadersMap = csvArrayHeadersLower.reduce( - (map: any, obj: string) => { - map[obj] = -1 - return map - }, - {} - ) - - let csvArrayData: any[] = [] - const rangeSheetRes: SheetInfo = this.getRangeAndSheet(wb) - missingHeaders = rangeSheetRes.missingHeaders - - if (rangeSheetRes.foundData) { - isComplete = true - csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap - const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName] - - this.appendUploadState( - `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}` - ) - - let startAddress = '' - let endAddress = '' - - for ( - let row = rangeSheetRes.startRow; - row < rangeSheetRes.endRow; - ++row - ) { - const arr: any[] = [] - - csvArrayHeadersLower.forEach((x) => { - const col = csvArrayHeadersMap[x] - const addr = XLSX.utils.encode_cell({ - r: rangeSheetRes.rangeStartRow + row, - c: rangeSheetRes.rangeStartCol + col - }) - - if (startAddress === '') startAddress = addr - endAddress = addr - - let cell - - if (!ws[addr]) { - cell = { v: '' } - } else { - cell = ws[addr] - } - arr.push(cell) - }) - - // If we found at least one non empty value it means it is not empty row - // othervise, it is empty row - let arrNonEmptyValue = arr.find((x) => x.v !== '') - - if (arrNonEmptyValue) csvArrayData.push(arr) - } - - this.eventService.showInfoModal( - 'Table Found', - `Sheet: ${rangeSheetRes.sheetName}\nRange: ${startAddress}:${endAddress}` - ) - } else { - missingHeaders = rangeSheetRes.missingHeaders - } - - if (missingHeaders.length > 0) { - missingHeaders.sort(function compareSecondColumn(a, b) { - if (a[1] === b[1]) { - return 0 - } else { - return a[1] > b[1] ? -1 : 1 - } - }) - let abortMsg = missingHeaders - .map((x) => x[0]) - .slice(0, 5) - .join('\n') - - this.eventService.showAbortModal(null, abortMsg) - - setTimeout(() => { - this.filename = '' - }) - - this.excelFileParsing = false - this.uploader.queue.pop() - return - } - - // If first row is empty, that means no data has been found - if (csvArrayData.length === 0 || csvArrayData[0].length === 0) { - let abortMsg = 'No relevant data found in File !' - this.eventService.showAbortModal(null, abortMsg) - - setTimeout(() => { - this.filename = '' - }) - - this.excelFileParsing = false - this.uploader.queue.pop() - return - } - - if ( - this.dateTimeHeaders.length > 0 || - this.dateHeaders.length > 0 || - this.timeHeaders.length > 0 - ) { - csvArrayData = this.updateDateTimeCols(csvArrayHeaders, csvArrayData) - } - - if (this.xlRules.length > 0) { - csvArrayData = this.updateXLRuleCols(csvArrayHeaders, csvArrayData) - } - - if (!isComplete) { - if (missingHeaders.length === 0) { - let abortMsg = 'No relevant data found in File !' - this.eventService.showAbortModal(null, abortMsg) - - setTimeout(() => { - this.filename = '' - }) - } else { - missingHeaders.sort(function compareSecondColumn(a, b) { - if (a[1] === b[1]) { - return 0 - } else { - return a[1] > b[1] ? -1 : 1 - } - }) - let abortMsg = missingHeaders - .map((x) => x[0]) - .slice(0, 5) - .join('\n') - - this.eventService.showAbortModal(null, abortMsg) - } - this.excelFileParsing = false - this.uploader.queue.pop() - return - } else { - this.headerShow = csvArrayHeaders - csvArrayData = csvArrayData.map((row: any) => - row.map((col: any) => (col.t === 'n' ? col.v : col.w)) - ) - - csvArrayData = csvArrayData.map((row: any) => { - return row.map((col: any, index: number) => { - if (!col && col !== 0) col = '' - - /** - * Keeping this for the reference - * Code below used to convert JSON to CSV - * now the XLSX is converting to CSV - */ - // if (isNaN(col)) { - // // Match and replace the double quotes, ignore the first and last char - // // in case they are double quotes already - // col = col.replace(/(? -1 || - // col.search(/\r|\n/g) > -1 - // ) { - // // Missing quotes at the end - // if (col.search(/"$/g) < 0) { - // col = col + '"' // So we add them - // } - - // // Missing quotes at the start - // if (col.search(/^"/g) < 0) { - // col = '"' + col // So we add them - // } - // } - // } - - const colName = this.headerShow[index] - const colRule = this.dcValidator?.getRule(colName) - - if (colRule?.type === 'numeric') { - if (isSpecialMissing(col) && !col.includes('.')) col = '.' + col - } - - return col - }) - }) - - this.data = csvArrayData - - // Apply licence rows limitation if exists, it is only affecting data - // which will be send to SAS - const strippedCsvArrayData = csvArrayData.slice( - 0, - this.licenceState.value.submit_rows_limit - ) - // To submit to sas service, we need clean version of CSV of file - // attached. XLSX will do the parsing and heavy lifting - // First we create worksheet of json (data we extracted) - let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { - skipHeader: true - }) - // create CSV to be uploaded from worksheet - let csvContentClean = XLSX.utils.sheet_to_csv(ws) - // Prepend headers - csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean - - if (this.encoding === 'WLATIN1') { - let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252') - let blob = new Blob([encoded], { type: 'application/csv' }) - let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv') - this.uploader.addToQueue([newCSVFile]) - } else { - let blob = new Blob([csvContentClean], { type: 'application/csv' }) - let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv') - this.uploader.addToQueue([newCSVFile]) - } - - this.excelFileReady = true - } - - if (this.data.length === 0) { - this.showUploadModal = false - this.uploadPreview = false - this.excelFileParsing = false - - this.eventService.showAbortModal( - null, - `Table in the file is empty. Data found on sheet: ${foundData.sheet}` - ) - - return - } + this.filename = file.name + this.spreadsheetService.parseExcelFile({ + file: file, + dcValidator: this.dcValidator!, + headerPks: this.headerPks, + headerArray: this.headerArray, + headerShow: this.headerShow, + timeHeaders: this.timeHeaders, + dateHeaders: this.dateHeaders, + dateTimeHeaders: this.dateTimeHeaders, + xlRules: this.xlRules, + encoding: this.encoding + }, (uploadState: string) => { + this.appendUploadState(uploadState) + }).then((parseResult: ParseResult | undefined) => { + if (parseResult) { this.excelFileReady = true + + this.data = parseResult.data + this.uploader = parseResult.uploader + this.getPendingExcelPreview() - - return } - reader.readAsArrayBuffer(file) - } else if (fileType.toLowerCase() === 'csv') { - if (this.licenceState.value.submit_rows_limit !== Infinity) { - this.eventService.showInfoModal( - 'Notice', - 'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io' - ) - this.excelFileReady = true - this.excelFileParsing = false - this.uploader.queue.pop() + }).catch((error: string) => { + this.eventService.showInfoModal('Error', error) - return - } + this.showUploadModal = false + this.uploadPreview = false - if (this.encoding === 'WLATIN1') { - let reader = new FileReader() - const self = this - // Closure to capture the file information. - reader.onload = (theFile: any) => { - let encoded = iconv.decode( - Buffer.from(theFile.target.result), - 'CP-1252' - ) - let blob = new Blob([encoded], { type: fileType }) - let encodedFile: File = this.blobToFile(blob, this.filename) - this.uploader.queue.pop() - this.uploader.addToQueue([encodedFile]) - this.excelFileReady = true - } - this.excelFileReady = true - this.excelFileParsing = false - reader.readAsArrayBuffer(file) - this.getFile() - } else { - this.excelFileReady = true - this.excelFileParsing = false - this.getFile() - } - } else { - let abortMsg = - 'Invalid file type "' + - this.filename + - '". Please upload csv or excel file.' - this.eventService.showAbortModal(null, abortMsg) - - this.excelFileReady = true + setTimeout(() => { + this.filename = '' + }) + }).finally(() => { this.excelFileParsing = false - this.uploader.queue.pop() - } + }) } /** @@ -1089,13 +528,14 @@ export class EditorComponent implements OnInit, AfterViewInit { return } - this.getFile() + this.uploadParsedFiles() } /** - * This method will run validations and upload all of the pending files that are in the uploader queue + * This method will run validations and upload all of the pending files + * that are in the uploader queue. */ - public getFile() { + public uploadParsedFiles() { if (this.checkInvalid()) { this.eventService.showAbortModal(null, 'Invalid values are present.') return @@ -1272,114 +712,6 @@ export class EditorComponent implements OnInit, AfterViewInit { } } - findValidHeaders( - row: string[], - headers: string[], - rowNumber: number, - tabName: string - ): Array { - let headersFound = false - let missingErrorArray = [] - let j = 0 - - while (j < row.length) { - if (headersFound) { - // return; - } else { - if (headers.indexOf(row[j]) !== -1) { - let breakIndex - let rowStart = 0 - let rowEnd = 0 - let arrStart = 0 - let foundHeadersArray: string[] = [] - let spaceBreak = false - - for (let i = j; i < row.length; i++) { - if ( - row[i] === '' || - (foundHeadersArray.indexOf(row[i]) !== -1 && - this.isColHeader(row[i])) - ) { - if (row[i] === '') { - spaceBreak = true - } - - breakIndex = i - break - } else { - foundHeadersArray.push(row[i]) - } - } - - let tempArray: string[] = [] - - if (breakIndex !== undefined) { - tempArray = row.slice(j, breakIndex) - arrStart = j - rowEnd = breakIndex - - if (spaceBreak) { - rowStart = j - j = breakIndex - } else { - rowStart = j - j = breakIndex - 1 - } - } else { - tempArray = row.slice(j) - rowStart = j - arrStart = j - rowEnd = row.length - j = row.length - } - - let foundHeaders = true - - //We check if there are missing headers - headers.forEach((x) => { - if (tempArray.indexOf(x) === -1) { - foundHeaders = false - } - }) - - if (foundHeaders) { - headersFound = true - - let mapHeaders: any[] = headers - - let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) { - map[obj] = -1 - return map - }, {}) - - let temp = row.slice(rowStart, rowEnd) - - headers.forEach((x) => { - csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart - }) - - return [true, csvArrayHeadersMap] - } else { - let missingHeaders = getMissingHeaders(tempArray, headers) - - let missingMessage = 'TAB(' + tabName + ')' - missingErrorArray.push([ - missingMessage + - ' - ' + - missingHeaders[1].join(',') + - ' ( missing ' + - missingHeaders[0].join(',') + - ' )', - missingHeaders[1].length - ]) - } - } - } - j++ - } - return [false, missingErrorArray] - } - isColPk(col: string) { return this.headerPks.indexOf(col) > -1 } @@ -1398,136 +730,6 @@ export class EditorComponent implements OnInit, AfterViewInit { this.sasStoreService.removeClause() } - updateDateTimeCols(headers: any, data: any) { - if (this.dateHeaders.length > 0) { - let dateCols: number[] = [] - this.dateHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - dateCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - dateCols.forEach((element) => { - const obj = row[element] - if (isStringNumber(obj.v)) { - const date = excelDateToJSDate(Number(obj.v)) - - obj.v = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - } else { - if (obj && obj.v && obj.v.toString().indexOf(':') === -1) { - const date = new Date(obj.v) - if (date.toUTCString() !== 'Invalid Date') { - obj.v = dateFormat(date) - } - } - } - row[element] = obj - }) - }) - } - if (this.timeHeaders.length > 0) { - let timeCols: number[] = [] - this.timeHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - timeCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - timeCols.forEach((element) => { - const obj = row[element] - if ( - isStringNumber(obj.v) || - isStringDecimal(obj.v) || - obj.v.includes('E-') - ) { - const date = excelDateToJSDate(Number(obj.v)) - - obj.v = dateToUtcTime(date) - } - row[element] = obj - }) - }) - } - if (this.dateTimeHeaders.length > 0) { - let dateTimeCols: number[] = [] - this.dateTimeHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - dateTimeCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - dateTimeCols.forEach((element) => { - const obj = row[element] - if (isStringNumber(obj.v) || isStringDecimal(obj.v)) { - let date = excelDateToJSDate(Number(obj.v)) - obj.v = dateFormat(date) + ' ' + dateToUtcTime(date) - } else { - if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) { - let str = obj.v.substring(0, obj.v.indexOf(':')) - str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1) - obj.v = str - } - let date = new Date(obj.v) - if (date.toUTCString() !== 'Invalid Date') { - obj.v = dateFormat(date) + ' ' + dateToTime(date) - } - } - row[element] = obj - }) - }) - } - return data - } - - updateXLRuleCols(headers: any, data: any) { - if (this.xlRules.length > 0) { - const xlRuleCols: any = [] - this.xlRules.forEach((element: any) => { - if (headers.indexOf(element.XL_COLUMN) !== -1) { - element['index'] = headers.indexOf(element.XL_COLUMN) - xlRuleCols.push(element) - } - }) - data.forEach((row: any[]) => { - xlRuleCols.forEach((element: any) => { - const obj = row[element.index] - if (element.XL_RULE === 'FORMULA') { - if ('f' in obj) { - if (obj['t'] === 'n') { - obj['v'] = '=' + obj['f'] - } else { - obj['w'] = '=' + obj['f'] - } - } - } - row[element] = obj - }) - }) - } - return data - } - - private blobToFile(theBlob: Blob, fileName: string): File { - const b: any = theBlob - b.lastModifiedDate = new Date() - b.name = fileName - return b as File - } - - public toBstr(res: any) { - let bytes = new Uint8Array(res) - let binary = '' - let length = bytes.byteLength - for (let i = 0; i < length; i++) { - binary += String.fromCharCode(bytes[i]) - } - return binary - } async sendClause() { this.submitLoading = true diff --git a/client/src/app/home/home-routing.module.ts b/client/src/app/home/home-routing.module.ts index 209272f..c1e8f68 100644 --- a/client/src/app/home/home-routing.module.ts +++ b/client/src/app/home/home-routing.module.ts @@ -3,6 +3,7 @@ import { RouterModule, Routes } from '@angular/router' import { HomeRouteComponent } from '../routes/home-route/home-route.component' import { HomeComponent } from './home.component' import { XLMapModule } from '../xlmap/xlmap.module' +import { MultiDatasetModule } from '../multi-dataset/multi-dataset.module' const routes: Routes = [ { @@ -11,7 +12,8 @@ const routes: Routes = [ children: [ { path: '', pathMatch: 'full', redirectTo: 'tables' }, { path: 'tables', component: HomeComponent }, - { path: 'files', loadChildren: () => XLMapModule } + { path: 'excel-maps', loadChildren: () => XLMapModule }, + { path: 'multi-load', loadChildren: () => MultiDatasetModule} ] } ] diff --git a/client/src/app/models/FileUploadEncoding.ts b/client/src/app/models/FileUploadEncoding.ts new file mode 100644 index 0000000..94f8b69 --- /dev/null +++ b/client/src/app/models/FileUploadEncoding.ts @@ -0,0 +1 @@ +export type FileUploadEncoding = 'UTF-8' | 'WLATIN1' \ No newline at end of file diff --git a/client/src/app/multi-dataset/multi-dataset-routing.module.ts b/client/src/app/multi-dataset/multi-dataset-routing.module.ts new file mode 100644 index 0000000..7ec0403 --- /dev/null +++ b/client/src/app/multi-dataset/multi-dataset-routing.module.ts @@ -0,0 +1,21 @@ +import { NgModule } from '@angular/core' +import { RouterModule, Routes } from '@angular/router' + +import { MultiDatasetRouteComponent } from '../routes/multi-dataset-route/multi-dataset-route.component' +import { MultiDatasetComponent } from './multi-dataset.component' + +const routes: Routes = [ + { + path: '', + component: MultiDatasetRouteComponent, + children: [ + { path: '', component: MultiDatasetComponent } + ] + } +] + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule] +}) +export class MultiDatasetRoutingModule {} diff --git a/client/src/app/multi-dataset/multi-dataset.component.html b/client/src/app/multi-dataset/multi-dataset.component.html new file mode 100644 index 0000000..00bfee3 --- /dev/null +++ b/client/src/app/multi-dataset/multi-dataset.component.html @@ -0,0 +1,132 @@ + + + + + + + Browse file + + + + 0"> + Found tables: + + + + + + + {{ dataset.name }} + + + + + + + + + + + + Multi Dataset Load + + + + + + Please upload a file + + + + + + Selected file: {{ selectedFile?.name }} + Paste or type the list of datasets to upload: + + Auto detect + + + + Every row is one dataset. Format: LIBRARY.TABLE + + + + Discard file + Upload + + + + Matched datasets: + {{ matchedDataset }} + + + + + + + + + diff --git a/client/src/app/multi-dataset/multi-dataset.component.scss b/client/src/app/multi-dataset/multi-dataset.component.scss new file mode 100644 index 0000000..7127636 --- /dev/null +++ b/client/src/app/multi-dataset/multi-dataset.component.scss @@ -0,0 +1,27 @@ +.no-table-selected { + display: flex; + justify-content: center; + flex-direction: column; + align-items: center; + position: absolute; + background: var(--clr-vertical-nav-bg-color); + z-index: 10; + width: 100%; + height: 100%; + top: 0; +} + +.header-row { + padding: 15px 0; + border-bottom: 1px solid #d3d3d3; +} + +.dataset-input-wrapper { + max-width: 350px; + width: 100%; + + textarea { + min-height: 200px; + height: 200px; + } +} \ No newline at end of file diff --git a/client/src/app/multi-dataset/multi-dataset.component.ts b/client/src/app/multi-dataset/multi-dataset.component.ts new file mode 100644 index 0000000..1c85302 --- /dev/null +++ b/client/src/app/multi-dataset/multi-dataset.component.ts @@ -0,0 +1,237 @@ +import { + Component, + HostBinding, + OnInit, +} from '@angular/core' +import { ActivatedRoute, Router } from '@angular/router' +import { + EventService, + HelperService, + LicenceService, + LoggerService, + SasService, + SasStoreService +} from '../services' +import * as XLSX from '@sheet/crypto' +import { globals } from '../_globals' + +@Component({ + selector: 'app-multi-dataset', + templateUrl: './multi-dataset.component.html', + styleUrls: ['./multi-dataset.component.scss'] +}) +export class MultiDatasetComponent implements OnInit { + @HostBinding('class.content-container') contentContainerClass = true + + public licenceState = this.licenceService.licenceState + + public hotTableLicenseKey: string | undefined = undefined + public hotTableMaxRows = + this.licenceState.value.viewer_rows_allowed || Infinity + + public selectedFile: File | null = null + public datasets: any[] = [ + // { + // name: 'LIB1.TABLE21', + // status: 'error' + // }, + // { + // name: 'LIB1.BLEJA', + // status: 'success' + // }, + // { + // name: 'LIB1.NIDZA', + // status: 'success' + // } + ] + public datasetsLoading: boolean = false + + public matchedDatasets: string[] = [] + public userInputDatasets: string = '' + + public libsAndTables: { + [key: string]: string[] + } = {} + + constructor( + private eventService: EventService, + private licenceService: LicenceService, + private helperService: HelperService, + private loggerService: LoggerService, + private route: ActivatedRoute, + private router: Router, + private sasStoreService: SasStoreService, + private sasService: SasService + ) {} + + public afterGetColHeader(column: number, th: any) { + // Dark mode + th.classList.add('darkTH') + } + + ngOnInit() { + this.licenceService.hot_license_key.subscribe( + (hot_license_key: string | undefined) => { + this.hotTableLicenseKey = hot_license_key + } + ) + } + + ngAfterContentInit(): void { + if (globals.editor.startupSet) { + this.getFromGlobals() + } else { + this.eventService.onStartupDataLoaded.subscribe(() => { + this.getFromGlobals() + }) + } + } + + public getFromGlobals() { + this.libsAndTables = globals.editor.libsAndTables + } + + onFileChange(event: any) { + // if (!event?.target?.files[0]) { + // this.eventService.showAbortModal(null, 'No file found.', null, 'File Upload') + // return + // } + + // const file = event.target.files[0]; + // const fileTitle = file.name; + // const fileExtension = fileTitle.split('.').pop() + + // if (!['xlsx', 'xlsm', 'xlm'].includes(fileExtension)) { + // this.eventService.showAbortModal(null, 'Only excel extensions are allowed. (xlsx)', null, 'Extension Error') + // return + // } + + // this.selectedFile = event.target.files[0] + // event.target.value = '' // Reset the upload input + } + + onDiscardFile() { + this.selectedFile = null + this.userInputDatasets = '' + } + + onUploadFile() { + + } + + onUserInputDatasetsChange() { + this.helperService.debounceCall(500, () => { + const inputDatasets = this.userInputDatasets.split('\n') + + this.matchedDatasets = [] + + + inputDatasets.forEach((dataset: string) => { + const trimmedDataset = dataset.trim() + + if (this.isValidDatasetFormat(trimmedDataset) && this.isValidDatasetReference(trimmedDataset)) { + this.matchedDatasets.push(trimmedDataset) + } else { + console.warn(`Sheet name: ${trimmedDataset} is not an actual dataset reference.`) + } + }) + + console.log('this.matchedDatasets', this.matchedDatasets) + }) + } + + /** + * Try to find the datasets in the provided file by looking in the + * sheet names and testing if they are in the scope of the dataset naming + * convention. {@link isValidDatasetFormat} + */ + async onAutoDetectColumns() { + const sheetNames = await this.parseExcelSheetNames() + + if (sheetNames) { + this.matchedDatasets = [] + this.userInputDatasets = '' + + sheetNames.forEach((sheetName: string, index: number) => { + if (this.isValidDatasetFormat(sheetName) && this.isValidDatasetReference(sheetName)) { + this.matchedDatasets.push(sheetName) + } else { + console.warn(`Sheet name: ${sheetName} is not an actual dataset reference.`) + } + }) + } + + // Set matched datasets to textarea, dataset per row + this.userInputDatasets = this.matchedDatasets.join('\n') + } + + /** + * Valid dataset format includes: + * - Name must contain a single period (.) + * - First part (before period) can be no more than 8 chars + * - Second part (after period) can be no more than 32 chars + * - (start with letter or underscore, and contain only letters / underscores / numbers) + * - can't start with a number + * - both left and right parts must be valid variable names + * + * example: LIB123.TABLE_123 + */ + isValidDatasetFormat(sheetName: string) { + const regex = /^\w{1,8}\.\w{1,32}$/gmi + const correctFormat = regex.test(sheetName) + + return correctFormat + } + + /** + * Checks if @param datasetRef is valid variable which references library and table + */ + isValidDatasetReference(datasetRef: string) { + const library = datasetRef.split('.')[0] + const table = datasetRef.split('.')[1] + + const libTable = this.libsAndTables[library]?.includes(table) + + if (libTable) return true + + return false + } + + parseExcelSheetNames(): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + + if (!this.selectedFile) { + console.warn('selectedFile is missing') + return resolve([]) + } + + reader.onload = (event: ProgressEvent) => { + if (!event?.target) { + console.warn('File reader event.target is missing') + return + } + + const data = event.target.result; + const workbook = XLSX.read(data, { + type: 'binary' + }); + + try { + const sheet_names_list = workbook.SheetNames; + + return resolve(sheet_names_list) + } + catch (e) { + console.error(e) + } + }; + + reader.onerror = function (ex) { + console.log(ex); + }; + + reader.readAsBinaryString(this.selectedFile); + }) + } +} diff --git a/client/src/app/multi-dataset/multi-dataset.module.ts b/client/src/app/multi-dataset/multi-dataset.module.ts new file mode 100644 index 0000000..9767e91 --- /dev/null +++ b/client/src/app/multi-dataset/multi-dataset.module.ts @@ -0,0 +1,31 @@ +import { CommonModule } from '@angular/common' +import { NgModule } from '@angular/core' +import { FormsModule } from '@angular/forms' +import { ClarityModule } from '@clr/angular' +import { HotTableModule } from '@handsontable/angular' +import { registerAllModules } from 'handsontable/registry' +import { AppSharedModule } from '../app-shared.module' +import { DirectivesModule } from '../directives/directives.module' +import { DcTreeModule } from '../shared/dc-tree/dc-tree.module' +import { MultiDatasetComponent } from './multi-dataset.component' +import { MultiDatasetRoutingModule } from './multi-dataset-routing.module' +import { MultiDatasetRouteComponent } from '../routes/multi-dataset-route/multi-dataset-route.component' + +// register Handsontable's modules +registerAllModules() + +@NgModule({ + declarations: [MultiDatasetRouteComponent, MultiDatasetComponent], + imports: [ + HotTableModule, + MultiDatasetRoutingModule, + FormsModule, + ClarityModule, + AppSharedModule, + CommonModule, + DcTreeModule, + DirectivesModule + ], + exports: [MultiDatasetComponent] +}) +export class MultiDatasetModule {} diff --git a/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.html b/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.html new file mode 100644 index 0000000..0680b43 --- /dev/null +++ b/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.html @@ -0,0 +1 @@ + diff --git a/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.scss b/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.scss new file mode 100644 index 0000000..e69de29 diff --git a/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.ts b/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.ts new file mode 100644 index 0000000..44eddf2 --- /dev/null +++ b/client/src/app/routes/multi-dataset-route/multi-dataset-route.component.ts @@ -0,0 +1,17 @@ +import { Component, OnInit, OnDestroy } from '@angular/core' + +@Component({ + selector: 'app-multi-dataset-route', + templateUrl: './multi-dataset-route.component.html', + styleUrls: ['./multi-dataset-route.component.scss'], + host: { + class: 'content-container' + } +}) +export class MultiDatasetRouteComponent implements OnInit, OnDestroy { + constructor() {} + + ngOnInit() {} + + ngOnDestroy() {} +} diff --git a/client/src/app/services/spreadsheet.service.ts b/client/src/app/services/spreadsheet.service.ts new file mode 100644 index 0000000..a32f8df --- /dev/null +++ b/client/src/app/services/spreadsheet.service.ts @@ -0,0 +1,859 @@ +import { Injectable } from '@angular/core'; +import * as XLSX from '@sheet/crypto' +import { ExcelPasswordModalService, Result } from '../shared/excel-password-modal/excel-password-modal.service'; +import { EventService } from './event.service'; +import { isSpecialMissing } from '@sasjs/utils'; +import { dateFormat, dateToUtcTime, dateToTime } from '../editor/utils/date.utils'; +import { excelDateToJSDate, getMissingHeaders } from '../editor/utils/grid.utils'; +import { isStringNumber, isStringDecimal } from '../editor/utils/types.utils'; +import SheetInfo from '../models/SheetInfo'; +import { blobToFile } from '../xlmap/utils/file.utils'; +import { ExcelRule } from '../models/TableData'; +import { DcValidator } from '../shared/dc-validator/dc-validator'; +import { LicenceService } from './licence.service'; +import { FileUploadEncoding } from '../models/FileUploadEncoding'; +import { FileUploader } from '../models/FileUploader.class'; + +/** + * Used in combination with buffer + */ +const iconv = require('iconv-lite') +/** + * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again + * Which will be send to backend + */ +const Buffer = require('buffer/').Buffer +type AOA = any[][] + +export interface ParseParams { + file: File, + dcValidator: DcValidator + headerPks: string[] + headerArray: string[] + headerShow: string[] + timeHeaders: string[] + dateHeaders: string[] + dateTimeHeaders: string[] + xlRules: ExcelRule[] + encoding: FileUploadEncoding +} + +export interface ParseResult { + data: any[] + uploader: FileUploader +} + +@Injectable({ + providedIn: 'root' +}) +export class SpreadsheetService { + + private licenceState = this.licenceService.licenceState + + constructor( + private excelPasswordModalService: ExcelPasswordModalService, + private eventService: EventService, + private licenceService: LicenceService + ) { } + + /** + * Parses attached file and searches fo the matching data + * + * @param parseParams params required for parsing the file + * @param onParseStateChange callback used to inform about parsing state + * so the user of the function can update the UI with latest info + * + * @returns parsed list of files to upload and JSON data ready for HOT usage + */ + public parseExcelFile(parseParams: ParseParams, onParseStateChange: (uploadState: string) => void): Promise { + return new Promise((resolve, reject) => { + let data: any[] = [] + let uploader: FileUploader = new FileUploader() + + let file: File = parseParams.file + let filename = file.name + + onParseStateChange(`Loading ${filename} into the browser`) + + let foundData = { + sheet: '' + } + + let fileType = filename.slice( + filename.lastIndexOf('.') + 1, + filename.lastIndexOf('.') + 4 + ) + + if (fileType.toLowerCase() === 'xls') { + let reader: FileReader = new FileReader() + const self = this + reader.onload = async (theFile: any) => { + /* read workbook */ + const bstr = this.toBstr(theFile.target.result) + let wb: XLSX.WorkBook | undefined = undefined + let fileUnlocking: boolean = false + + const xlsxOptions: XLSX.ParsingOptions = { + type: 'binary', + cellDates: false, + cellFormula: true, + cellStyles: true, + cellNF: false, + cellText: false + } + + try { + wb = XLSX.read(bstr, { + ...xlsxOptions + }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + fileUnlocking = true + + while (fileUnlocking) { + const password = await this.promptExcelPassword() + + if (password) { + try { + wb = XLSX.read(bstr, { + ...xlsxOptions, + password: password + }) + + fileUnlocking = false + } catch (err: any) { + this.excelPasswordModalService.open({ + error: true + }) + + if (!err.message.toLowerCase().includes('password')) { + fileUnlocking = false + } + } + } else { + fileUnlocking = false + } + } + } else { + return reject('Error reading the file') + } + } + + if (!wb) { + // RETURN ERROR: NO WB FOUND + return reject('No workbook found.') + } + + /* save data */ + let isComplete: boolean = false + let missingHeaders: string[] = [] + + const csvArrayHeaders: string[] = [ + '_____DELETE__THIS__RECORD_____', + ...parseParams.headerArray + ] + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) + let csvArrayHeadersMap = csvArrayHeadersLower.reduce( + (map: any, obj: string) => { + map[obj] = -1 + return map + }, + {} + ) + + let csvArrayData: any[] = [] + const rangeSheetRes: SheetInfo = this.getRangeAndSheet(wb, parseParams) + missingHeaders = rangeSheetRes.missingHeaders + + if (rangeSheetRes.foundData) { + isComplete = true + csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap + const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName] + + onParseStateChange(`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`) + + let startAddress = '' + let endAddress = '' + + for ( + let row = rangeSheetRes.startRow; + row < rangeSheetRes.endRow; + ++row + ) { + const arr: any[] = [] + + csvArrayHeadersLower.forEach((x) => { + const col = csvArrayHeadersMap[x] + const addr = XLSX.utils.encode_cell({ + r: rangeSheetRes.rangeStartRow + row, + c: rangeSheetRes.rangeStartCol + col + }) + + if (startAddress === '') startAddress = addr + endAddress = addr + + let cell + + if (!ws[addr]) { + cell = { v: '' } + } else { + cell = ws[addr] + } + arr.push(cell) + }) + + // If we found at least one non empty value it means it is not empty row + // othervise, it is empty row + let arrNonEmptyValue = arr.find((x) => x.v !== '') + + if (arrNonEmptyValue) csvArrayData.push(arr) + } + + this.eventService.showInfoModal( + 'Table Found', + `Sheet: ${rangeSheetRes.sheetName}\nRange: ${startAddress}:${endAddress}` + ) + } else { + missingHeaders = rangeSheetRes.missingHeaders + } + + if (missingHeaders.length > 0) { + missingHeaders.sort(function compareSecondColumn(a, b) { + if (a[1] === b[1]) { + return 0 + } else { + return a[1] > b[1] ? -1 : 1 + } + }) + let abortMsg = missingHeaders + .map((x) => x[0]) + .slice(0, 5) + .join('\n') + + uploader.queue.pop() + return reject(abortMsg) + } + + // If first row is empty, that means no data has been found + if (csvArrayData.length === 0 || csvArrayData[0].length === 0) { + let abortMsg = 'No relevant data found in File !' + + uploader.queue.pop() + return reject(abortMsg) + } + + if ( + parseParams.dateTimeHeaders.length > 0 || + parseParams.dateHeaders.length > 0 || + parseParams.timeHeaders.length > 0 + ) { + csvArrayData = this.updateDateTimeCols(csvArrayHeaders, csvArrayData, parseParams) + } + + if (parseParams.xlRules.length > 0) { + csvArrayData = this.updateXLRuleCols(csvArrayHeaders, csvArrayData, parseParams) + } + + if (!isComplete) { + let abortMsg = '' + + if (missingHeaders.length === 0) { + abortMsg = 'No relevant data found in File !' + } else { + missingHeaders.sort(function compareSecondColumn(a, b) { + if (a[1] === b[1]) { + return 0 + } else { + return a[1] > b[1] ? -1 : 1 + } + }) + abortMsg = missingHeaders + .map((x) => x[0]) + .slice(0, 5) + .join('\n') + } + + // abort message is fired, return undefined + uploader.queue.pop() + return reject(abortMsg) + } else { + parseParams.headerShow = csvArrayHeaders + csvArrayData = csvArrayData.map((row: any) => + row.map((col: any) => (col.t === 'n' ? col.v : col.w)) + ) + + csvArrayData = csvArrayData.map((row: any) => { + return row.map((col: any, index: number) => { + if (!col && col !== 0) col = '' + + /** + * Keeping this for the reference + * Code below used to convert JSON to CSV + * now the XLSX is converting to CSV + */ + // if (isNaN(col)) { + // // Match and replace the double quotes, ignore the first and last char + // // in case they are double quotes already + // col = col.replace(/(? -1 || + // col.search(/\r|\n/g) > -1 + // ) { + // // Missing quotes at the end + // if (col.search(/"$/g) < 0) { + // col = col + '"' // So we add them + // } + + // // Missing quotes at the start + // if (col.search(/^"/g) < 0) { + // col = '"' + col // So we add them + // } + // } + // } + + const colName = parseParams.headerShow[index] + const colRule = parseParams.dcValidator?.getRule(colName) + + if (colRule?.type === 'numeric') { + if (isSpecialMissing(col) && !col.includes('.')) col = '.' + col + } + + return col + }) + }) + + data = csvArrayData + + // Apply licence rows limitation if exists, it is only affecting data + // which will be send to SAS + const strippedCsvArrayData = csvArrayData.slice( + 0, + this.licenceState.value.submit_rows_limit + ) + // To submit to sas service, we need clean version of CSV of file + // attached. XLSX will do the parsing and heavy lifting + // First we create worksheet of json (data we extracted) + let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { + skipHeader: true + }) + // create CSV to be uploaded from worksheet + let csvContentClean = XLSX.utils.sheet_to_csv(ws) + // Prepend headers + csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean + + if (parseParams.encoding === 'WLATIN1') { + let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252') + let blob = new Blob([encoded], { type: 'application/csv' }) + let newCSVFile: File = blobToFile(blob, filename + '.csv') + uploader.addToQueue([newCSVFile]) + } else { + let blob = new Blob([csvContentClean], { type: 'application/csv' }) + let newCSVFile: File = blobToFile(blob, filename + '.csv') + uploader.addToQueue([newCSVFile]) + } + } + + if (data.length === 0) { + return reject(`Table in the file is empty. Data found on sheet: ${foundData.sheet}`) + } + + return resolve({ + uploader, + data + }) + } + reader.readAsArrayBuffer(file) + } else if (fileType.toLowerCase() === 'csv') { + if (this.licenceState.value.submit_rows_limit !== Infinity) { + uploader.queue.pop() + return reject('Excel files only. To unlock CSV uploads, please contact support@datacontroller.io') + } + + if (parseParams.encoding === 'WLATIN1') { + let reader = new FileReader() + const self = this + // Closure to capture the file information. + reader.onload = (theFile: any) => { + let encoded = iconv.decode( + Buffer.from(theFile.target.result), + 'CP-1252' + ) + let blob = new Blob([encoded], { type: fileType }) + let encodedFile: File = blobToFile(blob, filename) + uploader.queue.pop() + uploader.addToQueue([encodedFile]) + + resolve({ + uploader, + data + }) + } + + reader.readAsArrayBuffer(file) + } else { + return resolve({ + uploader, + data + }) + } + } else { + let abortMsg = + 'Invalid file type "' + + filename + + '". Please upload csv or excel file.' + + uploader.queue.pop() + return reject(abortMsg) + } + }) + } + + /** + * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers + * @param wb Excel workbook + * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range + */ + private getRangeAndSheet(wb: XLSX.WorkBook, parseParams: ParseParams): SheetInfo { + let data = [] + + let rangeStartRow: number = 0 + let rangeStartCol: number = 0 + let startRow: number = -1 + let endRow: number = -1 + let sheetName: string = '' + let isComplete = false + let missingHeaders: string[] = [] + let csvArrayHeaders: string[] = [ + '_____DELETE__THIS__RECORD_____', + ...parseParams.headerArray + ] + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) + let csvArrayHeadersMap = csvArrayHeadersLower.reduce( + (map: any, obj: string) => { + map[obj] = -1 + return map + }, + {} + ) + + wb.SheetNames.forEach((element: string) => { + // Checking for required data in each sheet in workbook/ + if (isComplete) { + return + } + + missingHeaders = [] + sheetName = element + const ws: XLSX.WorkSheet = wb.Sheets[sheetName] + + data = XLSX.utils.sheet_to_json(ws, { + header: 1, + blankrows: false, + defval: '' + }) + + if (data.length <= 1) { + return + } + + let tempArr: string[] = [] + parseParams.headerArray.forEach(() => tempArr.push('')) + data.push(tempArr) + + let foundHeaders = false + + data.forEach((row: any, index: number) => { + if (isComplete) { + return + } + + if (foundHeaders) { + let isDataEnd = true + let isPkNull = false + + csvArrayHeadersLower.forEach((x) => { + const col = csvArrayHeadersMap[x] + + if (row[col] !== '' && row[col] !== undefined) { + isDataEnd = false + } else { + if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) { + isPkNull = true + } + } + }) + + if (isDataEnd || isPkNull) { + endRow = index + isComplete = true + } else { + if (startRow === -1) { + startRow = index + } + } + } else { + const rowLowerCase: string[] = row.map((x: any) => + x.toString().toLowerCase() + ) + + // If in file there is no delete column, remove it from search of missing. + // This way delete column will be optional to provide in file + if (!rowLowerCase.includes('_____delete__this__record_____')) { + const deleteIndex = csvArrayHeadersLower.indexOf( + '_____delete__this__record_____' + ) + + if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1) + } + + foundHeaders = true + + csvArrayHeadersLower.forEach((x) => { + if (rowLowerCase.indexOf(x) === -1) { + foundHeaders = false + } + }) + + let result = [] + + result = this.findValidHeaders( + rowLowerCase, + csvArrayHeadersLower, + index, + sheetName, + parseParams + ) + + if (result[0] === false) { + foundHeaders = false + + if (result[1].length > 0) { + result[1].forEach((data: string) => { + missingHeaders.push(data) + }) + } + } else { + csvArrayHeadersMap = result[1] + } + } + }) + + if (isComplete) { + this.update_sheet_range(ws) + const worksheetSel = ws['!ref'] + + if (worksheetSel) { + const range = XLSX.utils.decode_range(ws['!ref'] || '') + rangeStartRow = range.s.r + rangeStartCol = range.s.c + } + } + }) + + // If start row is still -1 that means first row of found range is empty + if (startRow === -1) isComplete = false + + const returnObj: SheetInfo = { + foundData: isComplete, + sheetName, + startRow, + endRow, + csvArrayHeadersMap, + missingHeaders, + rangeStartRow, + rangeStartCol + } + + return returnObj + } + + private findValidHeaders( + row: string[], + headers: string[], + rowNumber: number, + tabName: string, + parseParams: ParseParams + ): Array { + let headersFound = false + let missingErrorArray = [] + let j = 0 + + while (j < row.length) { + if (headersFound) { + // return; + } else { + if (headers.indexOf(row[j]) !== -1) { + let breakIndex + let rowStart = 0 + let rowEnd = 0 + let arrStart = 0 + let foundHeadersArray: string[] = [] + let spaceBreak = false + + for (let i = j; i < row.length; i++) { + if ( + row[i] === '' || + (foundHeadersArray.indexOf(row[i]) !== -1 && + this.isColHeader(row[i], parseParams.headerArray)) + ) { + if (row[i] === '') { + spaceBreak = true + } + + breakIndex = i + break + } else { + foundHeadersArray.push(row[i]) + } + } + + let tempArray: string[] = [] + + if (breakIndex !== undefined) { + tempArray = row.slice(j, breakIndex) + arrStart = j + rowEnd = breakIndex + + if (spaceBreak) { + rowStart = j + j = breakIndex + } else { + rowStart = j + j = breakIndex - 1 + } + } else { + tempArray = row.slice(j) + rowStart = j + arrStart = j + rowEnd = row.length + j = row.length + } + + let foundHeaders = true + + //We check if there are missing headers + headers.forEach((x) => { + if (tempArray.indexOf(x) === -1) { + foundHeaders = false + } + }) + + if (foundHeaders) { + headersFound = true + + let mapHeaders: any[] = headers + + let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) { + map[obj] = -1 + return map + }, {}) + + let temp = row.slice(rowStart, rowEnd) + + headers.forEach((x) => { + csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart + }) + + return [true, csvArrayHeadersMap] + } else { + let missingHeaders = getMissingHeaders(tempArray, headers) + + let missingMessage = 'TAB(' + tabName + ')' + missingErrorArray.push([ + missingMessage + + ' - ' + + missingHeaders[1].join(',') + + ' ( missing ' + + missingHeaders[0].join(',') + + ' )', + missingHeaders[1].length + ]) + } + } + } + j++ + } + return [false, missingErrorArray] + } + + private isColHeader(col: string, headerArray: string[]) { + return headerArray.indexOf(col.toUpperCase()) > -1 + } + + /** + * Function that updates the !ref range value provided in official docs. + * @param ws worksheet to be updated + */ + private update_sheet_range(ws: XLSX.WorkSheet) { + var range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } } + Object.keys(ws) + .filter(function (x) { + return x.charAt(0) != '!' + }) + .map(XLSX.utils.decode_cell) + .forEach(function (x: any) { + range.s.c = Math.min(range.s.c, x.c) + range.s.r = Math.min(range.s.r, x.r) + range.e.c = Math.max(range.e.c, x.c) + range.e.r = Math.max(range.e.r, x.r) + }) + + ws['!ref'] = XLSX.utils.encode_range(range) + } + + /** + * When excel is password protected we will display the password prompt for user to type password in. + * @returns Password user input or undefined if discarded by user + */ + private promptExcelPassword(): Promise { + return new Promise((resolve, reject) => { + this.excelPasswordModalService.open().subscribe((result: Result) => { + resolve(result.password) + }) + + // Focus the password field + // setTimeout(() => { + // const filePasswordInputElement: any = + // document.querySelector('#filePasswordInput') + // if (filePasswordInputElement) { + // filePasswordInputElement.focus() + // filePasswordInputElement.value = '' + // } + // }, 100) + + // this.filePasswordSubject.subscribe((password: string | undefined) => { + // this.fileUnlockError = false + + // if (password) { + // resolve(password) + // } else { + // resolve(undefined) + // } + // }) + }) + } + + private updateDateTimeCols(headers: any, data: any, parseParams: ParseParams) { + if (parseParams.dateHeaders.length > 0) { + let dateCols: number[] = [] + parseParams.dateHeaders.forEach((element) => { + if (headers.indexOf(element) !== -1) { + dateCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + dateCols.forEach((element) => { + const obj = row[element] + if (isStringNumber(obj.v)) { + const date = excelDateToJSDate(Number(obj.v)) + + obj.v = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + } else { + if (obj && obj.v && obj.v.toString().indexOf(':') === -1) { + const date = new Date(obj.v) + if (date.toUTCString() !== 'Invalid Date') { + obj.v = dateFormat(date) + } + } + } + row[element] = obj + }) + }) + } + if (parseParams.timeHeaders.length > 0) { + let timeCols: number[] = [] + parseParams.timeHeaders.forEach((element) => { + if (headers.indexOf(element) !== -1) { + timeCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + timeCols.forEach((element) => { + const obj = row[element] + if ( + isStringNumber(obj.v) || + isStringDecimal(obj.v) || + obj.v.includes('E-') + ) { + const date = excelDateToJSDate(Number(obj.v)) + + obj.v = dateToUtcTime(date) + } + row[element] = obj + }) + }) + } + if (parseParams.dateTimeHeaders.length > 0) { + let dateTimeCols: number[] = [] + parseParams.dateTimeHeaders.forEach((element) => { + if (headers.indexOf(element) !== -1) { + dateTimeCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + dateTimeCols.forEach((element) => { + const obj = row[element] + if (isStringNumber(obj.v) || isStringDecimal(obj.v)) { + let date = excelDateToJSDate(Number(obj.v)) + obj.v = dateFormat(date) + ' ' + dateToUtcTime(date) + } else { + if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) { + let str = obj.v.substring(0, obj.v.indexOf(':')) + str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1) + obj.v = str + } + let date = new Date(obj.v) + if (date.toUTCString() !== 'Invalid Date') { + obj.v = dateFormat(date) + ' ' + dateToTime(date) + } + } + row[element] = obj + }) + }) + } + return data + } + + private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) { + if (parseParams.xlRules.length > 0) { + const xlRuleCols: any = [] + parseParams.xlRules.forEach((element: any) => { + if (headers.indexOf(element.XL_COLUMN) !== -1) { + element['index'] = headers.indexOf(element.XL_COLUMN) + xlRuleCols.push(element) + } + }) + data.forEach((row: any[]) => { + xlRuleCols.forEach((element: any) => { + const obj = row[element.index] + if (element.XL_RULE === 'FORMULA') { + if ('f' in obj) { + if (obj['t'] === 'n') { + obj['v'] = '=' + obj['f'] + } else { + obj['w'] = '=' + obj['f'] + } + } + } + row[element] = obj + }) + }) + } + return data + } + + private toBstr(res: any) { + let bytes = new Uint8Array(res) + let binary = '' + let length = bytes.byteLength + for (let i = 0; i < length; i++) { + binary += String.fromCharCode(bytes[i]) + } + return binary + } +} diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.html b/client/src/app/shared/excel-password-modal/excel-password-modal.component.html new file mode 100644 index 0000000..a782a43 --- /dev/null +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.html @@ -0,0 +1,44 @@ + + + + Password Protected File + + + Please enter password: + + + + + \ No newline at end of file diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss b/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss new file mode 100644 index 0000000..e69de29 diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts b/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts new file mode 100644 index 0000000..1c76791 --- /dev/null +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts @@ -0,0 +1,22 @@ +import { Component } from '@angular/core'; +import { Observable } from 'rxjs'; +import { ExcelPasswordModalService, Options } from './excel-password-modal.service'; + +@Component({ + selector: 'app-excel-password-modal', + templateUrl: './excel-password-modal.component.html', + styleUrl: './excel-password-modal.component.scss' +}) +export class ExcelPasswordModalComponent { + options$: Observable; + + fileUnlockError: boolean = false + + constructor(private excelPasswordModalService: ExcelPasswordModalService) { + this.options$ = this.excelPasswordModalService.optionsSubject$; + } + + close(password?: string) { + this.excelPasswordModalService.close(password); + } +} diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts b/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts new file mode 100644 index 0000000..9d23f5b --- /dev/null +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts @@ -0,0 +1,47 @@ +import { Injectable } from '@angular/core'; +import { Subject, Observable } from 'rxjs'; + +export interface Options extends OpenOptions { + open: boolean +} + +export interface Result { + password: string | undefined +} + +export interface OpenOptions { + error?: boolean +} + +@Injectable({ + providedIn: 'root' +}) +export class ExcelPasswordModalService { + public optionsSubject$: Subject = new Subject() + public resultChange$: Subject = new Subject() + + constructor() { + } + + public open(openOptions?: OpenOptions): Observable { + this.optionsSubject$.next({ + open: true, + ...openOptions + }); + + this.resultChange$ = new Subject(); + return this.resultChange$.asObservable(); + } + + + close(password?: string) { + this.optionsSubject$.next({ + open: false + }); + + this.resultChange$.next({ + password + }); + this.resultChange$.complete(); + } +} diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index 5361ac4..353aec7 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -14,6 +14,7 @@ import { TermsComponent } from './terms/terms.component' import { DirectivesModule } from '../directives/directives.module' import { DatasetInfoComponent } from './dataset-info/dataset-info.component' import { ContactLinkComponent } from './contact-link/contact-link.component' +import { ExcelPasswordModalComponent } from './excel-password-modal/excel-password-modal.component' @NgModule({ imports: [ @@ -30,7 +31,8 @@ import { ContactLinkComponent } from './contact-link/contact-link.component' AlertsComponent, TermsComponent, DatasetInfoComponent, - ContactLinkComponent + ContactLinkComponent, + ExcelPasswordModalComponent ], exports: [ LoadingIndicatorComponent, @@ -39,7 +41,8 @@ import { ContactLinkComponent } from './contact-link/contact-link.component' AlertsComponent, TermsComponent, DatasetInfoComponent, - ContactLinkComponent + ContactLinkComponent, + ExcelPasswordModalComponent ], providers: [UserService, AlertsService] }) diff --git a/client/src/app/shared/sidebar/sidebar.component.html b/client/src/app/shared/sidebar/sidebar.component.html index 41d6cdf..67f6694 100644 --- a/client/src/app/shared/sidebar/sidebar.component.html +++ b/client/src/app/shared/sidebar/sidebar.component.html @@ -124,8 +124,11 @@ routerLinkActive="active" >Tables - FilesExcel Maps + Multi Load diff --git a/client/src/app/stage/stage.component.ts b/client/src/app/stage/stage.component.ts index 7ebf90b..475e6b4 100644 --- a/client/src/app/stage/stage.component.ts +++ b/client/src/app/stage/stage.component.ts @@ -66,7 +66,7 @@ export class StageComponent implements OnInit { ) if (xlmap) { const id = this.hotTable.data[0].XLMAP_ID - this.route.navigateByUrl('/home/files/' + id) + this.route.navigateByUrl('/home/excel-maps/' + id) } else { this.route.navigateByUrl('/editor/' + this.tableDetails.BASE_TABLE) } diff --git a/client/src/app/xlmap/xlmap.component.scss b/client/src/app/xlmap/xlmap.component.scss index c50af08..c7b9777 100644 --- a/client/src/app/xlmap/xlmap.component.scss +++ b/client/src/app/xlmap/xlmap.component.scss @@ -39,8 +39,6 @@ clr-tree-node button { } .content-area { - padding: 0.5rem !important; - display: flex; flex-direction: column; } diff --git a/client/src/app/xlmap/xlmap.component.ts b/client/src/app/xlmap/xlmap.component.ts index 4facf5e..efbec34 100644 --- a/client/src/app/xlmap/xlmap.component.ts +++ b/client/src/app/xlmap/xlmap.component.ts @@ -159,7 +159,7 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit { } this.selectedTab = Tabs.Rules this.viewXLMapRules() - this.router.navigateByUrl('/home/files/' + xlmap.id) + this.router.navigateByUrl('/home/excel-maps/' + xlmap.id) } } diff --git a/client/src/styles.scss b/client/src/styles.scss index c5752dd..b199cb6 100644 --- a/client/src/styles.scss +++ b/client/src/styles.scss @@ -200,6 +200,10 @@ body[cds-theme="light"] { width: 100%; } +.w-100-i { + width: 100% !important; +} + .w-40 { width: 40%; } @@ -288,6 +292,10 @@ body[cds-theme="light"] { margin-left: 5px; } +.ml-5-i { + margin-left: 5px !important; +} + .ml-10 { margin-left: 10px; }
Found tables:
Multi Dataset Load
+ Please upload a file +
Selected file: {{ selectedFile?.name }}
Paste or type the list of datasets to upload:
Matched datasets:
{{ matchedDataset }}
Please enter password: