diff --git a/client/angular.json b/client/angular.json index cc51584..4bec685 100644 --- a/client/angular.json +++ b/client/angular.json @@ -69,7 +69,8 @@ ], "scripts": [ "node_modules/marked/marked.min.js" - ] + ], + "webWorkerTsConfig": "tsconfig.worker.json" }, "configurations": { "production": { @@ -148,7 +149,8 @@ "src/styles.scss" ], "scripts": [], - "karmaConfig": "karma.conf.js" + "karmaConfig": "karma.conf.js", + "webWorkerTsConfig": "tsconfig.worker.json" } }, "lint": { diff --git a/client/src/app/editor/editor.component.ts b/client/src/app/editor/editor.component.ts index 7f145f4..26b2a1a 100644 --- a/client/src/app/editor/editor.component.ts +++ b/client/src/app/editor/editor.component.ts @@ -57,12 +57,10 @@ import { LicenceService } from '../services/licence.service' import * as numbro from 'numbro' import * as languages from 'numbro/dist/languages.min' import { FileUploadEncoding } from '../models/FileUploadEncoding' -import { - ParseResult, - SpreadsheetService -} from '../services/spreadsheet.service' +import { SpreadsheetService } from '../services/spreadsheet.service' import { UploadFileResponse } from '../models/UploadFile' import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' +import { ParseResult } from '../models/ParseResult.interface' @Component({ selector: 'app-editor', diff --git a/client/src/app/models/ParseParams.interface.ts b/client/src/app/models/ParseParams.interface.ts new file mode 100644 index 0000000..bae81fa --- /dev/null +++ b/client/src/app/models/ParseParams.interface.ts @@ -0,0 +1,23 @@ +import { DcValidator } from "../shared/dc-validator/dc-validator" +import { FileUploadEncoding } from "./FileUploadEncoding" +import { FileUploader } from "./FileUploader.class" +import { ExcelRule } from "./TableData" + +export interface ParseParams { + file: File + password?: string + dcValidator: DcValidator + /** + * Parse function will manipulate and return the uploader array which can be provided with files already in the queue + * Otherwise new empty instance will be created. + */ + uploader?: FileUploader + headerPks: string[] + headerArray: string[] + headerShow: string[] + timeHeaders: string[] + dateHeaders: string[] + dateTimeHeaders: string[] + xlRules: ExcelRule[] + encoding?: FileUploadEncoding +} \ No newline at end of file diff --git a/client/src/app/models/ParseResult.interface.ts b/client/src/app/models/ParseResult.interface.ts new file mode 100644 index 0000000..55e6cb0 --- /dev/null +++ b/client/src/app/models/ParseResult.interface.ts @@ -0,0 +1,15 @@ +import { FileUploader } from "./FileUploader.class" +import SheetInfo from "./SheetInfo" + +export interface ParseResult { + /** + * In case of CSV file, won't be returned + */ + data?: any[] + /** + * In case of CSV file, won't be returned + */ + headerShow?: string[] + rangeSheetRes?: SheetInfo + uploader: FileUploader +} \ No newline at end of file diff --git a/client/src/app/multi-dataset/multi-dataset.component.ts b/client/src/app/multi-dataset/multi-dataset.component.ts index 9e3cf58..a12b953 100644 --- a/client/src/app/multi-dataset/multi-dataset.component.ts +++ b/client/src/app/multi-dataset/multi-dataset.component.ts @@ -11,7 +11,6 @@ import { SasService, SasStoreService } from '../services' -import * as XLSX from '@sheet/crypto' import { globals } from '../_globals' import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model' import { DcValidator } from '../shared/dc-validator/dc-validator' @@ -19,7 +18,6 @@ import { ExcelRule } from '../models/TableData' import { HotTableInterface } from '../models/HotTable.interface' import { Col } from '../shared/dc-validator/models/col.model' import { - ParseResult, SpreadsheetService } from '../services/spreadsheet.service' import Handsontable from 'handsontable' @@ -31,6 +29,7 @@ import { ColumnSettings } from 'handsontable/settings' import { UploadFile } from '@sasjs/adapter' import { UploadFileResponse } from '../models/UploadFile' import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse' +import { ParseResult } from '../models/ParseResult.interface' @Component({ selector: 'app-multi-dataset', @@ -292,6 +291,7 @@ export class MultiDatasetComponent implements OnInit { this.spreadsheetService .parseExcelFile({ file: this.selectedFile!, + password: this.selectedFile!.password || undefined, dcValidator: parsedDataset.datasetInfo.dcValidator!, headerPks: parsedDataset.datasetInfo.headerPks, headerArray: parsedDataset.datasetInfo.headerArray, @@ -518,7 +518,23 @@ export class MultiDatasetComponent implements OnInit { * convention. {@link isValidDatasetFormat} */ async onAutoDetectColumns() { - this.sheetNames = await this.parseExcelSheetNames() + let passwordError = false + + await this.parseExcelSheetNames() + .then((sheetNames) => { + this.sheetNames = sheetNames + }) + .catch((err) => { + if (err.includes('password')) { + passwordError = true + } + }) + + if (passwordError) { + this.onDiscardFile() + this.eventService.showInfoModal('Locked file', 'We failed to unlock the file.') + return + } if (this.sheetNames) { this.matchedDatasets = [] @@ -824,42 +840,18 @@ export class MultiDatasetComponent implements OnInit { * * @returns sheet names in string array */ - private parseExcelSheetNames(): Promise { + private async parseExcelSheetNames(): Promise { return new Promise((resolve, reject) => { - const reader = new FileReader() + if (!this.selectedFile) return resolve([]) - if (!this.selectedFile) { - console.warn('selectedFile is missing') - return resolve([]) - } + this.spreadsheetService.parseExcelSheetNames(this.selectedFile) + .then((parsed) => { + if (parsed.password) this.selectedFile!.password = parsed.password - reader.onload = (event: ProgressEvent) => { - if (!event?.target) { - console.warn('File reader event.target is missing') - return - } - - const data = event.target.result - const workbook = XLSX.read(data, { - // Load file minimally to parse sheets - bookSheets: true, - type: 'binary' - }) - - try { - const sheet_names_list = workbook.SheetNames - - return resolve(sheet_names_list) - } catch (e) { - console.error(e) - } - } - - reader.onerror = function (ex) { - console.log(ex) - } - - reader.readAsBinaryString(this.selectedFile) + return resolve(parsed.sheetNames) + }).catch((err) => { + return reject(err) + }) }) } @@ -1035,5 +1027,6 @@ export interface SubmittedCsvDatasetResult { } export interface SelectedFile extends File { - sizeMB?: number + sizeMB?: number, + password?: string } \ No newline at end of file diff --git a/client/src/app/services/spreadsheet.service.ts b/client/src/app/services/spreadsheet.service.ts index 778882f..80f14f3 100644 --- a/client/src/app/services/spreadsheet.service.ts +++ b/client/src/app/services/spreadsheet.service.ts @@ -1,70 +1,15 @@ import { Injectable } from '@angular/core' -import * as XLSX from '@sheet/crypto' import { ExcelPasswordModalService, - Result } from '../shared/excel-password-modal/excel-password-modal.service' import { EventService } from './event.service' -import { isSpecialMissing } from '@sasjs/utils/input/validators' -import { - dateFormat, - dateToUtcTime, - dateToTime -} from '../editor/utils/date.utils' -import { - excelDateToJSDate, - getMissingHeaders -} from '../editor/utils/grid.utils' -import { isStringNumber, isStringDecimal } from '../editor/utils/types.utils' -import SheetInfo from '../models/SheetInfo' -import { blobToFile } from '../xlmap/utils/file.utils' -import { ExcelRule } from '../models/TableData' -import { DcValidator } from '../shared/dc-validator/dc-validator' import { LicenceService } from './licence.service' -import { FileUploadEncoding } from '../models/FileUploadEncoding' -import { FileUploader } from '../models/FileUploader.class' - -/** - * Used in combination with buffer - */ -const iconv = require('iconv-lite') -/** - * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again - * Which will be send to backend - */ -const Buffer = require('buffer/').Buffer -type AOA = any[][] - -export interface ParseParams { - file: File - dcValidator: DcValidator - /** - * Parse function will manipulate and return the uploader array which can be provided with files already in the queue - * Otherwise new empty instance will be created. - */ - uploader?: FileUploader - headerPks: string[] - headerArray: string[] - headerShow: string[] - timeHeaders: string[] - dateHeaders: string[] - dateTimeHeaders: string[] - xlRules: ExcelRule[] - encoding?: FileUploadEncoding -} - -export interface ParseResult { - /** - * In case of CSV file, won't be returned - */ - data?: any[] - /** - * In case of CSV file, won't be returned - */ - headerShow?: string[] - rangeSheetRes?: SheetInfo - uploader: FileUploader -} +import { SpreadsheetUtil } from '../shared/spreadsheet-util/spreadsheet-util' +import { ParseParams } from '../models/ParseParams.interface' +import { ParseResult } from '../models/ParseResult.interface' +import { OpenOptions } from '../shared/excel-password-modal/models/options.interface' +import { Result } from '../shared/excel-password-modal/models/result.interface' +import * as XLSX from '@sheet/crypto' @Injectable({ providedIn: 'root' @@ -76,831 +21,138 @@ export class SpreadsheetService { private excelPasswordModalService: ExcelPasswordModalService, private eventService: EventService, private licenceService: LicenceService - ) {} + ) { + + } - /** - * Parses attached file and searches fo the matching data - * - * @param parseParams params required for parsing the file - * @param onParseStateChange callback used to inform about parsing state - * so the user of the function can update the UI with latest info - * @param onTableFoundEvent callback fired when table range is found in the file - * - * @returns parsed list of files to upload and JSON data ready for HOT usage - */ public parseExcelFile( parseParams: ParseParams, onParseStateChange?: (uploadState: string) => void, onTableFoundEvent?: (info: string) => void ): Promise { - return new Promise((resolve, reject) => { - let data: any[] = [] - const uploader: FileUploader = parseParams.uploader || new FileUploader() - - const file: File = parseParams.file - const filename = file.name - - if (!parseParams.encoding) parseParams.encoding = 'UTF-8' - - if (onParseStateChange) - onParseStateChange(`Loading ${filename} into the browser`) - - let foundData = { - sheet: '' - } - - let fileType = filename.slice( - filename.lastIndexOf('.') + 1, - filename.lastIndexOf('.') + 4 - ) - - if (fileType.toLowerCase() === 'xls') { - let reader: FileReader = new FileReader() - const self = this - reader.onload = async (theFile: any) => { - /* read workbook */ - const bstr = this.toBstr(theFile.target.result) - let wb: XLSX.WorkBook | undefined = undefined - let fileUnlocking: boolean = false - - const xlsxOptions: XLSX.ParsingOptions = { - type: 'binary', - cellDates: false, - cellFormula: true, - cellStyles: true, - cellNF: false, - cellText: false - } - - try { - wb = XLSX.read(bstr, { - ...xlsxOptions - }) - } catch (err: any) { - if (err.message.toLowerCase().includes('password')) { - fileUnlocking = true - - while (fileUnlocking) { - const password = await this.promptExcelPassword() - - if (password) { - try { - wb = XLSX.read(bstr, { - ...xlsxOptions, - password: password - }) - - fileUnlocking = false - } catch (err: any) { - this.excelPasswordModalService.open({ - error: true - }) - - if (!err.message.toLowerCase().includes('password')) { - fileUnlocking = false - } - } - } else { - fileUnlocking = false - } - } - } else { - return reject('Error reading the file') - } - } - - if (!wb) { - return reject('No workbook found.') - } - - /* save data */ - let isComplete: boolean = false - let missingHeaders: string[] = [] - - const csvArrayHeaders: string[] = [ - '_____DELETE__THIS__RECORD_____', - ...parseParams.headerArray - ] - let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) - let csvArrayHeadersMap = csvArrayHeadersLower.reduce( - (map: any, obj: string) => { - map[obj] = -1 - return map - }, - {} - ) - - let csvArrayData: any[] = [] - const rangeSheetRes: SheetInfo = this.getRangeAndSheet( - wb, - parseParams - ) - missingHeaders = rangeSheetRes.missingHeaders - - if (rangeSheetRes.foundData) { - isComplete = true - csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap - const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName] - - if (onParseStateChange) - onParseStateChange( - `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}` - ) - - let startAddress = '' - let endAddress = '' - - for ( - let row = rangeSheetRes.startRow; - row < rangeSheetRes.endRow; - ++row - ) { - const arr: any[] = [] - - csvArrayHeadersLower.forEach((x) => { - const col = csvArrayHeadersMap[x] - const addr = XLSX.utils.encode_cell({ - r: rangeSheetRes.rangeStartRow + row, - c: rangeSheetRes.rangeStartCol + col - }) - - let cell - - if (!ws[addr]) { - cell = { v: '' } - } else { - cell = ws[addr] - } - - if (startAddress === '' && ws[addr]) startAddress = addr - endAddress = addr - - arr.push(cell) - }) - - // If we found at least one non empty value it means it is not empty row - // othervise, it is empty row - let arrNonEmptyValue = arr.find((x) => x.v !== '') - - if (arrNonEmptyValue) csvArrayData.push(arr) - } - - rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}` - - if (onTableFoundEvent) - onTableFoundEvent( - `Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}` - ) - } else { - missingHeaders = rangeSheetRes.missingHeaders - } - - if (missingHeaders.length > 0) { - missingHeaders.sort(function compareSecondColumn(a, b) { - if (a[1] === b[1]) { - return 0 - } else { - return a[1] > b[1] ? -1 : 1 - } - }) - let abortMsg = missingHeaders - .map((x) => x[0]) - .slice(0, 5) - .join('\n') - - uploader.queue.pop() - return reject(abortMsg) - } - - // If first row is empty, that means no data has been found - if (csvArrayData.length === 0 || csvArrayData[0].length === 0) { - let abortMsg = 'No relevant data found in File !' - - uploader.queue.pop() - return reject(abortMsg) - } - - if ( - parseParams.dateTimeHeaders.length > 0 || - parseParams.dateHeaders.length > 0 || - parseParams.timeHeaders.length > 0 - ) { - csvArrayData = this.updateDateTimeCols( - csvArrayHeaders, - csvArrayData, - parseParams - ) - } - - if (parseParams.xlRules.length > 0) { - csvArrayData = this.updateXLRuleCols( - csvArrayHeaders, - csvArrayData, - parseParams - ) - } - - if (!isComplete) { - let abortMsg = '' - - if (missingHeaders.length === 0) { - abortMsg = 'No relevant data found in File !' - } else { - missingHeaders.sort(function compareSecondColumn(a, b) { - if (a[1] === b[1]) { - return 0 - } else { - return a[1] > b[1] ? -1 : 1 - } - }) - abortMsg = missingHeaders - .map((x) => x[0]) - .slice(0, 5) - .join('\n') - } - - // abort message is fired, return undefined - uploader.queue.pop() - return reject(abortMsg) - } else { - parseParams.headerShow = csvArrayHeaders - csvArrayData = csvArrayData.map((row: any) => - row.map((col: any) => (col.t === 'n' ? col.v : col.w)) - ) - - csvArrayData = csvArrayData.map((row: any) => { - return row.map((col: any, index: number) => { - if (!col && col !== 0) col = '' - - /** - * Keeping this for the reference - * Code below used to convert JSON to CSV - * now the XLSX is converting to CSV - */ - // if (isNaN(col)) { - // // Match and replace the double quotes, ignore the first and last char - // // in case they are double quotes already - // col = col.replace(/(? -1 || - // col.search(/\r|\n/g) > -1 - // ) { - // // Missing quotes at the end - // if (col.search(/"$/g) < 0) { - // col = col + '"' // So we add them - // } - - // // Missing quotes at the start - // if (col.search(/^"/g) < 0) { - // col = '"' + col // So we add them - // } - // } - // } - - const colName = parseParams.headerShow[index] - const colRule = parseParams.dcValidator?.getRule(colName) - - if (colRule?.type === 'numeric') { - if (isSpecialMissing(col) && !col.includes('.')) - col = '.' + col - } - - return col - }) - }) - - data = csvArrayData - - // Apply licence rows limitation if exists, it is only affecting data - // which will be send to SAS - const strippedCsvArrayData = csvArrayData.slice( - 0, - this.licenceState.value.submit_rows_limit - ) - // To submit to sas service, we need clean version of CSV of file - // attached. XLSX will do the parsing and heavy lifting - // First we create worksheet of json (data we extracted) - let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { - skipHeader: true - }) - // create CSV to be uploaded from worksheet - let csvContentClean = XLSX.utils.sheet_to_csv(ws) - // Prepend headers - csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean - - // Blob from which CSV file will be created depending of the selected - // encoding - let blob: Blob - - if (parseParams.encoding === 'WLATIN1') { - // WLATIN1 - let encoded = iconv.decode( - Buffer.from(csvContentClean), - 'CP-1252' - ) - blob = new Blob([encoded], { type: 'application/csv' }) - } else { - // UTF-8 - blob = new Blob([csvContentClean], { type: 'application/csv' }) - } - - let newCSVFile: File = blobToFile(blob, filename + '.csv') - uploader.addToQueue([newCSVFile]) - } - - if (data.length === 0) { - return reject( - `Table in the file is empty. Data found on sheet: ${foundData.sheet}` - ) - } - - return resolve({ - uploader, - data, - rangeSheetRes, - headerShow: parseParams.headerShow - }) - } - reader.readAsArrayBuffer(file) - } else if (fileType.toLowerCase() === 'csv') { - if (this.licenceState.value.submit_rows_limit !== Infinity) { - uploader.queue.pop() - return reject( - 'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io' - ) - } - - if (parseParams.encoding === 'WLATIN1') { - let reader = new FileReader() - const self = this - // Closure to capture the file information. - reader.onload = (theFile: any) => { - let encoded = iconv.decode( - Buffer.from(theFile.target.result), - 'CP-1252' - ) - let blob = new Blob([encoded], { type: fileType }) - let encodedFile: File = blobToFile(blob, filename) - uploader.queue.pop() - uploader.addToQueue([encodedFile]) - - resolve({ - uploader - }) - } - - reader.readAsArrayBuffer(file) - } else { - return resolve({ - uploader - }) - } - } else { - let abortMsg = - 'Invalid file type "' + - filename + - '". Please upload csv or excel file.' - - uploader.queue.pop() - return reject(abortMsg) - } + const spreadSheetUtil = new SpreadsheetUtil({ + licenceState: this.licenceState }) - } - /** - * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers - * @param wb Excel workbook - * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range - */ - private getRangeAndSheet( - wb: XLSX.WorkBook, - parseParams: ParseParams - ): SheetInfo { - let data = [] - - let rangeStartRow: number = 0 - let rangeStartCol: number = 0 - let startRow: number = -1 - let endRow: number = -1 - let sheetName: string = '' - let isComplete = false - let missingHeaders: string[] = [] - const csvArrayHeaders: string[] = [ - '_____DELETE__THIS__RECORD_____', - ...parseParams.headerArray - ] - let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) - let csvArrayHeadersMap = csvArrayHeadersLower.reduce( - (map: any, obj: string) => { - map[obj] = -1 - return map - }, - {} + return spreadSheetUtil.parseExcelFile( + parseParams, + this.promptExcelPassword, + onParseStateChange, + onTableFoundEvent ) - - wb.SheetNames.forEach((element: string) => { - // Checking for required data in each sheet in workbook/ - if (isComplete) { - return - } - - missingHeaders = [] - sheetName = element - const ws: XLSX.WorkSheet = wb.Sheets[sheetName] - - data = XLSX.utils.sheet_to_json(ws, { - header: 1, - blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong - defval: '' - }) - - if (data.length <= 1) { - return - } - - let tempArr: string[] = [] - parseParams.headerArray.forEach(() => tempArr.push('')) - data.push(tempArr) - - let foundHeaders = false - - data.forEach((row: any, index: number) => { - if (isComplete) { - return - } - - if (foundHeaders) { - let isDataEnd = true - let isPkNull = false - - csvArrayHeadersLower.forEach((x) => { - const col = csvArrayHeadersMap[x] - - if (row[col] !== '' && row[col] !== undefined) { - isDataEnd = false - } else { - if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) { - isPkNull = true - } - } - }) - - if (isDataEnd || isPkNull) { - endRow = index - isComplete = true - } else { - if (startRow === -1) { - startRow = index - } - } - } else { - const rowLowerCase: string[] = row.map((x: any) => - x.toString().toLowerCase() - ) - - // If in file there is no delete column, remove it from search of missing. - // This way delete column will be optional to provide in file - if (!rowLowerCase.includes('_____delete__this__record_____')) { - const deleteIndex = csvArrayHeadersLower.indexOf( - '_____delete__this__record_____' - ) - - if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1) - } - - foundHeaders = true - - csvArrayHeadersLower.forEach((x) => { - if (rowLowerCase.indexOf(x) === -1) { - foundHeaders = false - } - }) - - let result = [] - - result = this.findValidHeaders( - rowLowerCase, - csvArrayHeadersLower, - index, - sheetName, - parseParams - ) - - if (result[0] === false) { - foundHeaders = false - - if (result[1].length > 0) { - result[1].forEach((data: string) => { - missingHeaders.push(data) - }) - } - } else { - csvArrayHeadersMap = result[1] - } - } - }) - - if (isComplete) { - this.update_sheet_range(ws) - const worksheetSel = ws['!ref'] - - if (worksheetSel) { - const range = XLSX.utils.decode_range(ws['!ref'] || '') - rangeStartRow = range.s.r - rangeStartCol = range.s.c - } - } - }) - - // If start row is still -1 that means first row of found range is empty - if (startRow === -1) isComplete = false - - const returnObj: SheetInfo = { - foundData: isComplete, - sheetName, - startRow, - endRow, - csvArrayHeadersMap, - missingHeaders, - rangeStartRow, - rangeStartCol - } - - return returnObj - } - - private findValidHeaders( - row: string[], - headers: string[], - rowNumber: number, - tabName: string, - parseParams: ParseParams - ): Array { - let headersFound = false - const missingErrorArray = [] - let j = 0 - - while (j < row.length) { - if (headersFound) { - // return; - } else { - if (headers.indexOf(row[j]) !== -1) { - let breakIndex - let rowStart = 0 - let rowEnd = 0 - let arrStart = 0 - let foundHeadersArray: string[] = [] - let spaceBreak = false - - for (let i = j; i < row.length; i++) { - if ( - row[i] === '' || - (foundHeadersArray.indexOf(row[i]) !== -1 && - this.isColHeader(row[i], parseParams.headerArray)) - ) { - if (row[i] === '') { - spaceBreak = true - } - - breakIndex = i - break - } else { - foundHeadersArray.push(row[i]) - } - } - - let tempArray: string[] = [] - - if (breakIndex !== undefined) { - tempArray = row.slice(j, breakIndex) - arrStart = j - rowEnd = breakIndex - - if (spaceBreak) { - rowStart = j - j = breakIndex - } else { - rowStart = j - j = breakIndex - 1 - } - } else { - tempArray = row.slice(j) - rowStart = j - arrStart = j - rowEnd = row.length - j = row.length - } - - let foundHeaders = true - - //We check if there are missing headers - headers.forEach((x) => { - if (tempArray.indexOf(x) === -1) { - foundHeaders = false - } - }) - - if (foundHeaders) { - headersFound = true - - let mapHeaders: any[] = headers - - let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) { - map[obj] = -1 - return map - }, {}) - - let temp = row.slice(rowStart, rowEnd) - - headers.forEach((x) => { - csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart - }) - - return [true, csvArrayHeadersMap] - } else { - let missingHeaders = getMissingHeaders(tempArray, headers) - - let missingMessage = 'TAB(' + tabName + ')' - missingErrorArray.push([ - missingMessage + - ' - ' + - missingHeaders[1].join(',') + - ' ( missing ' + - missingHeaders[0].join(',') + - ' )', - missingHeaders[1].length - ]) - } - } - } - j++ - } - return [false, missingErrorArray] - } - - private isColHeader(col: string, headerArray: string[]) { - return headerArray.indexOf(col.toUpperCase()) > -1 } /** - * Function that updates the !ref range value provided in official docs. - * @param ws worksheet to be updated - */ - private update_sheet_range(ws: XLSX.WorkSheet) { - const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } } + * Read the file minimally just to get the sheet names, not reading full file + * to help boost the performance + * + * @returns sheet names in string array + */ + public async parseExcelSheetNames(file: File): Promise<{ + sheetNames: string[], + password?: string + }> { + return new Promise((resolve, reject) => { + const reader = new FileReader() - Object.keys(ws) - .filter(function (x) { - return x.charAt(0) != '!' - }) - .map(XLSX.utils.decode_cell) - .forEach(function (x: any) { - range.s.c = Math.min(range.s.c, x.c) - range.s.r = Math.min(range.s.r, x.r) - range.e.c = Math.max(range.e.c, x.c) - range.e.r = Math.max(range.e.r, x.r) - }) + if (!file) { + console.warn('file is missing') + return resolve({ sheetNames: [] }) + } - ws['!ref'] = XLSX.utils.encode_range(range) + reader.onload = async (event: ProgressEvent) => { + if (!event?.target) { + console.warn('File reader event.target is missing') + return + } + + let wb: XLSX.WorkBook | undefined = undefined + let fileUnlocking: boolean = false + let password: string | undefined + const data = event.target.result + + try { + wb = XLSX.read(data, { + // Load file minimally to parse sheets + bookSheets: true, + type: 'binary' + }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + fileUnlocking = true + + let passwordError = false + + while (fileUnlocking) { + password = await this.promptExcelPassword({ + error: passwordError + }) + + if (password) { + try { + wb = XLSX.read(data, { + // Load file minimally to parse sheets + bookSheets: true, + type: 'binary', + password: password + }) + + fileUnlocking = false + passwordError = false + } catch (err: any) { + passwordError = true + + if (!err.message.toLowerCase().includes('password')) { + fileUnlocking = false + } + } + + if (!password) return reject('Invalid password, failed to decrypt the file') + } else { + fileUnlocking = false + return reject('No password provided') + } + } + } else { + return reject('Error reading the file') + } + } + + if (!wb) return reject('Error parsing the workbook') + + try { + const sheetNames = wb.SheetNames + + return resolve({ + sheetNames: sheetNames, + password: password + }) + } catch (e) { + console.error(e) + } + } + + reader.onerror = function (ex) { + console.log(ex) + } + + reader.readAsBinaryString(file) + }) + } + + public bytesToMB(size: number): number { + return parseFloat((size / (1024*1024)).toFixed(2)) } /** * When excel is password protected we will display the password prompt for user to type password in. * @returns Password user input or undefined if discarded by user */ - private promptExcelPassword(): Promise { + private promptExcelPassword = (options?: OpenOptions): Promise => { return new Promise((resolve, reject) => { - this.excelPasswordModalService.open().subscribe((result: Result) => { + this.excelPasswordModalService.open(options).subscribe((result: Result) => { resolve(result.password) }) }) } - - private updateDateTimeCols( - headers: any, - data: any, - parseParams: ParseParams - ) { - if (parseParams.dateHeaders.length > 0) { - const dateCols: number[] = [] - parseParams.dateHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - dateCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - dateCols.forEach((element) => { - const obj = row[element] - if (isStringNumber(obj.v)) { - const date = excelDateToJSDate(Number(obj.v)) - - obj.v = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - } else { - if (obj && obj.v && obj.v.toString().indexOf(':') === -1) { - const date = new Date(obj.v) - if (date.toUTCString() !== 'Invalid Date') { - obj.v = dateFormat(date) - } - } - } - row[element] = obj - }) - }) - } - if (parseParams.timeHeaders.length > 0) { - let timeCols: number[] = [] - parseParams.timeHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - timeCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - timeCols.forEach((element) => { - const obj = row[element] - if ( - isStringNumber(obj.v) || - isStringDecimal(obj.v) || - obj.v.includes('E-') - ) { - const date = excelDateToJSDate(Number(obj.v)) - - obj.v = dateToUtcTime(date) - } - row[element] = obj - }) - }) - } - if (parseParams.dateTimeHeaders.length > 0) { - let dateTimeCols: number[] = [] - parseParams.dateTimeHeaders.forEach((element) => { - if (headers.indexOf(element) !== -1) { - dateTimeCols.push(headers.indexOf(element)) - } - }) - data.forEach((row: any[]) => { - dateTimeCols.forEach((element) => { - const obj = row[element] - if (isStringNumber(obj.v) || isStringDecimal(obj.v)) { - const date = excelDateToJSDate(Number(obj.v)) - obj.v = dateFormat(date) + ' ' + dateToUtcTime(date) - } else { - if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) { - let str = obj.v.substring(0, obj.v.indexOf(':')) - str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1) - obj.v = str - } - const date = new Date(obj.v) - if (date.toUTCString() !== 'Invalid Date') { - obj.v = dateFormat(date) + ' ' + dateToTime(date) - } - } - row[element] = obj - }) - }) - } - return data - } - - private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) { - if (parseParams.xlRules.length > 0) { - const xlRuleCols: any = [] - parseParams.xlRules.forEach((element: any) => { - if (headers.indexOf(element.XL_COLUMN) !== -1) { - element['index'] = headers.indexOf(element.XL_COLUMN) - xlRuleCols.push(element) - } - }) - data.forEach((row: any[]) => { - xlRuleCols.forEach((element: any) => { - const obj = row[element.index] - if (element.XL_RULE === 'FORMULA') { - if ('f' in obj) { - if (obj['t'] === 'n') { - obj['v'] = '=' + obj['f'] - } else { - obj['w'] = '=' + obj['f'] - } - } - } - row[element] = obj - }) - }) - } - return data - } - - private toBstr(res: any) { - let bytes = new Uint8Array(res) - let binary = '' - let length = bytes.byteLength - for (let i = 0; i < length; i++) { - binary += String.fromCharCode(bytes[i]) - } - return binary - } } diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.html b/client/src/app/shared/excel-password-modal/excel-password-modal.component.html index 28a48f6..9bdea64 100644 --- a/client/src/app/shared/excel-password-modal/excel-password-modal.component.html +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.html @@ -1,4 +1,4 @@ - +
Please enter password:

-

- Sorry that didn't work, try again. -

- - +
+

+ Sorry that didn't work, try again. +

+
+ +
+ + +
- + diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss b/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss new file mode 100644 index 0000000..dddaa9f --- /dev/null +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.scss @@ -0,0 +1,18 @@ +.excel-password-root { + ::ng-deep { + .modal { + z-index: 1060; + } + } +} + +.modal-footer { + display: flex; + align-items: center; + justify-content: space-between; + + .buttons { + display: flex; + gap: 5px; + } +} \ No newline at end of file diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts b/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts index d78a5c8..c456363 100644 --- a/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.component.ts @@ -1,24 +1,26 @@ import { Component } from '@angular/core' import { Observable } from 'rxjs' import { - ExcelPasswordModalService, - Options + ExcelPasswordModalService } from './excel-password-modal.service' +import { Options } from './models/options.interface' @Component({ selector: 'app-excel-password-modal', + styleUrls: ['./excel-password-modal.component.scss'], templateUrl: './excel-password-modal.component.html' }) export class ExcelPasswordModalComponent { - options$: Observable + options$: Observable = this.excelPasswordModalService.optionsSubject$ fileUnlockError: boolean = false - constructor(private excelPasswordModalService: ExcelPasswordModalService) { - this.options$ = this.excelPasswordModalService.optionsSubject$ - } + passwordInput: string = '' + + constructor(private excelPasswordModalService: ExcelPasswordModalService) {} close(password?: string) { + this.passwordInput = '' this.excelPasswordModalService.close(password) } } diff --git a/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts b/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts index 6f0e398..880123b 100644 --- a/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts +++ b/client/src/app/shared/excel-password-modal/excel-password-modal.service.ts @@ -1,17 +1,7 @@ import { Injectable } from '@angular/core' import { Subject, Observable } from 'rxjs' - -export interface Options extends OpenOptions { - open: boolean -} - -export interface Result { - password: string | undefined -} - -export interface OpenOptions { - error?: boolean -} +import { OpenOptions, Options } from './models/options.interface' +import { Result } from './models/result.interface' @Injectable({ providedIn: 'root' diff --git a/client/src/app/shared/excel-password-modal/models/options.interface.ts b/client/src/app/shared/excel-password-modal/models/options.interface.ts new file mode 100644 index 0000000..ae8bff1 --- /dev/null +++ b/client/src/app/shared/excel-password-modal/models/options.interface.ts @@ -0,0 +1,7 @@ +export interface OpenOptions { + error?: boolean +} + +export interface Options extends OpenOptions { + open: boolean +} \ No newline at end of file diff --git a/client/src/app/shared/excel-password-modal/models/result.interface.ts b/client/src/app/shared/excel-password-modal/models/result.interface.ts new file mode 100644 index 0000000..b22b65d --- /dev/null +++ b/client/src/app/shared/excel-password-modal/models/result.interface.ts @@ -0,0 +1,3 @@ +export interface Result { + password: string | undefined +} \ No newline at end of file diff --git a/client/src/app/shared/spreadsheet-util/random.class.ts b/client/src/app/shared/spreadsheet-util/random.class.ts new file mode 100644 index 0000000..6aa1f46 --- /dev/null +++ b/client/src/app/shared/spreadsheet-util/random.class.ts @@ -0,0 +1,5 @@ +export class RandomClass { + constructor(jo: string) { + + } +} \ No newline at end of file diff --git a/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts new file mode 100644 index 0000000..792a97d --- /dev/null +++ b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts @@ -0,0 +1,904 @@ +import { isSpecialMissing } from '@sasjs/utils/input/validators' +import { dateFormat, dateToUtcTime, dateToTime } from "src/app/editor/utils/date.utils" +import { getMissingHeaders, excelDateToJSDate } from "src/app/editor/utils/grid.utils" +import { isStringNumber, isStringDecimal } from "src/app/editor/utils/types.utils" +import { FileUploader } from "src/app/models/FileUploader.class" +import SheetInfo from "src/app/models/SheetInfo" +import { blobToFile } from "src/app/xlmap/utils/file.utils" +import * as XLSX from '@sheet/crypto' +import { LicenceState } from "src/app/models/LicenceState" +import { BehaviorSubject } from "rxjs" +import { ParseParams } from 'src/app/models/ParseParams.interface' +import { ParseResult } from 'src/app/models/ParseResult.interface' +import { OpenOptions } from '../excel-password-modal/models/options.interface' + +/** + * Used in combination with buffer + */ +import * as iconv from 'iconv-lite' +/** + * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again + * Which will be send to backend + */ +import { Buffer } from 'buffer' +type AOA = any[][] + +export interface ConstructorParams { + licenceState: BehaviorSubject +} + +export class SpreadsheetUtil { + private licenceState: BehaviorSubject + + constructor(params: ConstructorParams) { + this.licenceState = params.licenceState + } + + /** + * Parses attached file and searches fo the matching data + * + * @param parseParams params required for parsing the file + * @param onParseStateChange callback used to inform about parsing state + * so the user of the function can update the UI with latest info + * @param onTableFoundEvent callback fired when table range is found in the file + * + * @returns parsed list of files to upload and JSON data ready for HOT usage + */ + public parseExcelFile( + parseParams: ParseParams, + promptExcelPassword: (options?: OpenOptions) => Promise, + onParseStateChange?: (uploadState: string) => void, + onTableFoundEvent?: (info: string) => void + ): Promise { + return new Promise((resolve, reject) => { + // If file size is bigger then 2 MB we need to load its bytes in chunks + const sizeInMB = this.bytesToMB(parseParams.file.size) + + if (sizeInMB > 2) { + + } + + let data: any[] = [] + const uploader: FileUploader = parseParams.uploader || new FileUploader() + + const file: File = parseParams.file + const filename = file.name + + if (!parseParams.encoding) parseParams.encoding = 'UTF-8' + + if (onParseStateChange) + onParseStateChange(`Loading ${filename} into the browser`) + + let foundData = { + sheet: '' + } + + let fileType = filename.slice( + filename.lastIndexOf('.') + 1, + filename.lastIndexOf('.') + 4 + ) + + if (fileType.toLowerCase() === 'xls') { + let reader: FileReader = new FileReader() + + const self = this + reader.onload = async (theFile: any) => { + /* read workbook */ + const bstr = this.toBstr(theFile.target.result) + let wb: XLSX.WorkBook | undefined = undefined + let fileUnlocking: boolean = false + + const xlsxOptions: XLSX.ParsingOptions = { + type: 'binary', + cellDates: false, + cellFormula: true, + cellStyles: true, + cellNF: false, + cellText: false, + password: parseParams.password + } + + try { + // wb = this.xlsxRead(bstr, { + // ...xlsxOptions + // }) + wb = XLSX.read(bstr, { + ...xlsxOptions + }) + } catch (err: any) { + if (err.message.toLowerCase().includes('password')) { + fileUnlocking = true + + let passwordError = false + + while (fileUnlocking) { + const password = await promptExcelPassword({ + error: passwordError + }) + + if (password) { + try { + wb = XLSX.read(bstr, { + ...xlsxOptions, + password: password + }) + + fileUnlocking = false + passwordError = false + } catch (err: any) { + passwordError = true + + if (!err.message.toLowerCase().includes('password')) { + fileUnlocking = false + } + } + } else { + fileUnlocking = false + } + } + } else { + return reject('Error reading the file') + } + } + + if (!wb) { + return reject('No workbook found.') + } + + /* save data */ + let isComplete: boolean = false + let missingHeaders: string[] = [] + + const csvArrayHeaders: string[] = [ + '_____DELETE__THIS__RECORD_____', + ...parseParams.headerArray + ] + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) + let csvArrayHeadersMap = csvArrayHeadersLower.reduce( + (map: any, obj: string) => { + map[obj] = -1 + return map + }, + {} + ) + + let csvArrayData: any[] = [] + const rangeSheetRes: SheetInfo = this.getRangeAndSheet( + wb, + parseParams + ) + missingHeaders = rangeSheetRes.missingHeaders + + if (rangeSheetRes.foundData) { + isComplete = true + csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap + const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName] + + if (onParseStateChange) + onParseStateChange( + `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}` + ) + + let startAddress = '' + let endAddress = '' + + for ( + let row = rangeSheetRes.startRow; + row < rangeSheetRes.endRow; + ++row + ) { + const arr: any[] = [] + + csvArrayHeadersLower.forEach((x) => { + const col = csvArrayHeadersMap[x] + const addr = XLSX.utils.encode_cell({ + r: rangeSheetRes.rangeStartRow + row, + c: rangeSheetRes.rangeStartCol + col + }) + + let cell + + if (!ws[addr]) { + cell = { v: '' } + } else { + cell = ws[addr] + } + + if (startAddress === '' && ws[addr]) startAddress = addr + endAddress = addr + + arr.push(cell) + }) + + // If we found at least one non empty value it means it is not empty row + // othervise, it is empty row + let arrNonEmptyValue = arr.find((x) => x.v !== '') + + if (arrNonEmptyValue) csvArrayData.push(arr) + } + + rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}` + + if (onTableFoundEvent) + onTableFoundEvent( + `Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}` + ) + } else { + missingHeaders = rangeSheetRes.missingHeaders + } + + if (missingHeaders.length > 0) { + missingHeaders.sort(function compareSecondColumn(a, b) { + if (a[1] === b[1]) { + return 0 + } else { + return a[1] > b[1] ? -1 : 1 + } + }) + let abortMsg = missingHeaders + .map((x) => x[0]) + .slice(0, 5) + .join('\n') + + uploader.queue.pop() + return reject(abortMsg) + } + + // If first row is empty, that means no data has been found + if (csvArrayData.length === 0 || csvArrayData[0].length === 0) { + let abortMsg = 'No relevant data found in File !' + + uploader.queue.pop() + return reject(abortMsg) + } + + if ( + parseParams.dateTimeHeaders.length > 0 || + parseParams.dateHeaders.length > 0 || + parseParams.timeHeaders.length > 0 + ) { + csvArrayData = this.updateDateTimeCols( + csvArrayHeaders, + csvArrayData, + parseParams + ) + } + + if (parseParams.xlRules.length > 0) { + csvArrayData = this.updateXLRuleCols( + csvArrayHeaders, + csvArrayData, + parseParams + ) + } + + if (!isComplete) { + let abortMsg = '' + + if (missingHeaders.length === 0) { + abortMsg = 'No relevant data found in File !' + } else { + missingHeaders.sort(function compareSecondColumn(a, b) { + if (a[1] === b[1]) { + return 0 + } else { + return a[1] > b[1] ? -1 : 1 + } + }) + abortMsg = missingHeaders + .map((x) => x[0]) + .slice(0, 5) + .join('\n') + } + + // abort message is fired, return undefined + uploader.queue.pop() + return reject(abortMsg) + } else { + parseParams.headerShow = csvArrayHeaders + csvArrayData = csvArrayData.map((row: any) => + row.map((col: any) => (col.t === 'n' ? col.v : col.w)) + ) + + csvArrayData = csvArrayData.map((row: any) => { + return row.map((col: any, index: number) => { + if (!col && col !== 0) col = '' + + /** + * Keeping this for the reference + * Code below used to convert JSON to CSV + * now the XLSX is converting to CSV + */ + // if (isNaN(col)) { + // // Match and replace the double quotes, ignore the first and last char + // // in case they are double quotes already + // col = col.replace(/(? -1 || + // col.search(/\r|\n/g) > -1 + // ) { + // // Missing quotes at the end + // if (col.search(/"$/g) < 0) { + // col = col + '"' // So we add them + // } + + // // Missing quotes at the start + // if (col.search(/^"/g) < 0) { + // col = '"' + col // So we add them + // } + // } + // } + + const colName = parseParams.headerShow[index] + const colRule = parseParams.dcValidator?.getRule(colName) + + if (colRule?.type === 'numeric') { + if (isSpecialMissing(col) && !col.includes('.')) + col = '.' + col + } + + return col + }) + }) + + data = csvArrayData + + // Apply licence rows limitation if exists, it is only affecting data + // which will be send to SAS + const strippedCsvArrayData = csvArrayData.slice( + 0, + this.licenceState.value.submit_rows_limit + ) + // To submit to sas service, we need clean version of CSV of file + // attached. XLSX will do the parsing and heavy lifting + // First we create worksheet of json (data we extracted) + let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, { + skipHeader: true + }) + // create CSV to be uploaded from worksheet + let csvContentClean = XLSX.utils.sheet_to_csv(ws) + // Prepend headers + csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean + + // Blob from which CSV file will be created depending of the selected + // encoding + let blob: Blob + + if (parseParams.encoding === 'WLATIN1') { + // WLATIN1 + let encoded = iconv.decode( + Buffer.from(csvContentClean), + 'CP-1252' + ) + blob = new Blob([encoded], { type: 'application/csv' }) + } else { + // UTF-8 + blob = new Blob([csvContentClean], { type: 'application/csv' }) + } + + let newCSVFile: File = blobToFile(blob, filename + '.csv') + uploader.addToQueue([newCSVFile]) + } + + if (data.length === 0) { + return reject( + `Table in the file is empty. Data found on sheet: ${foundData.sheet}` + ) + } + + return resolve({ + uploader, + data, + rangeSheetRes, + headerShow: parseParams.headerShow + }) + } + console.log('before read file as array buffer') + reader.readAsArrayBuffer(file) + } else if (fileType.toLowerCase() === 'csv') { + if (this.licenceState.value.submit_rows_limit !== Infinity) { + uploader.queue.pop() + return reject( + 'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io' + ) + } + + if (parseParams.encoding === 'WLATIN1') { + let reader = new FileReader() + const self = this + // Closure to capture the file information. + reader.onload = (theFile: any) => { + let encoded = iconv.decode( + Buffer.from(theFile.target.result), + 'CP-1252' + ) + let blob = new Blob([encoded], { type: fileType }) + let encodedFile: File = blobToFile(blob, filename) + uploader.queue.pop() + uploader.addToQueue([encodedFile]) + + resolve({ + uploader + }) + } + + reader.readAsArrayBuffer(file) + } else { + return resolve({ + uploader + }) + } + } else { + let abortMsg = + 'Invalid file type "' + + filename + + '". Please upload csv or excel file.' + + uploader.queue.pop() + return reject(abortMsg) + } + }) + } + + public bytesToMB(size: number): number { + return parseFloat((size / (1024*1024)).toFixed(2)) + } + + private xlsxRead(data: any, opts?: XLSX.ParsingOptions | undefined): XLSX.WorkBook { + if (typeof Worker !== 'undefined') { + // Create a new + const worker = new Worker(new URL('../../spreadsheet.worker', import.meta.url)); + worker.onmessage = ({ data }) => { + + }; + + worker.postMessage({ + data, + opts + }); + + return XLSX.read(data, opts) // Put in worker + } else { + // Web workers are not supported in this environment. + // You should add a fallback so that your program still executes correctly. + return XLSX.read(data, opts) + } + } + + /** + * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers + * @param wb Excel workbook + * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range + */ + private getRangeAndSheet( + wb: XLSX.WorkBook, + parseParams: ParseParams + ): SheetInfo { + let data = [] + + let rangeStartRow: number = 0 + let rangeStartCol: number = 0 + let startRow: number = -1 + let endRow: number = -1 + let sheetName: string = '' + let isComplete = false + let missingHeaders: string[] = [] + const csvArrayHeaders: string[] = [ + '_____DELETE__THIS__RECORD_____', + ...parseParams.headerArray + ] + let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase()) + let csvArrayHeadersMap = csvArrayHeadersLower.reduce( + (map: any, obj: string) => { + map[obj] = -1 + return map + }, + {} + ) + + wb.SheetNames.forEach((element: string) => { + // Checking for required data in each sheet in workbook/ + if (isComplete) { + return + } + + missingHeaders = [] + sheetName = element + const ws: XLSX.WorkSheet = wb.Sheets[sheetName] + + data = XLSX.utils.sheet_to_json(ws, { + header: 1, + blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong + defval: '' + }) + + if (data.length <= 1) { + return + } + + let tempArr: string[] = [] + parseParams.headerArray.forEach(() => tempArr.push('')) + data.push(tempArr) + + let foundHeaders = false + + data.forEach((row: any, index: number) => { + if (isComplete) { + return + } + + if (foundHeaders) { + let isDataEnd = true + let isPkNull = false + + csvArrayHeadersLower.forEach((x) => { + const col = csvArrayHeadersMap[x] + + if (row[col] !== '' && row[col] !== undefined) { + isDataEnd = false + } else { + if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) { + isPkNull = true + } + } + }) + + if (isDataEnd || isPkNull) { + endRow = index + isComplete = true + } else { + if (startRow === -1) { + startRow = index + } + } + } else { + const rowLowerCase: string[] = row.map((x: any) => + x.toString().toLowerCase() + ) + + // If in file there is no delete column, remove it from search of missing. + // This way delete column will be optional to provide in file + if (!rowLowerCase.includes('_____delete__this__record_____')) { + const deleteIndex = csvArrayHeadersLower.indexOf( + '_____delete__this__record_____' + ) + + if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1) + } + + foundHeaders = true + + csvArrayHeadersLower.forEach((x) => { + if (rowLowerCase.indexOf(x) === -1) { + foundHeaders = false + } + }) + + let result = [] + + result = this.findValidHeaders( + rowLowerCase, + csvArrayHeadersLower, + index, + sheetName, + parseParams + ) + + if (result[0] === false) { + foundHeaders = false + + if (result[1].length > 0) { + result[1].forEach((data: string) => { + missingHeaders.push(data) + }) + } + } else { + csvArrayHeadersMap = result[1] + } + } + }) + + if (isComplete) { + this.update_sheet_range(ws) + const worksheetSel = ws['!ref'] + + if (worksheetSel) { + const range = XLSX.utils.decode_range(ws['!ref'] || '') + rangeStartRow = range.s.r + rangeStartCol = range.s.c + } + } + }) + + // If start row is still -1 that means first row of found range is empty + if (startRow === -1) isComplete = false + + const returnObj: SheetInfo = { + foundData: isComplete, + sheetName, + startRow, + endRow, + csvArrayHeadersMap, + missingHeaders, + rangeStartRow, + rangeStartCol + } + + return returnObj + } + + private findValidHeaders( + row: string[], + headers: string[], + rowNumber: number, + tabName: string, + parseParams: ParseParams + ): Array { + let headersFound = false + const missingErrorArray = [] + let j = 0 + + while (j < row.length) { + if (headersFound) { + // return; + } else { + if (headers.indexOf(row[j]) !== -1) { + let breakIndex + let rowStart = 0 + let rowEnd = 0 + let arrStart = 0 + let foundHeadersArray: string[] = [] + let spaceBreak = false + + for (let i = j; i < row.length; i++) { + if ( + row[i] === '' || + (foundHeadersArray.indexOf(row[i]) !== -1 && + this.isColHeader(row[i], parseParams.headerArray)) + ) { + if (row[i] === '') { + spaceBreak = true + } + + breakIndex = i + break + } else { + foundHeadersArray.push(row[i]) + } + } + + let tempArray: string[] = [] + + if (breakIndex !== undefined) { + tempArray = row.slice(j, breakIndex) + arrStart = j + rowEnd = breakIndex + + if (spaceBreak) { + rowStart = j + j = breakIndex + } else { + rowStart = j + j = breakIndex - 1 + } + } else { + tempArray = row.slice(j) + rowStart = j + arrStart = j + rowEnd = row.length + j = row.length + } + + let foundHeaders = true + + //We check if there are missing headers + headers.forEach((x) => { + if (tempArray.indexOf(x) === -1) { + foundHeaders = false + } + }) + + if (foundHeaders) { + headersFound = true + + let mapHeaders: any[] = headers + + let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) { + map[obj] = -1 + return map + }, {}) + + let temp = row.slice(rowStart, rowEnd) + + headers.forEach((x) => { + csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart + }) + + return [true, csvArrayHeadersMap] + } else { + let missingHeaders = getMissingHeaders(tempArray, headers) + + let missingMessage = 'TAB(' + tabName + ')' + missingErrorArray.push([ + missingMessage + + ' - ' + + missingHeaders[1].join(',') + + ' ( missing ' + + missingHeaders[0].join(',') + + ' )', + missingHeaders[1].length + ]) + } + } + } + j++ + } + return [false, missingErrorArray] + } + + private isColHeader(col: string, headerArray: string[]) { + return headerArray.indexOf(col.toUpperCase()) > -1 + } + + /** + * Function that updates the !ref range value provided in official docs. + * @param ws worksheet to be updated + */ + private update_sheet_range(ws: XLSX.WorkSheet) { + const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } } + + Object.keys(ws) + .filter(function (x) { + return x.charAt(0) != '!' + }) + .map(XLSX.utils.decode_cell) + .forEach(function (x: any) { + range.s.c = Math.min(range.s.c, x.c) + range.s.r = Math.min(range.s.r, x.r) + range.e.c = Math.max(range.e.c, x.c) + range.e.r = Math.max(range.e.r, x.r) + }) + + ws['!ref'] = XLSX.utils.encode_range(range) + } + + /** + * When excel is password protected we will display the password prompt for user to type password in. + * @returns Password user input or undefined if discarded by user + */ + // private promptExcelPassword(): Promise { + // return new Promise((resolve, reject) => { + // this.excelPasswordModalService.open().subscribe((result: Result) => { + // resolve(result.password) + // }) + // }) + // } + + private updateDateTimeCols( + headers: any, + data: any, + parseParams: ParseParams + ) { + if (parseParams.dateHeaders.length > 0) { + const dateCols: number[] = [] + parseParams.dateHeaders.forEach((element: any) => { + if (headers.indexOf(element) !== -1) { + dateCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + dateCols.forEach((element) => { + const obj = row[element] + if (isStringNumber(obj.v)) { + const date = excelDateToJSDate(Number(obj.v)) + + obj.v = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + } else { + if (obj && obj.v && obj.v.toString().indexOf(':') === -1) { + const date = new Date(obj.v) + if (date.toUTCString() !== 'Invalid Date') { + obj.v = dateFormat(date) + } + } + } + row[element] = obj + }) + }) + } + if (parseParams.timeHeaders.length > 0) { + let timeCols: number[] = [] + parseParams.timeHeaders.forEach((element: any) => { + if (headers.indexOf(element) !== -1) { + timeCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + timeCols.forEach((element) => { + const obj = row[element] + if ( + isStringNumber(obj.v) || + isStringDecimal(obj.v) || + obj.v.includes('E-') + ) { + const date = excelDateToJSDate(Number(obj.v)) + + obj.v = dateToUtcTime(date) + } + row[element] = obj + }) + }) + } + if (parseParams.dateTimeHeaders.length > 0) { + let dateTimeCols: number[] = [] + parseParams.dateTimeHeaders.forEach((element: any) => { + if (headers.indexOf(element) !== -1) { + dateTimeCols.push(headers.indexOf(element)) + } + }) + data.forEach((row: any[]) => { + dateTimeCols.forEach((element) => { + const obj = row[element] + if (isStringNumber(obj.v) || isStringDecimal(obj.v)) { + const date = excelDateToJSDate(Number(obj.v)) + obj.v = dateFormat(date) + ' ' + dateToUtcTime(date) + } else { + if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) { + let str = obj.v.substring(0, obj.v.indexOf(':')) + str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1) + obj.v = str + } + const date = new Date(obj.v) + if (date.toUTCString() !== 'Invalid Date') { + obj.v = dateFormat(date) + ' ' + dateToTime(date) + } + } + row[element] = obj + }) + }) + } + return data + } + + private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) { + if (parseParams.xlRules.length > 0) { + const xlRuleCols: any = [] + parseParams.xlRules.forEach((element: any) => { + if (headers.indexOf(element.XL_COLUMN) !== -1) { + element['index'] = headers.indexOf(element.XL_COLUMN) + xlRuleCols.push(element) + } + }) + data.forEach((row: any[]) => { + xlRuleCols.forEach((element: any) => { + const obj = row[element.index] + if (element.XL_RULE === 'FORMULA') { + if ('f' in obj) { + if (obj['t'] === 'n') { + obj['v'] = '=' + obj['f'] + } else { + obj['w'] = '=' + obj['f'] + } + } + } + row[element] = obj + }) + }) + } + return data + } + + private toBstr(res: any) { + let bytes = new Uint8Array(res) + let binary = '' + let length = bytes.byteLength + for (let i = 0; i < length; i++) { + binary += String.fromCharCode(bytes[i]) + } + return binary + } +} \ No newline at end of file diff --git a/client/src/app/spreadsheet.worker.ts b/client/src/app/spreadsheet.worker.ts new file mode 100644 index 0000000..a79e2b5 --- /dev/null +++ b/client/src/app/spreadsheet.worker.ts @@ -0,0 +1,12 @@ +/// + +import * as XLSX from '@sheet/crypto' + +addEventListener('message', ({ data }) => { + const input = data as { + data: any, + opts?: XLSX.ParsingOptions | undefined + } + + console.log('input', input) +}); \ No newline at end of file diff --git a/client/tsconfig.worker.json b/client/tsconfig.worker.json new file mode 100644 index 0000000..22dc454 --- /dev/null +++ b/client/tsconfig.worker.json @@ -0,0 +1,15 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/worker", + "lib": [ + "es2018", + "webworker" + ], + "types": [] + }, + "include": [ + "src/**/*.worker.ts" + ] +}