fix: excel with password flow, introducing web worker for XLSX.read

client/angular.json

@@ -69,7 +69,8 @@
             ],
             "scripts": [
               "node_modules/marked/marked.min.js"
-            ]
+            ],
+            "webWorkerTsConfig": "tsconfig.worker.json"
           },
           "configurations": {
             "production": {
@@ -148,7 +149,8 @@
               "src/styles.scss"
             ],
             "scripts": [],
-            "karmaConfig": "karma.conf.js"
+            "karmaConfig": "karma.conf.js",
+            "webWorkerTsConfig": "tsconfig.worker.json"
           }
         },
         "lint": {

client/src/app/editor/editor.component.ts

@@ -57,12 +57,10 @@ import { LicenceService } from '../services/licence.service'
 import * as numbro from 'numbro'
 import * as languages from 'numbro/dist/languages.min'
 import { FileUploadEncoding } from '../models/FileUploadEncoding'
-import {
+import { SpreadsheetService } from '../services/spreadsheet.service'
-  ParseResult,
-  SpreadsheetService
-} from '../services/spreadsheet.service'
 import { UploadFileResponse } from '../models/UploadFile'
 import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { ParseResult } from '../models/ParseResult.interface'
 
 @Component({
   selector: 'app-editor',

client/src/app/models/ParseParams.interface.ts

@@ -0,0 +1,23 @@
+import { DcValidator } from "../shared/dc-validator/dc-validator"
+import { FileUploadEncoding } from "./FileUploadEncoding"
+import { FileUploader } from "./FileUploader.class"
+import { ExcelRule } from "./TableData"
+
+export interface ParseParams {
+  file: File
+  password?: string
+  dcValidator: DcValidator
+  /**
+   * Parse function will manipulate and return the uploader array which can be provided with files already in the queue
+   * Otherwise new empty instance will be created.
+   */
+  uploader?: FileUploader
+  headerPks: string[]
+  headerArray: string[]
+  headerShow: string[]
+  timeHeaders: string[]
+  dateHeaders: string[]
+  dateTimeHeaders: string[]
+  xlRules: ExcelRule[]
+  encoding?: FileUploadEncoding
+}

client/src/app/models/ParseResult.interface.ts

@@ -0,0 +1,15 @@
+import { FileUploader } from "./FileUploader.class"
+import SheetInfo from "./SheetInfo"
+
+export interface ParseResult {
+  /**
+   * In case of CSV file, won't be returned
+   */
+  data?: any[]
+  /**
+   * In case of CSV file, won't be returned
+   */
+  headerShow?: string[]
+  rangeSheetRes?: SheetInfo
+  uploader: FileUploader
+}

client/src/app/multi-dataset/multi-dataset.component.ts

@@ -11,7 +11,6 @@ import {
   SasService,
   SasStoreService
 } from '../services'
-import * as XLSX from '@sheet/crypto'
 import { globals } from '../_globals'
 import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
 import { DcValidator } from '../shared/dc-validator/dc-validator'
@@ -19,7 +18,6 @@ import { ExcelRule } from '../models/TableData'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { Col } from '../shared/dc-validator/models/col.model'
 import {
-  ParseResult,
   SpreadsheetService
 } from '../services/spreadsheet.service'
 import Handsontable from 'handsontable'
@@ -31,6 +29,7 @@ import { ColumnSettings } from 'handsontable/settings'
 import { UploadFile } from '@sasjs/adapter'
 import { UploadFileResponse } from '../models/UploadFile'
 import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { ParseResult } from '../models/ParseResult.interface'
 
 @Component({
   selector: 'app-multi-dataset',
@@ -292,6 +291,7 @@ export class MultiDatasetComponent implements OnInit {
         this.spreadsheetService
           .parseExcelFile({
             file: this.selectedFile!,
+            password: this.selectedFile!.password || undefined,
             dcValidator: parsedDataset.datasetInfo.dcValidator!,
             headerPks: parsedDataset.datasetInfo.headerPks,
             headerArray: parsedDataset.datasetInfo.headerArray,
@@ -518,7 +518,23 @@ export class MultiDatasetComponent implements OnInit {
    * convention. {@link isValidDatasetFormat}
    */
   async onAutoDetectColumns() {
-    this.sheetNames = await this.parseExcelSheetNames()
+    let passwordError = false
+
+    await this.parseExcelSheetNames()
+    .then((sheetNames) => {
+      this.sheetNames = sheetNames
+    })
+    .catch((err) => {
+      if (err.includes('password')) {
+        passwordError = true
+      }
+    })
+
+    if (passwordError) {
+      this.onDiscardFile()
+      this.eventService.showInfoModal('Locked file', 'We failed to unlock the file.')
+      return
+    }
 
     if (this.sheetNames) {
       this.matchedDatasets = []
@@ -824,42 +840,18 @@ export class MultiDatasetComponent implements OnInit {
    *
    * @returns sheet names in string array
    */
-  private parseExcelSheetNames(): Promise<string[]> {
+  private async parseExcelSheetNames(): Promise<string[]> {
     return new Promise((resolve, reject) => {
-      const reader = new FileReader()
+      if (!this.selectedFile) return resolve([])
 
-      if (!this.selectedFile) {
+      this.spreadsheetService.parseExcelSheetNames(this.selectedFile)
-        console.warn('selectedFile is missing')
+      .then((parsed) => {
-        return resolve([])
+        if (parsed.password) this.selectedFile!.password = parsed.password
-      }
 
-      reader.onload = (event: ProgressEvent<FileReader>) => {
+        return resolve(parsed.sheetNames)
-        if (!event?.target) {
+      }).catch((err) => {
-          console.warn('File reader event.target is missing')
+        return reject(err)
-          return
-        }
-
-        const data = event.target.result
-        const workbook = XLSX.read(data, {
-          // Load file minimally to parse sheets
-          bookSheets: true,
-          type: 'binary'
       })
-
-        try {
-          const sheet_names_list = workbook.SheetNames
-
-          return resolve(sheet_names_list)
-        } catch (e) {
-          console.error(e)
-        }
-      }
-
-      reader.onerror = function (ex) {
-        console.log(ex)
-      }
-
-      reader.readAsBinaryString(this.selectedFile)
     })
   }
 
@@ -1035,5 +1027,6 @@ export interface SubmittedCsvDatasetResult {
 }
 
 export interface SelectedFile extends File {
-  sizeMB?: number
+  sizeMB?: number,
+  password?: string
 }

client/src/app/services/spreadsheet.service.ts

@@ -1,70 +1,15 @@
 import { Injectable } from '@angular/core'
-import * as XLSX from '@sheet/crypto'
 import {
   ExcelPasswordModalService,
-  Result
 } from '../shared/excel-password-modal/excel-password-modal.service'
 import { EventService } from './event.service'
-import { isSpecialMissing } from '@sasjs/utils/input/validators'
-import {
-  dateFormat,
-  dateToUtcTime,
-  dateToTime
-} from '../editor/utils/date.utils'
-import {
-  excelDateToJSDate,
-  getMissingHeaders
-} from '../editor/utils/grid.utils'
-import { isStringNumber, isStringDecimal } from '../editor/utils/types.utils'
-import SheetInfo from '../models/SheetInfo'
-import { blobToFile } from '../xlmap/utils/file.utils'
-import { ExcelRule } from '../models/TableData'
-import { DcValidator } from '../shared/dc-validator/dc-validator'
 import { LicenceService } from './licence.service'
-import { FileUploadEncoding } from '../models/FileUploadEncoding'
+import { SpreadsheetUtil } from '../shared/spreadsheet-util/spreadsheet-util'
-import { FileUploader } from '../models/FileUploader.class'
+import { ParseParams } from '../models/ParseParams.interface'
-
+import { ParseResult } from '../models/ParseResult.interface'
-/**
+import { OpenOptions } from '../shared/excel-password-modal/models/options.interface'
- * Used in combination with buffer
+import { Result } from '../shared/excel-password-modal/models/result.interface'
- */
+import * as XLSX from '@sheet/crypto'
-const iconv = require('iconv-lite')
-/**
- * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
- * Which will be send to backend
- */
-const Buffer = require('buffer/').Buffer
-type AOA = any[][]
-
-export interface ParseParams {
-  file: File
-  dcValidator: DcValidator
-  /**
-   * Parse function will manipulate and return the uploader array which can be provided with files already in the queue
-   * Otherwise new empty instance will be created.
-   */
-  uploader?: FileUploader
-  headerPks: string[]
-  headerArray: string[]
-  headerShow: string[]
-  timeHeaders: string[]
-  dateHeaders: string[]
-  dateTimeHeaders: string[]
-  xlRules: ExcelRule[]
-  encoding?: FileUploadEncoding
-}
-
-export interface ParseResult {
-  /**
-   * In case of CSV file, won't be returned
-   */
-  data?: any[]
-  /**
-   * In case of CSV file, won't be returned
-   */
-  headerShow?: string[]
-  rangeSheetRes?: SheetInfo
-  uploader: FileUploader
-}
 
 @Injectable({
   providedIn: 'root'
@@ -76,92 +21,96 @@ export class SpreadsheetService {
     private excelPasswordModalService: ExcelPasswordModalService,
     private eventService: EventService,
     private licenceService: LicenceService
-  ) {}
+  ) {
+
+  }
 
-  /**
-   * Parses attached file and searches fo the matching data
-   *
-   * @param parseParams params required for parsing the file
-   * @param onParseStateChange callback used to inform about parsing state
-   * so the user of the function can update the UI with latest info
-   * @param onTableFoundEvent callback fired when table range is found in the file
-   *
-   * @returns parsed list of files to upload and JSON data ready for HOT usage
-   */
   public parseExcelFile(
     parseParams: ParseParams,
     onParseStateChange?: (uploadState: string) => void,
     onTableFoundEvent?: (info: string) => void
   ): Promise<ParseResult | undefined> {
-    return new Promise((resolve, reject) => {
+    const spreadSheetUtil = new SpreadsheetUtil({
-      let data: any[] = []
+      licenceState: this.licenceState
-      const uploader: FileUploader = parseParams.uploader || new FileUploader()
+    })
 
-      const file: File = parseParams.file
+    return spreadSheetUtil.parseExcelFile(
-      const filename = file.name
+      parseParams,
-
+      this.promptExcelPassword,
-      if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
+      onParseStateChange,
-
+      onTableFoundEvent
-      if (onParseStateChange)
+    )
-        onParseStateChange(`Loading ${filename} into the browser`)
-
-      let foundData = {
-        sheet: ''
   }
 
-      let fileType = filename.slice(
+  /**
-        filename.lastIndexOf('.') + 1,
+  * Read the file minimally just to get the sheet names, not reading full file
-        filename.lastIndexOf('.') + 4
+  * to help boost the performance
-      )
+  *
+  * @returns sheet names in string array
+  */
+ public async parseExcelSheetNames(file: File): Promise<{
+  sheetNames: string[],
+  password?: string
+ }> {
+   return new Promise((resolve, reject) => {
+     const reader = new FileReader()
+
+     if (!file) {
+       console.warn('file is missing')
+       return resolve({ sheetNames: [] })
+     }
+
+     reader.onload = async (event: ProgressEvent<FileReader>) => {
+       if (!event?.target) {
+         console.warn('File reader event.target is missing')
+         return
+       }
 
-      if (fileType.toLowerCase() === 'xls') {
-        let reader: FileReader = new FileReader()
-        const self = this
-        reader.onload = async (theFile: any) => {
-          /* read workbook */
-          const bstr = this.toBstr(theFile.target.result)
        let wb: XLSX.WorkBook | undefined = undefined
        let fileUnlocking: boolean = false
-
+       let password: string | undefined
-          const xlsxOptions: XLSX.ParsingOptions = {
+       const data = event.target.result
-            type: 'binary',
-            cellDates: false,
-            cellFormula: true,
-            cellStyles: true,
-            cellNF: false,
-            cellText: false
-          }
 
        try {
-            wb = XLSX.read(bstr, {
+          wb = XLSX.read(data, {
-              ...xlsxOptions
+            // Load file minimally to parse sheets
+            bookSheets: true,
+            type: 'binary'
           })
         } catch (err: any) {
           if (err.message.toLowerCase().includes('password')) {
             fileUnlocking = true
 
+            let passwordError = false
+
             while (fileUnlocking) {
-                const password = await this.promptExcelPassword()
+              password = await this.promptExcelPassword({
+                error: passwordError
+              })
 
               if (password) {
                 try {
-                    wb = XLSX.read(bstr, {
+                  wb = XLSX.read(data, {
-                      ...xlsxOptions,
+                    // Load file minimally to parse sheets
+                    bookSheets: true,
+                    type: 'binary',
                     password: password
                   })
 
                   fileUnlocking = false
+                  passwordError = false
                 } catch (err: any) {
-                    this.excelPasswordModalService.open({
+                  passwordError = true
-                      error: true
-                    })
 
                   if (!err.message.toLowerCase().includes('password')) {
                     fileUnlocking = false
                   }
                 }
+
+                if (!password) return reject('Invalid password, failed to decrypt the file')
               } else {
                 fileUnlocking = false
+                return reject('No password provided')
               }
             }
           } else {
@@ -169,738 +118,41 @@ export class SpreadsheetService {
           }
         }
 
-          if (!wb) {
+        if (!wb) return reject('Error parsing the workbook')
-            return reject('No workbook found.')
-          }
 
-          /* save data */
+        try {
-          let isComplete: boolean = false
+          const sheetNames = wb.SheetNames
-          let missingHeaders: string[] = []
-
-          const csvArrayHeaders: string[] = [
-            '_____DELETE__THIS__RECORD_____',
-            ...parseParams.headerArray
-          ]
-          let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
-          let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
-            (map: any, obj: string) => {
-              map[obj] = -1
-              return map
-            },
-            {}
-          )
-
-          let csvArrayData: any[] = []
-          const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
-            wb,
-            parseParams
-          )
-          missingHeaders = rangeSheetRes.missingHeaders
-
-          if (rangeSheetRes.foundData) {
-            isComplete = true
-            csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
-            const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
-
-            if (onParseStateChange)
-              onParseStateChange(
-                `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
-              )
-
-            let startAddress = ''
-            let endAddress = ''
-
-            for (
-              let row = rangeSheetRes.startRow;
-              row < rangeSheetRes.endRow;
-              ++row
-            ) {
-              const arr: any[] = []
-
-              csvArrayHeadersLower.forEach((x) => {
-                const col = csvArrayHeadersMap[x]
-                const addr = XLSX.utils.encode_cell({
-                  r: rangeSheetRes.rangeStartRow + row,
-                  c: rangeSheetRes.rangeStartCol + col
-                })
-
-                let cell
-
-                if (!ws[addr]) {
-                  cell = { v: '' }
-                } else {
-                  cell = ws[addr]
-                }
-
-                if (startAddress === '' && ws[addr]) startAddress = addr
-                endAddress = addr
-
-                arr.push(cell)
-              })
-
-              // If we found at least one non empty value it means it is not empty row
-              // othervise, it is empty row
-              let arrNonEmptyValue = arr.find((x) => x.v !== '')
-
-              if (arrNonEmptyValue) csvArrayData.push(arr)
-            }
-
-            rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
-
-            if (onTableFoundEvent)
-              onTableFoundEvent(
-                `Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
-              )
-          } else {
-            missingHeaders = rangeSheetRes.missingHeaders
-          }
-
-          if (missingHeaders.length > 0) {
-            missingHeaders.sort(function compareSecondColumn(a, b) {
-              if (a[1] === b[1]) {
-                return 0
-              } else {
-                return a[1] > b[1] ? -1 : 1
-              }
-            })
-            let abortMsg = missingHeaders
-              .map((x) => x[0])
-              .slice(0, 5)
-              .join('\n')
-
-            uploader.queue.pop()
-            return reject(abortMsg)
-          }
-
-          // If first row is empty, that means no data has been found
-          if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
-            let abortMsg = 'No relevant data found in File !'
-
-            uploader.queue.pop()
-            return reject(abortMsg)
-          }
-
-          if (
-            parseParams.dateTimeHeaders.length > 0 ||
-            parseParams.dateHeaders.length > 0 ||
-            parseParams.timeHeaders.length > 0
-          ) {
-            csvArrayData = this.updateDateTimeCols(
-              csvArrayHeaders,
-              csvArrayData,
-              parseParams
-            )
-          }
-
-          if (parseParams.xlRules.length > 0) {
-            csvArrayData = this.updateXLRuleCols(
-              csvArrayHeaders,
-              csvArrayData,
-              parseParams
-            )
-          }
-
-          if (!isComplete) {
-            let abortMsg = ''
-
-            if (missingHeaders.length === 0) {
-              abortMsg = 'No relevant data found in File !'
-            } else {
-              missingHeaders.sort(function compareSecondColumn(a, b) {
-                if (a[1] === b[1]) {
-                  return 0
-                } else {
-                  return a[1] > b[1] ? -1 : 1
-                }
-              })
-              abortMsg = missingHeaders
-                .map((x) => x[0])
-                .slice(0, 5)
-                .join('\n')
-            }
-
-            // abort message is fired, return undefined
-            uploader.queue.pop()
-            return reject(abortMsg)
-          } else {
-            parseParams.headerShow = csvArrayHeaders
-            csvArrayData = csvArrayData.map((row: any) =>
-              row.map((col: any) => (col.t === 'n' ? col.v : col.w))
-            )
-
-            csvArrayData = csvArrayData.map((row: any) => {
-              return row.map((col: any, index: number) => {
-                if (!col && col !== 0) col = ''
-
-                /**
-                 * Keeping this for the reference
-                 * Code below used to convert JSON to CSV
-                 * now the XLSX is converting to CSV
-                 */
-                // if (isNaN(col)) {
-                //   // Match and replace the double quotes, ignore the first and last char
-                //   // in case they are double quotes already
-                //   col = col.replace(/(?<!^)"(?!$)/g, '""')
-
-                //   if (col.search(/,/g) > -1 ||
-                //       col.search(/\r|\n/g) > -1
-                //   ) {
-                //     // Missing quotes at the end
-                //     if (col.search(/"$/g) < 0) {
-                //       col = col + '"' // So we add them
-                //     }
-
-                //     // Missing quotes at the start
-                //     if (col.search(/^"/g) < 0) {
-                //       col = '"' + col // So we add them
-                //     }
-                //   }
-                // }
-
-                const colName = parseParams.headerShow[index]
-                const colRule = parseParams.dcValidator?.getRule(colName)
-
-                if (colRule?.type === 'numeric') {
-                  if (isSpecialMissing(col) && !col.includes('.'))
-                    col = '.' + col
-                }
-
-                return col
-              })
-            })
-
-            data = csvArrayData
-
-            // Apply licence rows limitation if exists, it is only affecting data
-            // which will be send to SAS
-            const strippedCsvArrayData = csvArrayData.slice(
-              0,
-              this.licenceState.value.submit_rows_limit
-            )
-            // To submit to sas service, we need clean version of CSV of file
-            // attached. XLSX will do the parsing and heavy lifting
-            // First we create worksheet of json (data we extracted)
-            let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
-              skipHeader: true
-            })
-            // create CSV to be uploaded from worksheet
-            let csvContentClean = XLSX.utils.sheet_to_csv(ws)
-            // Prepend headers
-            csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
-
-            // Blob from which CSV file will be created depending of the selected
-            // encoding
-            let blob: Blob
-
-            if (parseParams.encoding === 'WLATIN1') {
-              // WLATIN1
-              let encoded = iconv.decode(
-                Buffer.from(csvContentClean),
-                'CP-1252'
-              )
-              blob = new Blob([encoded], { type: 'application/csv' })
-            } else {
-              // UTF-8
-              blob = new Blob([csvContentClean], { type: 'application/csv' })
-            }
-
-            let newCSVFile: File = blobToFile(blob, filename + '.csv')
-            uploader.addToQueue([newCSVFile])
-          }
-
-          if (data.length === 0) {
-            return reject(
-              `Table in the file is empty. Data found on sheet: ${foundData.sheet}`
-            )
-          }
 
           return resolve({
-            uploader,
+            sheetNames: sheetNames,
-            data,
+            password: password
-            rangeSheetRes,
-            headerShow: parseParams.headerShow
           })
+        } catch (e) {
+          console.error(e)
         }
-        reader.readAsArrayBuffer(file)
-      } else if (fileType.toLowerCase() === 'csv') {
-        if (this.licenceState.value.submit_rows_limit !== Infinity) {
-          uploader.queue.pop()
-          return reject(
-            'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
-          )
      }
 
-        if (parseParams.encoding === 'WLATIN1') {
+     reader.onerror = function (ex) {
-          let reader = new FileReader()
+       console.log(ex)
-          const self = this
+     }
-          // Closure to capture the file information.
-          reader.onload = (theFile: any) => {
-            let encoded = iconv.decode(
-              Buffer.from(theFile.target.result),
-              'CP-1252'
-            )
-            let blob = new Blob([encoded], { type: fileType })
-            let encodedFile: File = blobToFile(blob, filename)
-            uploader.queue.pop()
-            uploader.addToQueue([encodedFile])
 
-            resolve({
+     reader.readAsBinaryString(file)
-              uploader
    })
  }
 
-          reader.readAsArrayBuffer(file)
+  public bytesToMB(size: number): number {
-        } else {
+    return parseFloat((size / (1024*1024)).toFixed(2))
-          return resolve({
-            uploader
-          })
-        }
-      } else {
-        let abortMsg =
-          'Invalid file type "<b>' +
-          filename +
-          '</b>". Please upload csv or excel file.'
-
-        uploader.queue.pop()
-        return reject(abortMsg)
-      }
-    })
-  }
-
-  /**
-   * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
-   * @param wb Excel workbook
-   * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
-   */
-  private getRangeAndSheet(
-    wb: XLSX.WorkBook,
-    parseParams: ParseParams
-  ): SheetInfo {
-    let data = []
-
-    let rangeStartRow: number = 0
-    let rangeStartCol: number = 0
-    let startRow: number = -1
-    let endRow: number = -1
-    let sheetName: string = ''
-    let isComplete = false
-    let missingHeaders: string[] = []
-    const csvArrayHeaders: string[] = [
-      '_____DELETE__THIS__RECORD_____',
-      ...parseParams.headerArray
-    ]
-    let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
-    let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
-      (map: any, obj: string) => {
-        map[obj] = -1
-        return map
-      },
-      {}
-    )
-
-    wb.SheetNames.forEach((element: string) => {
-      // Checking for required data in each sheet in workbook/
-      if (isComplete) {
-        return
-      }
-
-      missingHeaders = []
-      sheetName = element
-      const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
-
-      data = <AOA>XLSX.utils.sheet_to_json(ws, {
-        header: 1,
-        blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
-        defval: ''
-      })
-
-      if (data.length <= 1) {
-        return
-      }
-
-      let tempArr: string[] = []
-      parseParams.headerArray.forEach(() => tempArr.push(''))
-      data.push(tempArr)
-
-      let foundHeaders = false
-
-      data.forEach((row: any, index: number) => {
-        if (isComplete) {
-          return
-        }
-
-        if (foundHeaders) {
-          let isDataEnd = true
-          let isPkNull = false
-
-          csvArrayHeadersLower.forEach((x) => {
-            const col = csvArrayHeadersMap[x]
-
-            if (row[col] !== '' && row[col] !== undefined) {
-              isDataEnd = false
-            } else {
-              if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
-                isPkNull = true
-              }
-            }
-          })
-
-          if (isDataEnd || isPkNull) {
-            endRow = index
-            isComplete = true
-          } else {
-            if (startRow === -1) {
-              startRow = index
-            }
-          }
-        } else {
-          const rowLowerCase: string[] = row.map((x: any) =>
-            x.toString().toLowerCase()
-          )
-
-          // If in file there is no delete column, remove it from search of missing.
-          // This way delete column will be optional to provide in file
-          if (!rowLowerCase.includes('_____delete__this__record_____')) {
-            const deleteIndex = csvArrayHeadersLower.indexOf(
-              '_____delete__this__record_____'
-            )
-
-            if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
-          }
-
-          foundHeaders = true
-
-          csvArrayHeadersLower.forEach((x) => {
-            if (rowLowerCase.indexOf(x) === -1) {
-              foundHeaders = false
-            }
-          })
-
-          let result = []
-
-          result = this.findValidHeaders(
-            rowLowerCase,
-            csvArrayHeadersLower,
-            index,
-            sheetName,
-            parseParams
-          )
-
-          if (result[0] === false) {
-            foundHeaders = false
-
-            if (result[1].length > 0) {
-              result[1].forEach((data: string) => {
-                missingHeaders.push(data)
-              })
-            }
-          } else {
-            csvArrayHeadersMap = result[1]
-          }
-        }
-      })
-
-      if (isComplete) {
-        this.update_sheet_range(ws)
-        const worksheetSel = ws['!ref']
-
-        if (worksheetSel) {
-          const range = XLSX.utils.decode_range(ws['!ref'] || '')
-          rangeStartRow = range.s.r
-          rangeStartCol = range.s.c
-        }
-      }
-    })
-
-    // If start row is still -1 that means first row of found range is empty
-    if (startRow === -1) isComplete = false
-
-    const returnObj: SheetInfo = {
-      foundData: isComplete,
-      sheetName,
-      startRow,
-      endRow,
-      csvArrayHeadersMap,
-      missingHeaders,
-      rangeStartRow,
-      rangeStartCol
-    }
-
-    return returnObj
-  }
-
-  private findValidHeaders(
-    row: string[],
-    headers: string[],
-    rowNumber: number,
-    tabName: string,
-    parseParams: ParseParams
-  ): Array<any> {
-    let headersFound = false
-    const missingErrorArray = []
-    let j = 0
-
-    while (j < row.length) {
-      if (headersFound) {
-        // return;
-      } else {
-        if (headers.indexOf(row[j]) !== -1) {
-          let breakIndex
-          let rowStart = 0
-          let rowEnd = 0
-          let arrStart = 0
-          let foundHeadersArray: string[] = []
-          let spaceBreak = false
-
-          for (let i = j; i < row.length; i++) {
-            if (
-              row[i] === '' ||
-              (foundHeadersArray.indexOf(row[i]) !== -1 &&
-                this.isColHeader(row[i], parseParams.headerArray))
-            ) {
-              if (row[i] === '') {
-                spaceBreak = true
-              }
-
-              breakIndex = i
-              break
-            } else {
-              foundHeadersArray.push(row[i])
-            }
-          }
-
-          let tempArray: string[] = []
-
-          if (breakIndex !== undefined) {
-            tempArray = row.slice(j, breakIndex)
-            arrStart = j
-            rowEnd = breakIndex
-
-            if (spaceBreak) {
-              rowStart = j
-              j = breakIndex
-            } else {
-              rowStart = j
-              j = breakIndex - 1
-            }
-          } else {
-            tempArray = row.slice(j)
-            rowStart = j
-            arrStart = j
-            rowEnd = row.length
-            j = row.length
-          }
-
-          let foundHeaders = true
-
-          //We check if there are missing headers
-          headers.forEach((x) => {
-            if (tempArray.indexOf(x) === -1) {
-              foundHeaders = false
-            }
-          })
-
-          if (foundHeaders) {
-            headersFound = true
-
-            let mapHeaders: any[] = headers
-
-            let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
-              map[obj] = -1
-              return map
-            }, {})
-
-            let temp = row.slice(rowStart, rowEnd)
-
-            headers.forEach((x) => {
-              csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
-            })
-
-            return [true, csvArrayHeadersMap]
-          } else {
-            let missingHeaders = getMissingHeaders(tempArray, headers)
-
-            let missingMessage = '<b>TAB(' + tabName + ')</b>'
-            missingErrorArray.push([
-              missingMessage +
-                ' - ' +
-                missingHeaders[1].join(',') +
-                ' ( missing ' +
-                missingHeaders[0].join(',') +
-                ' )',
-              missingHeaders[1].length
-            ])
-          }
-        }
-      }
-      j++
-    }
-    return [false, missingErrorArray]
-  }
-
-  private isColHeader(col: string, headerArray: string[]) {
-    return headerArray.indexOf(col.toUpperCase()) > -1
-  }
-
-  /**
-   * Function that updates the !ref range value provided in official docs.
-   * @param ws worksheet to be updated
-   */
-  private update_sheet_range(ws: XLSX.WorkSheet) {
-    const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
-
-    Object.keys(ws)
-      .filter(function (x) {
-        return x.charAt(0) != '!'
-      })
-      .map(XLSX.utils.decode_cell)
-      .forEach(function (x: any) {
-        range.s.c = Math.min(range.s.c, x.c)
-        range.s.r = Math.min(range.s.r, x.r)
-        range.e.c = Math.max(range.e.c, x.c)
-        range.e.r = Math.max(range.e.r, x.r)
-      })
-
-    ws['!ref'] = XLSX.utils.encode_range(range)
   }
 
   /**
    * When excel is password protected we will display the password prompt for user to type password in.
    * @returns Password user input or undefined if discarded by user
    */
-  private promptExcelPassword(): Promise<string | undefined> {
+  private promptExcelPassword = (options?: OpenOptions): Promise<string | undefined> => {
     return new Promise((resolve, reject) => {
-      this.excelPasswordModalService.open().subscribe((result: Result) => {
+      this.excelPasswordModalService.open(options).subscribe((result: Result) => {
         resolve(result.password)
       })
     })
   }
-
-  private updateDateTimeCols(
-    headers: any,
-    data: any,
-    parseParams: ParseParams
-  ) {
-    if (parseParams.dateHeaders.length > 0) {
-      const dateCols: number[] = []
-      parseParams.dateHeaders.forEach((element) => {
-        if (headers.indexOf(element) !== -1) {
-          dateCols.push(headers.indexOf(element))
-        }
-      })
-      data.forEach((row: any[]) => {
-        dateCols.forEach((element) => {
-          const obj = row[element]
-          if (isStringNumber(obj.v)) {
-            const date = excelDateToJSDate(Number(obj.v))
-
-            obj.v =
-              date.getFullYear() +
-              '-' +
-              ('0' + (date.getMonth() + 1)).slice(-2) +
-              '-' +
-              ('0' + date.getDate()).slice(-2)
-          } else {
-            if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
-              const date = new Date(obj.v)
-              if (date.toUTCString() !== 'Invalid Date') {
-                obj.v = dateFormat(date)
-              }
-            }
-          }
-          row[element] = obj
-        })
-      })
-    }
-    if (parseParams.timeHeaders.length > 0) {
-      let timeCols: number[] = []
-      parseParams.timeHeaders.forEach((element) => {
-        if (headers.indexOf(element) !== -1) {
-          timeCols.push(headers.indexOf(element))
-        }
-      })
-      data.forEach((row: any[]) => {
-        timeCols.forEach((element) => {
-          const obj = row[element]
-          if (
-            isStringNumber(obj.v) ||
-            isStringDecimal(obj.v) ||
-            obj.v.includes('E-')
-          ) {
-            const date = excelDateToJSDate(Number(obj.v))
-
-            obj.v = dateToUtcTime(date)
-          }
-          row[element] = obj
-        })
-      })
-    }
-    if (parseParams.dateTimeHeaders.length > 0) {
-      let dateTimeCols: number[] = []
-      parseParams.dateTimeHeaders.forEach((element) => {
-        if (headers.indexOf(element) !== -1) {
-          dateTimeCols.push(headers.indexOf(element))
-        }
-      })
-      data.forEach((row: any[]) => {
-        dateTimeCols.forEach((element) => {
-          const obj = row[element]
-          if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
-            const date = excelDateToJSDate(Number(obj.v))
-            obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
-          } else {
-            if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
-              let str = obj.v.substring(0, obj.v.indexOf(':'))
-              str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
-              obj.v = str
-            }
-            const date = new Date(obj.v)
-            if (date.toUTCString() !== 'Invalid Date') {
-              obj.v = dateFormat(date) + ' ' + dateToTime(date)
-            }
-          }
-          row[element] = obj
-        })
-      })
-    }
-    return data
-  }
-
-  private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
-    if (parseParams.xlRules.length > 0) {
-      const xlRuleCols: any = []
-      parseParams.xlRules.forEach((element: any) => {
-        if (headers.indexOf(element.XL_COLUMN) !== -1) {
-          element['index'] = headers.indexOf(element.XL_COLUMN)
-          xlRuleCols.push(element)
-        }
-      })
-      data.forEach((row: any[]) => {
-        xlRuleCols.forEach((element: any) => {
-          const obj = row[element.index]
-          if (element.XL_RULE === 'FORMULA') {
-            if ('f' in obj) {
-              if (obj['t'] === 'n') {
-                obj['v'] = '=' + obj['f']
-              } else {
-                obj['w'] = '=' + obj['f']
-              }
-            }
-          }
-          row[element] = obj
-        })
-      })
-    }
-    return data
-  }
-
-  private toBstr(res: any) {
-    let bytes = new Uint8Array(res)
-    let binary = ''
-    let length = bytes.byteLength
-    for (let i = 0; i < length; i++) {
-      binary += String.fromCharCode(bytes[i])
-    }
-    return binary
-  }
 }

client/src/app/shared/excel-password-modal/excel-password-modal.component.html

@@ -1,4 +1,4 @@
-<ng-container *ngIf="options$ | async as options">
+<div *ngIf="options$ | async as options" class="excel-password-root">
   <clr-modal
     [clrModalOpen]="options.open"
     [clrModalSize]="'md'"
@@ -11,6 +11,7 @@
       <p class="m-0">Please enter password:</p>
       <input
         #filePasswordInput
+        [(ngModel)]="passwordInput"
         data-lpignore="true"
         autocomplete="off"
         id="filePasswordInput"
@@ -19,9 +20,13 @@
       />
     </div>
     <div class="modal-footer">
+      <div>
         <p *ngIf="options.error" class="m-0 color-red">
           Sorry that didn't work, try again.
         </p>
+      </div>
+
+      <div class="buttons">
         <button type="button" class="btn btn-sm btn-outline" (click)="close()">
           Cancel
         </button>
@@ -34,5 +39,6 @@
           Unlock
         </button>
       </div>
+    </div>
   </clr-modal>
-</ng-container>
+</div>

client/src/app/shared/excel-password-modal/excel-password-modal.component.scss

@@ -0,0 +1,18 @@
+.excel-password-root {
+  ::ng-deep {
+    .modal {
+      z-index: 1060;
+    }
+  }
+}
+
+.modal-footer {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+
+  .buttons {
+    display: flex;
+    gap: 5px;
+  }
+}

client/src/app/shared/excel-password-modal/excel-password-modal.component.ts

@@ -1,24 +1,26 @@
 import { Component } from '@angular/core'
 import { Observable } from 'rxjs'
 import {
-  ExcelPasswordModalService,
+  ExcelPasswordModalService
-  Options
 } from './excel-password-modal.service'
+import { Options } from './models/options.interface'
 
 @Component({
   selector: 'app-excel-password-modal',
+  styleUrls: ['./excel-password-modal.component.scss'],
   templateUrl: './excel-password-modal.component.html'
 })
 export class ExcelPasswordModalComponent {
-  options$: Observable<Options>
+  options$: Observable<Options> = this.excelPasswordModalService.optionsSubject$
 
   fileUnlockError: boolean = false
 
-  constructor(private excelPasswordModalService: ExcelPasswordModalService) {
+  passwordInput: string = ''
-    this.options$ = this.excelPasswordModalService.optionsSubject$
+
-  }
+  constructor(private excelPasswordModalService: ExcelPasswordModalService) {}
 
   close(password?: string) {
+    this.passwordInput = ''
     this.excelPasswordModalService.close(password)
   }
 }

client/src/app/shared/excel-password-modal/excel-password-modal.service.ts

@@ -1,17 +1,7 @@
 import { Injectable } from '@angular/core'
 import { Subject, Observable } from 'rxjs'
-
+import { OpenOptions, Options } from './models/options.interface'
-export interface Options extends OpenOptions {
+import { Result } from './models/result.interface'
-  open: boolean
-}
-
-export interface Result {
-  password: string | undefined
-}
-
-export interface OpenOptions {
-  error?: boolean
-}
 
 @Injectable({
   providedIn: 'root'

client/src/app/shared/excel-password-modal/models/options.interface.ts

@@ -0,0 +1,7 @@
+export interface OpenOptions {
+  error?: boolean
+}
+
+export interface Options extends OpenOptions {
+  open: boolean
+}

client/src/app/shared/excel-password-modal/models/result.interface.ts

@@ -0,0 +1,3 @@
+export interface Result {
+  password: string | undefined
+}

client/src/app/shared/spreadsheet-util/random.class.ts

@@ -0,0 +1,5 @@
+export class RandomClass {
+  constructor(jo: string) {
+
+  }
+}

client/src/app/shared/spreadsheet-util/spreadsheet-util.ts

@@ -0,0 +1,904 @@
+import { isSpecialMissing } from '@sasjs/utils/input/validators'
+import { dateFormat, dateToUtcTime, dateToTime } from "src/app/editor/utils/date.utils"
+import { getMissingHeaders, excelDateToJSDate } from "src/app/editor/utils/grid.utils"
+import { isStringNumber, isStringDecimal } from "src/app/editor/utils/types.utils"
+import { FileUploader } from "src/app/models/FileUploader.class"
+import SheetInfo from "src/app/models/SheetInfo"
+import { blobToFile } from "src/app/xlmap/utils/file.utils"
+import * as XLSX from '@sheet/crypto'
+import { LicenceState } from "src/app/models/LicenceState"
+import { BehaviorSubject } from "rxjs"
+import { ParseParams } from 'src/app/models/ParseParams.interface'
+import { ParseResult } from 'src/app/models/ParseResult.interface'
+import { OpenOptions } from '../excel-password-modal/models/options.interface'
+
+/**
+ * Used in combination with buffer
+ */
+import * as iconv from 'iconv-lite'
+/**
+ * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
+ * Which will be send to backend
+ */
+import { Buffer } from 'buffer'
+type AOA = any[][]
+
+export interface ConstructorParams {
+  licenceState: BehaviorSubject<LicenceState>
+}
+
+export class SpreadsheetUtil {
+  private licenceState: BehaviorSubject<LicenceState>
+
+  constructor(params: ConstructorParams) {
+    this.licenceState = params.licenceState
+  }
+
+  /**
+   * Parses attached file and searches fo the matching data
+   *
+   * @param parseParams params required for parsing the file
+   * @param onParseStateChange callback used to inform about parsing state
+   * so the user of the function can update the UI with latest info
+   * @param onTableFoundEvent callback fired when table range is found in the file
+   *
+   * @returns parsed list of files to upload and JSON data ready for HOT usage
+   */
+  public parseExcelFile(
+    parseParams: ParseParams,
+    promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
+    onParseStateChange?: (uploadState: string) => void,
+    onTableFoundEvent?: (info: string) => void
+  ): Promise<ParseResult | undefined> {
+    return new Promise((resolve, reject) => {
+      // If file size is bigger then 2 MB we need to load its bytes in chunks
+      const sizeInMB = this.bytesToMB(parseParams.file.size)
+
+      if (sizeInMB > 2) {
+
+      }
+
+      let data: any[] = []
+      const uploader: FileUploader = parseParams.uploader || new FileUploader()
+
+      const file: File = parseParams.file
+      const filename = file.name
+
+      if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
+
+      if (onParseStateChange)
+        onParseStateChange(`Loading ${filename} into the browser`)
+
+      let foundData = {
+        sheet: ''
+      }
+
+      let fileType = filename.slice(
+        filename.lastIndexOf('.') + 1,
+        filename.lastIndexOf('.') + 4
+      )
+
+      if (fileType.toLowerCase() === 'xls') {
+        let reader: FileReader = new FileReader()
+
+        const self = this
+        reader.onload = async (theFile: any) => {
+          /* read workbook */
+          const bstr = this.toBstr(theFile.target.result)
+          let wb: XLSX.WorkBook | undefined = undefined
+          let fileUnlocking: boolean = false
+
+          const xlsxOptions: XLSX.ParsingOptions = {
+            type: 'binary',
+            cellDates: false,
+            cellFormula: true,
+            cellStyles: true,
+            cellNF: false,
+            cellText: false,
+            password: parseParams.password
+          }
+
+          try {
+            // wb = this.xlsxRead(bstr, {
+            //   ...xlsxOptions
+            // })
+            wb = XLSX.read(bstr, {
+              ...xlsxOptions
+            })
+          } catch (err: any) {
+            if (err.message.toLowerCase().includes('password')) {
+              fileUnlocking = true
+
+              let passwordError = false
+
+              while (fileUnlocking) {
+                const password = await promptExcelPassword({
+                  error: passwordError
+                })
+
+                if (password) {
+                  try {
+                    wb = XLSX.read(bstr, {
+                      ...xlsxOptions,
+                      password: password
+                    })
+
+                    fileUnlocking = false
+                    passwordError = false
+                  } catch (err: any) {
+                    passwordError = true
+
+                    if (!err.message.toLowerCase().includes('password')) {
+                      fileUnlocking = false
+                    }
+                  }
+                } else {
+                  fileUnlocking = false
+                }
+              }
+            } else {
+              return reject('Error reading the file')
+            }
+          }
+
+          if (!wb) {
+            return reject('No workbook found.')
+          }
+
+          /* save data */
+          let isComplete: boolean = false
+          let missingHeaders: string[] = []
+
+          const csvArrayHeaders: string[] = [
+            '_____DELETE__THIS__RECORD_____',
+            ...parseParams.headerArray
+          ]
+          let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
+          let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
+            (map: any, obj: string) => {
+              map[obj] = -1
+              return map
+            },
+            {}
+          )
+
+          let csvArrayData: any[] = []
+          const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
+            wb,
+            parseParams
+          )
+          missingHeaders = rangeSheetRes.missingHeaders
+
+          if (rangeSheetRes.foundData) {
+            isComplete = true
+            csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
+            const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
+
+            if (onParseStateChange)
+              onParseStateChange(
+                `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
+              )
+
+            let startAddress = ''
+            let endAddress = ''
+
+            for (
+              let row = rangeSheetRes.startRow;
+              row < rangeSheetRes.endRow;
+              ++row
+            ) {
+              const arr: any[] = []
+
+              csvArrayHeadersLower.forEach((x) => {
+                const col = csvArrayHeadersMap[x]
+                const addr = XLSX.utils.encode_cell({
+                  r: rangeSheetRes.rangeStartRow + row,
+                  c: rangeSheetRes.rangeStartCol + col
+                })
+
+                let cell
+
+                if (!ws[addr]) {
+                  cell = { v: '' }
+                } else {
+                  cell = ws[addr]
+                }
+
+                if (startAddress === '' && ws[addr]) startAddress = addr
+                endAddress = addr
+
+                arr.push(cell)
+              })
+
+              // If we found at least one non empty value it means it is not empty row
+              // othervise, it is empty row
+              let arrNonEmptyValue = arr.find((x) => x.v !== '')
+
+              if (arrNonEmptyValue) csvArrayData.push(arr)
+            }
+
+            rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
+
+            if (onTableFoundEvent)
+              onTableFoundEvent(
+                `Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
+              )
+          } else {
+            missingHeaders = rangeSheetRes.missingHeaders
+          }
+
+          if (missingHeaders.length > 0) {
+            missingHeaders.sort(function compareSecondColumn(a, b) {
+              if (a[1] === b[1]) {
+                return 0
+              } else {
+                return a[1] > b[1] ? -1 : 1
+              }
+            })
+            let abortMsg = missingHeaders
+              .map((x) => x[0])
+              .slice(0, 5)
+              .join('\n')
+
+            uploader.queue.pop()
+            return reject(abortMsg)
+          }
+
+          // If first row is empty, that means no data has been found
+          if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
+            let abortMsg = 'No relevant data found in File !'
+
+            uploader.queue.pop()
+            return reject(abortMsg)
+          }
+
+          if (
+            parseParams.dateTimeHeaders.length > 0 ||
+            parseParams.dateHeaders.length > 0 ||
+            parseParams.timeHeaders.length > 0
+          ) {
+            csvArrayData = this.updateDateTimeCols(
+              csvArrayHeaders,
+              csvArrayData,
+              parseParams
+            )
+          }
+
+          if (parseParams.xlRules.length > 0) {
+            csvArrayData = this.updateXLRuleCols(
+              csvArrayHeaders,
+              csvArrayData,
+              parseParams
+            )
+          }
+
+          if (!isComplete) {
+            let abortMsg = ''
+
+            if (missingHeaders.length === 0) {
+              abortMsg = 'No relevant data found in File !'
+            } else {
+              missingHeaders.sort(function compareSecondColumn(a, b) {
+                if (a[1] === b[1]) {
+                  return 0
+                } else {
+                  return a[1] > b[1] ? -1 : 1
+                }
+              })
+              abortMsg = missingHeaders
+                .map((x) => x[0])
+                .slice(0, 5)
+                .join('\n')
+            }
+
+            // abort message is fired, return undefined
+            uploader.queue.pop()
+            return reject(abortMsg)
+          } else {
+            parseParams.headerShow = csvArrayHeaders
+            csvArrayData = csvArrayData.map((row: any) =>
+              row.map((col: any) => (col.t === 'n' ? col.v : col.w))
+            )
+
+            csvArrayData = csvArrayData.map((row: any) => {
+              return row.map((col: any, index: number) => {
+                if (!col && col !== 0) col = ''
+
+                /**
+                 * Keeping this for the reference
+                 * Code below used to convert JSON to CSV
+                 * now the XLSX is converting to CSV
+                 */
+                // if (isNaN(col)) {
+                //   // Match and replace the double quotes, ignore the first and last char
+                //   // in case they are double quotes already
+                //   col = col.replace(/(?<!^)"(?!$)/g, '""')
+
+                //   if (col.search(/,/g) > -1 ||
+                //       col.search(/\r|\n/g) > -1
+                //   ) {
+                //     // Missing quotes at the end
+                //     if (col.search(/"$/g) < 0) {
+                //       col = col + '"' // So we add them
+                //     }
+
+                //     // Missing quotes at the start
+                //     if (col.search(/^"/g) < 0) {
+                //       col = '"' + col // So we add them
+                //     }
+                //   }
+                // }
+
+                const colName = parseParams.headerShow[index]
+                const colRule = parseParams.dcValidator?.getRule(colName)
+
+                if (colRule?.type === 'numeric') {
+                  if (isSpecialMissing(col) && !col.includes('.'))
+                    col = '.' + col
+                }
+
+                return col
+              })
+            })
+
+            data = csvArrayData
+
+            // Apply licence rows limitation if exists, it is only affecting data
+            // which will be send to SAS
+            const strippedCsvArrayData = csvArrayData.slice(
+              0,
+              this.licenceState.value.submit_rows_limit
+            )
+            // To submit to sas service, we need clean version of CSV of file
+            // attached. XLSX will do the parsing and heavy lifting
+            // First we create worksheet of json (data we extracted)
+            let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+              skipHeader: true
+            })
+            // create CSV to be uploaded from worksheet
+            let csvContentClean = XLSX.utils.sheet_to_csv(ws)
+            // Prepend headers
+            csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
+
+            // Blob from which CSV file will be created depending of the selected
+            // encoding
+            let blob: Blob
+
+            if (parseParams.encoding === 'WLATIN1') {
+              // WLATIN1
+              let encoded = iconv.decode(
+                Buffer.from(csvContentClean),
+                'CP-1252'
+              )
+              blob = new Blob([encoded], { type: 'application/csv' })
+            } else {
+              // UTF-8
+              blob = new Blob([csvContentClean], { type: 'application/csv' })
+            }
+
+            let newCSVFile: File = blobToFile(blob, filename + '.csv')
+            uploader.addToQueue([newCSVFile])
+          }
+
+          if (data.length === 0) {
+            return reject(
+              `Table in the file is empty. Data found on sheet: ${foundData.sheet}`
+            )
+          }
+
+          return resolve({
+            uploader,
+            data,
+            rangeSheetRes,
+            headerShow: parseParams.headerShow
+          })
+        }
+        console.log('before read file as array buffer')
+        reader.readAsArrayBuffer(file)
+      } else if (fileType.toLowerCase() === 'csv') {
+        if (this.licenceState.value.submit_rows_limit !== Infinity) {
+          uploader.queue.pop()
+          return reject(
+            'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
+          )
+        }
+
+        if (parseParams.encoding === 'WLATIN1') {
+          let reader = new FileReader()
+          const self = this
+          // Closure to capture the file information.
+          reader.onload = (theFile: any) => {
+            let encoded = iconv.decode(
+              Buffer.from(theFile.target.result),
+              'CP-1252'
+            )
+            let blob = new Blob([encoded], { type: fileType })
+            let encodedFile: File = blobToFile(blob, filename)
+            uploader.queue.pop()
+            uploader.addToQueue([encodedFile])
+
+            resolve({
+              uploader
+            })
+          }
+
+          reader.readAsArrayBuffer(file)
+        } else {
+          return resolve({
+            uploader
+          })
+        }
+      } else {
+        let abortMsg =
+          'Invalid file type "<b>' +
+          filename +
+          '</b>". Please upload csv or excel file.'
+
+        uploader.queue.pop()
+        return reject(abortMsg)
+      }
+    })
+  }
+
+  public bytesToMB(size: number): number {
+    return parseFloat((size / (1024*1024)).toFixed(2))
+  }
+
+  private xlsxRead(data: any, opts?: XLSX.ParsingOptions | undefined): XLSX.WorkBook {
+    if (typeof Worker !== 'undefined') {
+      // Create a new
+      const worker = new Worker(new URL('../../spreadsheet.worker', import.meta.url));
+      worker.onmessage = ({ data }) => {
+
+      };
+
+      worker.postMessage({
+        data,
+        opts
+      });
+
+      return  XLSX.read(data, opts) // Put in worker
+    } else {
+      // Web workers are not supported in this environment.
+      // You should add a fallback so that your program still executes correctly.
+      return  XLSX.read(data, opts)
+    }
+  }
+
+  /**
+   * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
+   * @param wb Excel workbook
+   * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
+   */
+  private getRangeAndSheet(
+    wb: XLSX.WorkBook,
+    parseParams: ParseParams
+  ): SheetInfo {
+    let data = []
+
+    let rangeStartRow: number = 0
+    let rangeStartCol: number = 0
+    let startRow: number = -1
+    let endRow: number = -1
+    let sheetName: string = ''
+    let isComplete = false
+    let missingHeaders: string[] = []
+    const csvArrayHeaders: string[] = [
+      '_____DELETE__THIS__RECORD_____',
+      ...parseParams.headerArray
+    ]
+    let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
+    let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
+      (map: any, obj: string) => {
+        map[obj] = -1
+        return map
+      },
+      {}
+    )
+
+    wb.SheetNames.forEach((element: string) => {
+      // Checking for required data in each sheet in workbook/
+      if (isComplete) {
+        return
+      }
+
+      missingHeaders = []
+      sheetName = element
+      const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
+
+      data = <AOA>XLSX.utils.sheet_to_json(ws, {
+        header: 1,
+        blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
+        defval: ''
+      })
+
+      if (data.length <= 1) {
+        return
+      }
+
+      let tempArr: string[] = []
+      parseParams.headerArray.forEach(() => tempArr.push(''))
+      data.push(tempArr)
+
+      let foundHeaders = false
+
+      data.forEach((row: any, index: number) => {
+        if (isComplete) {
+          return
+        }
+
+        if (foundHeaders) {
+          let isDataEnd = true
+          let isPkNull = false
+
+          csvArrayHeadersLower.forEach((x) => {
+            const col = csvArrayHeadersMap[x]
+
+            if (row[col] !== '' && row[col] !== undefined) {
+              isDataEnd = false
+            } else {
+              if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
+                isPkNull = true
+              }
+            }
+          })
+
+          if (isDataEnd || isPkNull) {
+            endRow = index
+            isComplete = true
+          } else {
+            if (startRow === -1) {
+              startRow = index
+            }
+          }
+        } else {
+          const rowLowerCase: string[] = row.map((x: any) =>
+            x.toString().toLowerCase()
+          )
+
+          // If in file there is no delete column, remove it from search of missing.
+          // This way delete column will be optional to provide in file
+          if (!rowLowerCase.includes('_____delete__this__record_____')) {
+            const deleteIndex = csvArrayHeadersLower.indexOf(
+              '_____delete__this__record_____'
+            )
+
+            if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
+          }
+
+          foundHeaders = true
+
+          csvArrayHeadersLower.forEach((x) => {
+            if (rowLowerCase.indexOf(x) === -1) {
+              foundHeaders = false
+            }
+          })
+
+          let result = []
+
+          result = this.findValidHeaders(
+            rowLowerCase,
+            csvArrayHeadersLower,
+            index,
+            sheetName,
+            parseParams
+          )
+
+          if (result[0] === false) {
+            foundHeaders = false
+
+            if (result[1].length > 0) {
+              result[1].forEach((data: string) => {
+                missingHeaders.push(data)
+              })
+            }
+          } else {
+            csvArrayHeadersMap = result[1]
+          }
+        }
+      })
+
+      if (isComplete) {
+        this.update_sheet_range(ws)
+        const worksheetSel = ws['!ref']
+
+        if (worksheetSel) {
+          const range = XLSX.utils.decode_range(ws['!ref'] || '')
+          rangeStartRow = range.s.r
+          rangeStartCol = range.s.c
+        }
+      }
+    })
+
+    // If start row is still -1 that means first row of found range is empty
+    if (startRow === -1) isComplete = false
+
+    const returnObj: SheetInfo = {
+      foundData: isComplete,
+      sheetName,
+      startRow,
+      endRow,
+      csvArrayHeadersMap,
+      missingHeaders,
+      rangeStartRow,
+      rangeStartCol
+    }
+
+    return returnObj
+  }
+
+  private findValidHeaders(
+    row: string[],
+    headers: string[],
+    rowNumber: number,
+    tabName: string,
+    parseParams: ParseParams
+  ): Array<any> {
+    let headersFound = false
+    const missingErrorArray = []
+    let j = 0
+
+    while (j < row.length) {
+      if (headersFound) {
+        // return;
+      } else {
+        if (headers.indexOf(row[j]) !== -1) {
+          let breakIndex
+          let rowStart = 0
+          let rowEnd = 0
+          let arrStart = 0
+          let foundHeadersArray: string[] = []
+          let spaceBreak = false
+
+          for (let i = j; i < row.length; i++) {
+            if (
+              row[i] === '' ||
+              (foundHeadersArray.indexOf(row[i]) !== -1 &&
+                this.isColHeader(row[i], parseParams.headerArray))
+            ) {
+              if (row[i] === '') {
+                spaceBreak = true
+              }
+
+              breakIndex = i
+              break
+            } else {
+              foundHeadersArray.push(row[i])
+            }
+          }
+
+          let tempArray: string[] = []
+
+          if (breakIndex !== undefined) {
+            tempArray = row.slice(j, breakIndex)
+            arrStart = j
+            rowEnd = breakIndex
+
+            if (spaceBreak) {
+              rowStart = j
+              j = breakIndex
+            } else {
+              rowStart = j
+              j = breakIndex - 1
+            }
+          } else {
+            tempArray = row.slice(j)
+            rowStart = j
+            arrStart = j
+            rowEnd = row.length
+            j = row.length
+          }
+
+          let foundHeaders = true
+
+          //We check if there are missing headers
+          headers.forEach((x) => {
+            if (tempArray.indexOf(x) === -1) {
+              foundHeaders = false
+            }
+          })
+
+          if (foundHeaders) {
+            headersFound = true
+
+            let mapHeaders: any[] = headers
+
+            let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
+              map[obj] = -1
+              return map
+            }, {})
+
+            let temp = row.slice(rowStart, rowEnd)
+
+            headers.forEach((x) => {
+              csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
+            })
+
+            return [true, csvArrayHeadersMap]
+          } else {
+            let missingHeaders = getMissingHeaders(tempArray, headers)
+
+            let missingMessage = '<b>TAB(' + tabName + ')</b>'
+            missingErrorArray.push([
+              missingMessage +
+                ' - ' +
+                missingHeaders[1].join(',') +
+                ' ( missing ' +
+                missingHeaders[0].join(',') +
+                ' )',
+              missingHeaders[1].length
+            ])
+          }
+        }
+      }
+      j++
+    }
+    return [false, missingErrorArray]
+  }
+
+  private isColHeader(col: string, headerArray: string[]) {
+    return headerArray.indexOf(col.toUpperCase()) > -1
+  }
+
+  /**
+   * Function that updates the !ref range value provided in official docs.
+   * @param ws worksheet to be updated
+   */
+  private update_sheet_range(ws: XLSX.WorkSheet) {
+    const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
+
+    Object.keys(ws)
+      .filter(function (x) {
+        return x.charAt(0) != '!'
+      })
+      .map(XLSX.utils.decode_cell)
+      .forEach(function (x: any) {
+        range.s.c = Math.min(range.s.c, x.c)
+        range.s.r = Math.min(range.s.r, x.r)
+        range.e.c = Math.max(range.e.c, x.c)
+        range.e.r = Math.max(range.e.r, x.r)
+      })
+
+    ws['!ref'] = XLSX.utils.encode_range(range)
+  }
+
+  /**
+   * When excel is password protected we will display the password prompt for user to type password in.
+   * @returns Password user input or undefined if discarded by user
+   */
+  // private promptExcelPassword(): Promise<string | undefined> {
+  //   return new Promise((resolve, reject) => {
+  //     this.excelPasswordModalService.open().subscribe((result: Result) => {
+  //       resolve(result.password)
+  //     })
+  //   })
+  // }
+
+  private updateDateTimeCols(
+    headers: any,
+    data: any,
+    parseParams: ParseParams
+  ) {
+    if (parseParams.dateHeaders.length > 0) {
+      const dateCols: number[] = []
+      parseParams.dateHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          dateCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        dateCols.forEach((element) => {
+          const obj = row[element]
+          if (isStringNumber(obj.v)) {
+            const date = excelDateToJSDate(Number(obj.v))
+
+            obj.v =
+              date.getFullYear() +
+              '-' +
+              ('0' + (date.getMonth() + 1)).slice(-2) +
+              '-' +
+              ('0' + date.getDate()).slice(-2)
+          } else {
+            if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
+              const date = new Date(obj.v)
+              if (date.toUTCString() !== 'Invalid Date') {
+                obj.v = dateFormat(date)
+              }
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    if (parseParams.timeHeaders.length > 0) {
+      let timeCols: number[] = []
+      parseParams.timeHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          timeCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        timeCols.forEach((element) => {
+          const obj = row[element]
+          if (
+            isStringNumber(obj.v) ||
+            isStringDecimal(obj.v) ||
+            obj.v.includes('E-')
+          ) {
+            const date = excelDateToJSDate(Number(obj.v))
+
+            obj.v = dateToUtcTime(date)
+          }
+          row[element] = obj
+        })
+      })
+    }
+    if (parseParams.dateTimeHeaders.length > 0) {
+      let dateTimeCols: number[] = []
+      parseParams.dateTimeHeaders.forEach((element: any) => {
+        if (headers.indexOf(element) !== -1) {
+          dateTimeCols.push(headers.indexOf(element))
+        }
+      })
+      data.forEach((row: any[]) => {
+        dateTimeCols.forEach((element) => {
+          const obj = row[element]
+          if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
+            const date = excelDateToJSDate(Number(obj.v))
+            obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
+          } else {
+            if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
+              let str = obj.v.substring(0, obj.v.indexOf(':'))
+              str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
+              obj.v = str
+            }
+            const date = new Date(obj.v)
+            if (date.toUTCString() !== 'Invalid Date') {
+              obj.v = dateFormat(date) + ' ' + dateToTime(date)
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    return data
+  }
+
+  private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
+    if (parseParams.xlRules.length > 0) {
+      const xlRuleCols: any = []
+      parseParams.xlRules.forEach((element: any) => {
+        if (headers.indexOf(element.XL_COLUMN) !== -1) {
+          element['index'] = headers.indexOf(element.XL_COLUMN)
+          xlRuleCols.push(element)
+        }
+      })
+      data.forEach((row: any[]) => {
+        xlRuleCols.forEach((element: any) => {
+          const obj = row[element.index]
+          if (element.XL_RULE === 'FORMULA') {
+            if ('f' in obj) {
+              if (obj['t'] === 'n') {
+                obj['v'] = '=' + obj['f']
+              } else {
+                obj['w'] = '=' + obj['f']
+              }
+            }
+          }
+          row[element] = obj
+        })
+      })
+    }
+    return data
+  }
+
+  private toBstr(res: any) {
+    let bytes = new Uint8Array(res)
+    let binary = ''
+    let length = bytes.byteLength
+    for (let i = 0; i < length; i++) {
+      binary += String.fromCharCode(bytes[i])
+    }
+    return binary
+  }
+}

client/src/app/spreadsheet.worker.ts

@@ -0,0 +1,12 @@
+/// <reference lib="webworker" />
+
+import * as XLSX from '@sheet/crypto'
+
+addEventListener('message', ({ data }) => {
+  const input = data as {
+    data: any,
+    opts?: XLSX.ParsingOptions | undefined
+  }
+
+  console.log('input', input)
+});

client/tsconfig.worker.json

@@ -0,0 +1,15 @@
+/* To learn more about this file see: https://angular.io/config/tsconfig. */
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./out-tsc/worker",
+    "lib": [
+      "es2018",
+      "webworker"
+    ],
+    "types": []
+  },
+  "include": [
+    "src/**/*.worker.ts"
+  ]
+}