fix: excel with password flow, introducing web worker for XLSX.read
This commit is contained in:
Mihajlo Medjedovic
parent
67436f4ff9
commit
a3ce367950
@ -69,7 +69,8 @@
|
||||
],
|
||||
"scripts": [
|
||||
"node_modules/marked/marked.min.js"
|
||||
]
|
||||
],
|
||||
"webWorkerTsConfig": "tsconfig.worker.json"
|
||||
},
|
||||
"configurations": {
|
||||
"production": {
|
||||
@ -148,7 +149,8 @@
|
||||
"src/styles.scss"
|
||||
],
|
||||
"scripts": [],
|
||||
"karmaConfig": "karma.conf.js"
|
||||
"karmaConfig": "karma.conf.js",
|
||||
"webWorkerTsConfig": "tsconfig.worker.json"
|
||||
}
|
||||
},
|
||||
"lint": {
|
||||
|
||||
@ -57,12 +57,10 @@ import { LicenceService } from '../services/licence.service'
|
||||
import * as numbro from 'numbro'
|
||||
import * as languages from 'numbro/dist/languages.min'
|
||||
import { FileUploadEncoding } from '../models/FileUploadEncoding'
|
||||
import {
|
||||
ParseResult,
|
||||
SpreadsheetService
|
||||
} from '../services/spreadsheet.service'
|
||||
import { SpreadsheetService } from '../services/spreadsheet.service'
|
||||
import { UploadFileResponse } from '../models/UploadFile'
|
||||
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
|
||||
import { ParseResult } from '../models/ParseResult.interface'
|
||||
|
||||
@Component({
|
||||
selector: 'app-editor',
|
||||
|
||||
23
client/src/app/models/ParseParams.interface.ts
Normal file
23
client/src/app/models/ParseParams.interface.ts
Normal file
@ -0,0 +1,23 @@
|
||||
import { DcValidator } from "../shared/dc-validator/dc-validator"
|
||||
import { FileUploadEncoding } from "./FileUploadEncoding"
|
||||
import { FileUploader } from "./FileUploader.class"
|
||||
import { ExcelRule } from "./TableData"
|
||||
|
||||
export interface ParseParams {
|
||||
file: File
|
||||
password?: string
|
||||
dcValidator: DcValidator
|
||||
/**
|
||||
* Parse function will manipulate and return the uploader array which can be provided with files already in the queue
|
||||
* Otherwise new empty instance will be created.
|
||||
*/
|
||||
uploader?: FileUploader
|
||||
headerPks: string[]
|
||||
headerArray: string[]
|
||||
headerShow: string[]
|
||||
timeHeaders: string[]
|
||||
dateHeaders: string[]
|
||||
dateTimeHeaders: string[]
|
||||
xlRules: ExcelRule[]
|
||||
encoding?: FileUploadEncoding
|
||||
}
|
||||
15
client/src/app/models/ParseResult.interface.ts
Normal file
15
client/src/app/models/ParseResult.interface.ts
Normal file
@ -0,0 +1,15 @@
|
||||
import { FileUploader } from "./FileUploader.class"
|
||||
import SheetInfo from "./SheetInfo"
|
||||
|
||||
export interface ParseResult {
|
||||
/**
|
||||
* In case of CSV file, won't be returned
|
||||
*/
|
||||
data?: any[]
|
||||
/**
|
||||
* In case of CSV file, won't be returned
|
||||
*/
|
||||
headerShow?: string[]
|
||||
rangeSheetRes?: SheetInfo
|
||||
uploader: FileUploader
|
||||
}
|
||||
@ -11,7 +11,6 @@ import {
|
||||
SasService,
|
||||
SasStoreService
|
||||
} from '../services'
|
||||
import * as XLSX from '@sheet/crypto'
|
||||
import { globals } from '../_globals'
|
||||
import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
|
||||
import { DcValidator } from '../shared/dc-validator/dc-validator'
|
||||
@ -19,7 +18,6 @@ import { ExcelRule } from '../models/TableData'
|
||||
import { HotTableInterface } from '../models/HotTable.interface'
|
||||
import { Col } from '../shared/dc-validator/models/col.model'
|
||||
import {
|
||||
ParseResult,
|
||||
SpreadsheetService
|
||||
} from '../services/spreadsheet.service'
|
||||
import Handsontable from 'handsontable'
|
||||
@ -31,6 +29,7 @@ import { ColumnSettings } from 'handsontable/settings'
|
||||
import { UploadFile } from '@sasjs/adapter'
|
||||
import { UploadFileResponse } from '../models/UploadFile'
|
||||
import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
|
||||
import { ParseResult } from '../models/ParseResult.interface'
|
||||
|
||||
@Component({
|
||||
selector: 'app-multi-dataset',
|
||||
@ -292,6 +291,7 @@ export class MultiDatasetComponent implements OnInit {
|
||||
this.spreadsheetService
|
||||
.parseExcelFile({
|
||||
file: this.selectedFile!,
|
||||
password: this.selectedFile!.password || undefined,
|
||||
dcValidator: parsedDataset.datasetInfo.dcValidator!,
|
||||
headerPks: parsedDataset.datasetInfo.headerPks,
|
||||
headerArray: parsedDataset.datasetInfo.headerArray,
|
||||
@ -518,7 +518,23 @@ export class MultiDatasetComponent implements OnInit {
|
||||
* convention. {@link isValidDatasetFormat}
|
||||
*/
|
||||
async onAutoDetectColumns() {
|
||||
this.sheetNames = await this.parseExcelSheetNames()
|
||||
let passwordError = false
|
||||
|
||||
await this.parseExcelSheetNames()
|
||||
.then((sheetNames) => {
|
||||
this.sheetNames = sheetNames
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err.includes('password')) {
|
||||
passwordError = true
|
||||
}
|
||||
})
|
||||
|
||||
if (passwordError) {
|
||||
this.onDiscardFile()
|
||||
this.eventService.showInfoModal('Locked file', 'We failed to unlock the file.')
|
||||
return
|
||||
}
|
||||
|
||||
if (this.sheetNames) {
|
||||
this.matchedDatasets = []
|
||||
@ -824,42 +840,18 @@ export class MultiDatasetComponent implements OnInit {
|
||||
*
|
||||
* @returns sheet names in string array
|
||||
*/
|
||||
private parseExcelSheetNames(): Promise<string[]> {
|
||||
private async parseExcelSheetNames(): Promise<string[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader()
|
||||
if (!this.selectedFile) return resolve([])
|
||||
|
||||
if (!this.selectedFile) {
|
||||
console.warn('selectedFile is missing')
|
||||
return resolve([])
|
||||
}
|
||||
this.spreadsheetService.parseExcelSheetNames(this.selectedFile)
|
||||
.then((parsed) => {
|
||||
if (parsed.password) this.selectedFile!.password = parsed.password
|
||||
|
||||
reader.onload = (event: ProgressEvent<FileReader>) => {
|
||||
if (!event?.target) {
|
||||
console.warn('File reader event.target is missing')
|
||||
return
|
||||
}
|
||||
|
||||
const data = event.target.result
|
||||
const workbook = XLSX.read(data, {
|
||||
// Load file minimally to parse sheets
|
||||
bookSheets: true,
|
||||
type: 'binary'
|
||||
})
|
||||
|
||||
try {
|
||||
const sheet_names_list = workbook.SheetNames
|
||||
|
||||
return resolve(sheet_names_list)
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
}
|
||||
}
|
||||
|
||||
reader.onerror = function (ex) {
|
||||
console.log(ex)
|
||||
}
|
||||
|
||||
reader.readAsBinaryString(this.selectedFile)
|
||||
return resolve(parsed.sheetNames)
|
||||
}).catch((err) => {
|
||||
return reject(err)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@ -1035,5 +1027,6 @@ export interface SubmittedCsvDatasetResult {
|
||||
}
|
||||
|
||||
export interface SelectedFile extends File {
|
||||
sizeMB?: number
|
||||
sizeMB?: number,
|
||||
password?: string
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,4 +1,4 @@
|
||||
<ng-container *ngIf="options$ | async as options">
|
||||
<div *ngIf="options$ | async as options" class="excel-password-root">
|
||||
<clr-modal
|
||||
[clrModalOpen]="options.open"
|
||||
[clrModalSize]="'md'"
|
||||
@ -11,6 +11,7 @@
|
||||
<p class="m-0">Please enter password:</p>
|
||||
<input
|
||||
#filePasswordInput
|
||||
[(ngModel)]="passwordInput"
|
||||
data-lpignore="true"
|
||||
autocomplete="off"
|
||||
id="filePasswordInput"
|
||||
@ -19,20 +20,25 @@
|
||||
/>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<p *ngIf="options.error" class="m-0 color-red">
|
||||
Sorry that didn't work, try again.
|
||||
</p>
|
||||
<button type="button" class="btn btn-sm btn-outline" (click)="close()">
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="btn btn-sm btn-success-outline"
|
||||
[disabled]="filePasswordInput.value.length < 1"
|
||||
(click)="close(filePasswordInput.value)"
|
||||
>
|
||||
Unlock
|
||||
</button>
|
||||
<div>
|
||||
<p *ngIf="options.error" class="m-0 color-red">
|
||||
Sorry that didn't work, try again.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="buttons">
|
||||
<button type="button" class="btn btn-sm btn-outline" (click)="close()">
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="btn btn-sm btn-success-outline"
|
||||
[disabled]="filePasswordInput.value.length < 1"
|
||||
(click)="close(filePasswordInput.value)"
|
||||
>
|
||||
Unlock
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</clr-modal>
|
||||
</ng-container>
|
||||
</div>
|
||||
|
||||
@ -0,0 +1,18 @@
|
||||
.excel-password-root {
|
||||
::ng-deep {
|
||||
.modal {
|
||||
z-index: 1060;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.modal-footer {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
|
||||
.buttons {
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
}
|
||||
}
|
||||
@ -1,24 +1,26 @@
|
||||
import { Component } from '@angular/core'
|
||||
import { Observable } from 'rxjs'
|
||||
import {
|
||||
ExcelPasswordModalService,
|
||||
Options
|
||||
ExcelPasswordModalService
|
||||
} from './excel-password-modal.service'
|
||||
import { Options } from './models/options.interface'
|
||||
|
||||
@Component({
|
||||
selector: 'app-excel-password-modal',
|
||||
styleUrls: ['./excel-password-modal.component.scss'],
|
||||
templateUrl: './excel-password-modal.component.html'
|
||||
})
|
||||
export class ExcelPasswordModalComponent {
|
||||
options$: Observable<Options>
|
||||
options$: Observable<Options> = this.excelPasswordModalService.optionsSubject$
|
||||
|
||||
fileUnlockError: boolean = false
|
||||
|
||||
constructor(private excelPasswordModalService: ExcelPasswordModalService) {
|
||||
this.options$ = this.excelPasswordModalService.optionsSubject$
|
||||
}
|
||||
passwordInput: string = ''
|
||||
|
||||
constructor(private excelPasswordModalService: ExcelPasswordModalService) {}
|
||||
|
||||
close(password?: string) {
|
||||
this.passwordInput = ''
|
||||
this.excelPasswordModalService.close(password)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,17 +1,7 @@
|
||||
import { Injectable } from '@angular/core'
|
||||
import { Subject, Observable } from 'rxjs'
|
||||
|
||||
export interface Options extends OpenOptions {
|
||||
open: boolean
|
||||
}
|
||||
|
||||
export interface Result {
|
||||
password: string | undefined
|
||||
}
|
||||
|
||||
export interface OpenOptions {
|
||||
error?: boolean
|
||||
}
|
||||
import { OpenOptions, Options } from './models/options.interface'
|
||||
import { Result } from './models/result.interface'
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
export interface OpenOptions {
|
||||
error?: boolean
|
||||
}
|
||||
|
||||
export interface Options extends OpenOptions {
|
||||
open: boolean
|
||||
}
|
||||
@ -0,0 +1,3 @@
|
||||
export interface Result {
|
||||
password: string | undefined
|
||||
}
|
||||
5
client/src/app/shared/spreadsheet-util/random.class.ts
Normal file
5
client/src/app/shared/spreadsheet-util/random.class.ts
Normal file
@ -0,0 +1,5 @@
|
||||
export class RandomClass {
|
||||
constructor(jo: string) {
|
||||
|
||||
}
|
||||
}
|
||||
904
client/src/app/shared/spreadsheet-util/spreadsheet-util.ts
Normal file
904
client/src/app/shared/spreadsheet-util/spreadsheet-util.ts
Normal file
@ -0,0 +1,904 @@
|
||||
import { isSpecialMissing } from '@sasjs/utils/input/validators'
|
||||
import { dateFormat, dateToUtcTime, dateToTime } from "src/app/editor/utils/date.utils"
|
||||
import { getMissingHeaders, excelDateToJSDate } from "src/app/editor/utils/grid.utils"
|
||||
import { isStringNumber, isStringDecimal } from "src/app/editor/utils/types.utils"
|
||||
import { FileUploader } from "src/app/models/FileUploader.class"
|
||||
import SheetInfo from "src/app/models/SheetInfo"
|
||||
import { blobToFile } from "src/app/xlmap/utils/file.utils"
|
||||
import * as XLSX from '@sheet/crypto'
|
||||
import { LicenceState } from "src/app/models/LicenceState"
|
||||
import { BehaviorSubject } from "rxjs"
|
||||
import { ParseParams } from 'src/app/models/ParseParams.interface'
|
||||
import { ParseResult } from 'src/app/models/ParseResult.interface'
|
||||
import { OpenOptions } from '../excel-password-modal/models/options.interface'
|
||||
|
||||
/**
|
||||
* Used in combination with buffer
|
||||
*/
|
||||
import * as iconv from 'iconv-lite'
|
||||
/**
|
||||
* In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
|
||||
* Which will be send to backend
|
||||
*/
|
||||
import { Buffer } from 'buffer'
|
||||
type AOA = any[][]
|
||||
|
||||
export interface ConstructorParams {
|
||||
licenceState: BehaviorSubject<LicenceState>
|
||||
}
|
||||
|
||||
export class SpreadsheetUtil {
|
||||
private licenceState: BehaviorSubject<LicenceState>
|
||||
|
||||
constructor(params: ConstructorParams) {
|
||||
this.licenceState = params.licenceState
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses attached file and searches fo the matching data
|
||||
*
|
||||
* @param parseParams params required for parsing the file
|
||||
* @param onParseStateChange callback used to inform about parsing state
|
||||
* so the user of the function can update the UI with latest info
|
||||
* @param onTableFoundEvent callback fired when table range is found in the file
|
||||
*
|
||||
* @returns parsed list of files to upload and JSON data ready for HOT usage
|
||||
*/
|
||||
public parseExcelFile(
|
||||
parseParams: ParseParams,
|
||||
promptExcelPassword: (options?: OpenOptions) => Promise<string | undefined>,
|
||||
onParseStateChange?: (uploadState: string) => void,
|
||||
onTableFoundEvent?: (info: string) => void
|
||||
): Promise<ParseResult | undefined> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// If file size is bigger then 2 MB we need to load its bytes in chunks
|
||||
const sizeInMB = this.bytesToMB(parseParams.file.size)
|
||||
|
||||
if (sizeInMB > 2) {
|
||||
|
||||
}
|
||||
|
||||
let data: any[] = []
|
||||
const uploader: FileUploader = parseParams.uploader || new FileUploader()
|
||||
|
||||
const file: File = parseParams.file
|
||||
const filename = file.name
|
||||
|
||||
if (!parseParams.encoding) parseParams.encoding = 'UTF-8'
|
||||
|
||||
if (onParseStateChange)
|
||||
onParseStateChange(`Loading ${filename} into the browser`)
|
||||
|
||||
let foundData = {
|
||||
sheet: ''
|
||||
}
|
||||
|
||||
let fileType = filename.slice(
|
||||
filename.lastIndexOf('.') + 1,
|
||||
filename.lastIndexOf('.') + 4
|
||||
)
|
||||
|
||||
if (fileType.toLowerCase() === 'xls') {
|
||||
let reader: FileReader = new FileReader()
|
||||
|
||||
const self = this
|
||||
reader.onload = async (theFile: any) => {
|
||||
/* read workbook */
|
||||
const bstr = this.toBstr(theFile.target.result)
|
||||
let wb: XLSX.WorkBook | undefined = undefined
|
||||
let fileUnlocking: boolean = false
|
||||
|
||||
const xlsxOptions: XLSX.ParsingOptions = {
|
||||
type: 'binary',
|
||||
cellDates: false,
|
||||
cellFormula: true,
|
||||
cellStyles: true,
|
||||
cellNF: false,
|
||||
cellText: false,
|
||||
password: parseParams.password
|
||||
}
|
||||
|
||||
try {
|
||||
// wb = this.xlsxRead(bstr, {
|
||||
// ...xlsxOptions
|
||||
// })
|
||||
wb = XLSX.read(bstr, {
|
||||
...xlsxOptions
|
||||
})
|
||||
} catch (err: any) {
|
||||
if (err.message.toLowerCase().includes('password')) {
|
||||
fileUnlocking = true
|
||||
|
||||
let passwordError = false
|
||||
|
||||
while (fileUnlocking) {
|
||||
const password = await promptExcelPassword({
|
||||
error: passwordError
|
||||
})
|
||||
|
||||
if (password) {
|
||||
try {
|
||||
wb = XLSX.read(bstr, {
|
||||
...xlsxOptions,
|
||||
password: password
|
||||
})
|
||||
|
||||
fileUnlocking = false
|
||||
passwordError = false
|
||||
} catch (err: any) {
|
||||
passwordError = true
|
||||
|
||||
if (!err.message.toLowerCase().includes('password')) {
|
||||
fileUnlocking = false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fileUnlocking = false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return reject('Error reading the file')
|
||||
}
|
||||
}
|
||||
|
||||
if (!wb) {
|
||||
return reject('No workbook found.')
|
||||
}
|
||||
|
||||
/* save data */
|
||||
let isComplete: boolean = false
|
||||
let missingHeaders: string[] = []
|
||||
|
||||
const csvArrayHeaders: string[] = [
|
||||
'_____DELETE__THIS__RECORD_____',
|
||||
...parseParams.headerArray
|
||||
]
|
||||
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
|
||||
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
|
||||
(map: any, obj: string) => {
|
||||
map[obj] = -1
|
||||
return map
|
||||
},
|
||||
{}
|
||||
)
|
||||
|
||||
let csvArrayData: any[] = []
|
||||
const rangeSheetRes: SheetInfo = this.getRangeAndSheet(
|
||||
wb,
|
||||
parseParams
|
||||
)
|
||||
missingHeaders = rangeSheetRes.missingHeaders
|
||||
|
||||
if (rangeSheetRes.foundData) {
|
||||
isComplete = true
|
||||
csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
|
||||
const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]
|
||||
|
||||
if (onParseStateChange)
|
||||
onParseStateChange(
|
||||
`Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
|
||||
)
|
||||
|
||||
let startAddress = ''
|
||||
let endAddress = ''
|
||||
|
||||
for (
|
||||
let row = rangeSheetRes.startRow;
|
||||
row < rangeSheetRes.endRow;
|
||||
++row
|
||||
) {
|
||||
const arr: any[] = []
|
||||
|
||||
csvArrayHeadersLower.forEach((x) => {
|
||||
const col = csvArrayHeadersMap[x]
|
||||
const addr = XLSX.utils.encode_cell({
|
||||
r: rangeSheetRes.rangeStartRow + row,
|
||||
c: rangeSheetRes.rangeStartCol + col
|
||||
})
|
||||
|
||||
let cell
|
||||
|
||||
if (!ws[addr]) {
|
||||
cell = { v: '' }
|
||||
} else {
|
||||
cell = ws[addr]
|
||||
}
|
||||
|
||||
if (startAddress === '' && ws[addr]) startAddress = addr
|
||||
endAddress = addr
|
||||
|
||||
arr.push(cell)
|
||||
})
|
||||
|
||||
// If we found at least one non empty value it means it is not empty row
|
||||
// othervise, it is empty row
|
||||
let arrNonEmptyValue = arr.find((x) => x.v !== '')
|
||||
|
||||
if (arrNonEmptyValue) csvArrayData.push(arr)
|
||||
}
|
||||
|
||||
rangeSheetRes.rangeAddress = `${startAddress}:${endAddress}`
|
||||
|
||||
if (onTableFoundEvent)
|
||||
onTableFoundEvent(
|
||||
`Sheet: ${rangeSheetRes.sheetName}\nRange: ${rangeSheetRes.rangeAddress}`
|
||||
)
|
||||
} else {
|
||||
missingHeaders = rangeSheetRes.missingHeaders
|
||||
}
|
||||
|
||||
if (missingHeaders.length > 0) {
|
||||
missingHeaders.sort(function compareSecondColumn(a, b) {
|
||||
if (a[1] === b[1]) {
|
||||
return 0
|
||||
} else {
|
||||
return a[1] > b[1] ? -1 : 1
|
||||
}
|
||||
})
|
||||
let abortMsg = missingHeaders
|
||||
.map((x) => x[0])
|
||||
.slice(0, 5)
|
||||
.join('\n')
|
||||
|
||||
uploader.queue.pop()
|
||||
return reject(abortMsg)
|
||||
}
|
||||
|
||||
// If first row is empty, that means no data has been found
|
||||
if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
|
||||
let abortMsg = 'No relevant data found in File !'
|
||||
|
||||
uploader.queue.pop()
|
||||
return reject(abortMsg)
|
||||
}
|
||||
|
||||
if (
|
||||
parseParams.dateTimeHeaders.length > 0 ||
|
||||
parseParams.dateHeaders.length > 0 ||
|
||||
parseParams.timeHeaders.length > 0
|
||||
) {
|
||||
csvArrayData = this.updateDateTimeCols(
|
||||
csvArrayHeaders,
|
||||
csvArrayData,
|
||||
parseParams
|
||||
)
|
||||
}
|
||||
|
||||
if (parseParams.xlRules.length > 0) {
|
||||
csvArrayData = this.updateXLRuleCols(
|
||||
csvArrayHeaders,
|
||||
csvArrayData,
|
||||
parseParams
|
||||
)
|
||||
}
|
||||
|
||||
if (!isComplete) {
|
||||
let abortMsg = ''
|
||||
|
||||
if (missingHeaders.length === 0) {
|
||||
abortMsg = 'No relevant data found in File !'
|
||||
} else {
|
||||
missingHeaders.sort(function compareSecondColumn(a, b) {
|
||||
if (a[1] === b[1]) {
|
||||
return 0
|
||||
} else {
|
||||
return a[1] > b[1] ? -1 : 1
|
||||
}
|
||||
})
|
||||
abortMsg = missingHeaders
|
||||
.map((x) => x[0])
|
||||
.slice(0, 5)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
// abort message is fired, return undefined
|
||||
uploader.queue.pop()
|
||||
return reject(abortMsg)
|
||||
} else {
|
||||
parseParams.headerShow = csvArrayHeaders
|
||||
csvArrayData = csvArrayData.map((row: any) =>
|
||||
row.map((col: any) => (col.t === 'n' ? col.v : col.w))
|
||||
)
|
||||
|
||||
csvArrayData = csvArrayData.map((row: any) => {
|
||||
return row.map((col: any, index: number) => {
|
||||
if (!col && col !== 0) col = ''
|
||||
|
||||
/**
|
||||
* Keeping this for the reference
|
||||
* Code below used to convert JSON to CSV
|
||||
* now the XLSX is converting to CSV
|
||||
*/
|
||||
// if (isNaN(col)) {
|
||||
// // Match and replace the double quotes, ignore the first and last char
|
||||
// // in case they are double quotes already
|
||||
// col = col.replace(/(?<!^)"(?!$)/g, '""')
|
||||
|
||||
// if (col.search(/,/g) > -1 ||
|
||||
// col.search(/\r|\n/g) > -1
|
||||
// ) {
|
||||
// // Missing quotes at the end
|
||||
// if (col.search(/"$/g) < 0) {
|
||||
// col = col + '"' // So we add them
|
||||
// }
|
||||
|
||||
// // Missing quotes at the start
|
||||
// if (col.search(/^"/g) < 0) {
|
||||
// col = '"' + col // So we add them
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
const colName = parseParams.headerShow[index]
|
||||
const colRule = parseParams.dcValidator?.getRule(colName)
|
||||
|
||||
if (colRule?.type === 'numeric') {
|
||||
if (isSpecialMissing(col) && !col.includes('.'))
|
||||
col = '.' + col
|
||||
}
|
||||
|
||||
return col
|
||||
})
|
||||
})
|
||||
|
||||
data = csvArrayData
|
||||
|
||||
// Apply licence rows limitation if exists, it is only affecting data
|
||||
// which will be send to SAS
|
||||
const strippedCsvArrayData = csvArrayData.slice(
|
||||
0,
|
||||
this.licenceState.value.submit_rows_limit
|
||||
)
|
||||
// To submit to sas service, we need clean version of CSV of file
|
||||
// attached. XLSX will do the parsing and heavy lifting
|
||||
// First we create worksheet of json (data we extracted)
|
||||
let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
|
||||
skipHeader: true
|
||||
})
|
||||
// create CSV to be uploaded from worksheet
|
||||
let csvContentClean = XLSX.utils.sheet_to_csv(ws)
|
||||
// Prepend headers
|
||||
csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
|
||||
|
||||
// Blob from which CSV file will be created depending of the selected
|
||||
// encoding
|
||||
let blob: Blob
|
||||
|
||||
if (parseParams.encoding === 'WLATIN1') {
|
||||
// WLATIN1
|
||||
let encoded = iconv.decode(
|
||||
Buffer.from(csvContentClean),
|
||||
'CP-1252'
|
||||
)
|
||||
blob = new Blob([encoded], { type: 'application/csv' })
|
||||
} else {
|
||||
// UTF-8
|
||||
blob = new Blob([csvContentClean], { type: 'application/csv' })
|
||||
}
|
||||
|
||||
let newCSVFile: File = blobToFile(blob, filename + '.csv')
|
||||
uploader.addToQueue([newCSVFile])
|
||||
}
|
||||
|
||||
if (data.length === 0) {
|
||||
return reject(
|
||||
`Table in the file is empty. Data found on sheet: ${foundData.sheet}`
|
||||
)
|
||||
}
|
||||
|
||||
return resolve({
|
||||
uploader,
|
||||
data,
|
||||
rangeSheetRes,
|
||||
headerShow: parseParams.headerShow
|
||||
})
|
||||
}
|
||||
console.log('before read file as array buffer')
|
||||
reader.readAsArrayBuffer(file)
|
||||
} else if (fileType.toLowerCase() === 'csv') {
|
||||
if (this.licenceState.value.submit_rows_limit !== Infinity) {
|
||||
uploader.queue.pop()
|
||||
return reject(
|
||||
'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
|
||||
)
|
||||
}
|
||||
|
||||
if (parseParams.encoding === 'WLATIN1') {
|
||||
let reader = new FileReader()
|
||||
const self = this
|
||||
// Closure to capture the file information.
|
||||
reader.onload = (theFile: any) => {
|
||||
let encoded = iconv.decode(
|
||||
Buffer.from(theFile.target.result),
|
||||
'CP-1252'
|
||||
)
|
||||
let blob = new Blob([encoded], { type: fileType })
|
||||
let encodedFile: File = blobToFile(blob, filename)
|
||||
uploader.queue.pop()
|
||||
uploader.addToQueue([encodedFile])
|
||||
|
||||
resolve({
|
||||
uploader
|
||||
})
|
||||
}
|
||||
|
||||
reader.readAsArrayBuffer(file)
|
||||
} else {
|
||||
return resolve({
|
||||
uploader
|
||||
})
|
||||
}
|
||||
} else {
|
||||
let abortMsg =
|
||||
'Invalid file type "<b>' +
|
||||
filename +
|
||||
'</b>". Please upload csv or excel file.'
|
||||
|
||||
uploader.queue.pop()
|
||||
return reject(abortMsg)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
public bytesToMB(size: number): number {
|
||||
return parseFloat((size / (1024*1024)).toFixed(2))
|
||||
}
|
||||
|
||||
private xlsxRead(data: any, opts?: XLSX.ParsingOptions | undefined): XLSX.WorkBook {
|
||||
if (typeof Worker !== 'undefined') {
|
||||
// Create a new
|
||||
const worker = new Worker(new URL('../../spreadsheet.worker', import.meta.url));
|
||||
worker.onmessage = ({ data }) => {
|
||||
|
||||
};
|
||||
|
||||
worker.postMessage({
|
||||
data,
|
||||
opts
|
||||
});
|
||||
|
||||
return XLSX.read(data, opts) // Put in worker
|
||||
} else {
|
||||
// Web workers are not supported in this environment.
|
||||
// You should add a fallback so that your program still executes correctly.
|
||||
return XLSX.read(data, opts)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
|
||||
* @param wb Excel workbook
|
||||
* @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
|
||||
*/
|
||||
private getRangeAndSheet(
|
||||
wb: XLSX.WorkBook,
|
||||
parseParams: ParseParams
|
||||
): SheetInfo {
|
||||
let data = []
|
||||
|
||||
let rangeStartRow: number = 0
|
||||
let rangeStartCol: number = 0
|
||||
let startRow: number = -1
|
||||
let endRow: number = -1
|
||||
let sheetName: string = ''
|
||||
let isComplete = false
|
||||
let missingHeaders: string[] = []
|
||||
const csvArrayHeaders: string[] = [
|
||||
'_____DELETE__THIS__RECORD_____',
|
||||
...parseParams.headerArray
|
||||
]
|
||||
let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
|
||||
let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
|
||||
(map: any, obj: string) => {
|
||||
map[obj] = -1
|
||||
return map
|
||||
},
|
||||
{}
|
||||
)
|
||||
|
||||
wb.SheetNames.forEach((element: string) => {
|
||||
// Checking for required data in each sheet in workbook/
|
||||
if (isComplete) {
|
||||
return
|
||||
}
|
||||
|
||||
missingHeaders = []
|
||||
sheetName = element
|
||||
const ws: XLSX.WorkSheet = wb.Sheets[sheetName]
|
||||
|
||||
data = <AOA>XLSX.utils.sheet_to_json(ws, {
|
||||
header: 1,
|
||||
blankrows: true, // Without empty rows, if another table is below a table separated by the empty rows, startRow index is wrong
|
||||
defval: ''
|
||||
})
|
||||
|
||||
if (data.length <= 1) {
|
||||
return
|
||||
}
|
||||
|
||||
let tempArr: string[] = []
|
||||
parseParams.headerArray.forEach(() => tempArr.push(''))
|
||||
data.push(tempArr)
|
||||
|
||||
let foundHeaders = false
|
||||
|
||||
data.forEach((row: any, index: number) => {
|
||||
if (isComplete) {
|
||||
return
|
||||
}
|
||||
|
||||
if (foundHeaders) {
|
||||
let isDataEnd = true
|
||||
let isPkNull = false
|
||||
|
||||
csvArrayHeadersLower.forEach((x) => {
|
||||
const col = csvArrayHeadersMap[x]
|
||||
|
||||
if (row[col] !== '' && row[col] !== undefined) {
|
||||
isDataEnd = false
|
||||
} else {
|
||||
if (parseParams.headerPks.indexOf(x.toUpperCase()) !== -1) {
|
||||
isPkNull = true
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (isDataEnd || isPkNull) {
|
||||
endRow = index
|
||||
isComplete = true
|
||||
} else {
|
||||
if (startRow === -1) {
|
||||
startRow = index
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const rowLowerCase: string[] = row.map((x: any) =>
|
||||
x.toString().toLowerCase()
|
||||
)
|
||||
|
||||
// If in file there is no delete column, remove it from search of missing.
|
||||
// This way delete column will be optional to provide in file
|
||||
if (!rowLowerCase.includes('_____delete__this__record_____')) {
|
||||
const deleteIndex = csvArrayHeadersLower.indexOf(
|
||||
'_____delete__this__record_____'
|
||||
)
|
||||
|
||||
if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
|
||||
}
|
||||
|
||||
foundHeaders = true
|
||||
|
||||
csvArrayHeadersLower.forEach((x) => {
|
||||
if (rowLowerCase.indexOf(x) === -1) {
|
||||
foundHeaders = false
|
||||
}
|
||||
})
|
||||
|
||||
let result = []
|
||||
|
||||
result = this.findValidHeaders(
|
||||
rowLowerCase,
|
||||
csvArrayHeadersLower,
|
||||
index,
|
||||
sheetName,
|
||||
parseParams
|
||||
)
|
||||
|
||||
if (result[0] === false) {
|
||||
foundHeaders = false
|
||||
|
||||
if (result[1].length > 0) {
|
||||
result[1].forEach((data: string) => {
|
||||
missingHeaders.push(data)
|
||||
})
|
||||
}
|
||||
} else {
|
||||
csvArrayHeadersMap = result[1]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (isComplete) {
|
||||
this.update_sheet_range(ws)
|
||||
const worksheetSel = ws['!ref']
|
||||
|
||||
if (worksheetSel) {
|
||||
const range = XLSX.utils.decode_range(ws['!ref'] || '')
|
||||
rangeStartRow = range.s.r
|
||||
rangeStartCol = range.s.c
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// If start row is still -1 that means first row of found range is empty
|
||||
if (startRow === -1) isComplete = false
|
||||
|
||||
const returnObj: SheetInfo = {
|
||||
foundData: isComplete,
|
||||
sheetName,
|
||||
startRow,
|
||||
endRow,
|
||||
csvArrayHeadersMap,
|
||||
missingHeaders,
|
||||
rangeStartRow,
|
||||
rangeStartCol
|
||||
}
|
||||
|
||||
return returnObj
|
||||
}
|
||||
|
||||
private findValidHeaders(
|
||||
row: string[],
|
||||
headers: string[],
|
||||
rowNumber: number,
|
||||
tabName: string,
|
||||
parseParams: ParseParams
|
||||
): Array<any> {
|
||||
let headersFound = false
|
||||
const missingErrorArray = []
|
||||
let j = 0
|
||||
|
||||
while (j < row.length) {
|
||||
if (headersFound) {
|
||||
// return;
|
||||
} else {
|
||||
if (headers.indexOf(row[j]) !== -1) {
|
||||
let breakIndex
|
||||
let rowStart = 0
|
||||
let rowEnd = 0
|
||||
let arrStart = 0
|
||||
let foundHeadersArray: string[] = []
|
||||
let spaceBreak = false
|
||||
|
||||
for (let i = j; i < row.length; i++) {
|
||||
if (
|
||||
row[i] === '' ||
|
||||
(foundHeadersArray.indexOf(row[i]) !== -1 &&
|
||||
this.isColHeader(row[i], parseParams.headerArray))
|
||||
) {
|
||||
if (row[i] === '') {
|
||||
spaceBreak = true
|
||||
}
|
||||
|
||||
breakIndex = i
|
||||
break
|
||||
} else {
|
||||
foundHeadersArray.push(row[i])
|
||||
}
|
||||
}
|
||||
|
||||
let tempArray: string[] = []
|
||||
|
||||
if (breakIndex !== undefined) {
|
||||
tempArray = row.slice(j, breakIndex)
|
||||
arrStart = j
|
||||
rowEnd = breakIndex
|
||||
|
||||
if (spaceBreak) {
|
||||
rowStart = j
|
||||
j = breakIndex
|
||||
} else {
|
||||
rowStart = j
|
||||
j = breakIndex - 1
|
||||
}
|
||||
} else {
|
||||
tempArray = row.slice(j)
|
||||
rowStart = j
|
||||
arrStart = j
|
||||
rowEnd = row.length
|
||||
j = row.length
|
||||
}
|
||||
|
||||
let foundHeaders = true
|
||||
|
||||
//We check if there are missing headers
|
||||
headers.forEach((x) => {
|
||||
if (tempArray.indexOf(x) === -1) {
|
||||
foundHeaders = false
|
||||
}
|
||||
})
|
||||
|
||||
if (foundHeaders) {
|
||||
headersFound = true
|
||||
|
||||
let mapHeaders: any[] = headers
|
||||
|
||||
let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
|
||||
map[obj] = -1
|
||||
return map
|
||||
}, {})
|
||||
|
||||
let temp = row.slice(rowStart, rowEnd)
|
||||
|
||||
headers.forEach((x) => {
|
||||
csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
|
||||
})
|
||||
|
||||
return [true, csvArrayHeadersMap]
|
||||
} else {
|
||||
let missingHeaders = getMissingHeaders(tempArray, headers)
|
||||
|
||||
let missingMessage = '<b>TAB(' + tabName + ')</b>'
|
||||
missingErrorArray.push([
|
||||
missingMessage +
|
||||
' - ' +
|
||||
missingHeaders[1].join(',') +
|
||||
' ( missing ' +
|
||||
missingHeaders[0].join(',') +
|
||||
' )',
|
||||
missingHeaders[1].length
|
||||
])
|
||||
}
|
||||
}
|
||||
}
|
||||
j++
|
||||
}
|
||||
return [false, missingErrorArray]
|
||||
}
|
||||
|
||||
private isColHeader(col: string, headerArray: string[]) {
|
||||
return headerArray.indexOf(col.toUpperCase()) > -1
|
||||
}
|
||||
|
||||
/**
|
||||
* Function that updates the !ref range value provided in official docs.
|
||||
* @param ws worksheet to be updated
|
||||
*/
|
||||
private update_sheet_range(ws: XLSX.WorkSheet) {
|
||||
const range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
|
||||
|
||||
Object.keys(ws)
|
||||
.filter(function (x) {
|
||||
return x.charAt(0) != '!'
|
||||
})
|
||||
.map(XLSX.utils.decode_cell)
|
||||
.forEach(function (x: any) {
|
||||
range.s.c = Math.min(range.s.c, x.c)
|
||||
range.s.r = Math.min(range.s.r, x.r)
|
||||
range.e.c = Math.max(range.e.c, x.c)
|
||||
range.e.r = Math.max(range.e.r, x.r)
|
||||
})
|
||||
|
||||
ws['!ref'] = XLSX.utils.encode_range(range)
|
||||
}
|
||||
|
||||
/**
|
||||
* When excel is password protected we will display the password prompt for user to type password in.
|
||||
* @returns Password user input or undefined if discarded by user
|
||||
*/
|
||||
// private promptExcelPassword(): Promise<string | undefined> {
|
||||
// return new Promise((resolve, reject) => {
|
||||
// this.excelPasswordModalService.open().subscribe((result: Result) => {
|
||||
// resolve(result.password)
|
||||
// })
|
||||
// })
|
||||
// }
|
||||
|
||||
private updateDateTimeCols(
|
||||
headers: any,
|
||||
data: any,
|
||||
parseParams: ParseParams
|
||||
) {
|
||||
if (parseParams.dateHeaders.length > 0) {
|
||||
const dateCols: number[] = []
|
||||
parseParams.dateHeaders.forEach((element: any) => {
|
||||
if (headers.indexOf(element) !== -1) {
|
||||
dateCols.push(headers.indexOf(element))
|
||||
}
|
||||
})
|
||||
data.forEach((row: any[]) => {
|
||||
dateCols.forEach((element) => {
|
||||
const obj = row[element]
|
||||
if (isStringNumber(obj.v)) {
|
||||
const date = excelDateToJSDate(Number(obj.v))
|
||||
|
||||
obj.v =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
} else {
|
||||
if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
|
||||
const date = new Date(obj.v)
|
||||
if (date.toUTCString() !== 'Invalid Date') {
|
||||
obj.v = dateFormat(date)
|
||||
}
|
||||
}
|
||||
}
|
||||
row[element] = obj
|
||||
})
|
||||
})
|
||||
}
|
||||
if (parseParams.timeHeaders.length > 0) {
|
||||
let timeCols: number[] = []
|
||||
parseParams.timeHeaders.forEach((element: any) => {
|
||||
if (headers.indexOf(element) !== -1) {
|
||||
timeCols.push(headers.indexOf(element))
|
||||
}
|
||||
})
|
||||
data.forEach((row: any[]) => {
|
||||
timeCols.forEach((element) => {
|
||||
const obj = row[element]
|
||||
if (
|
||||
isStringNumber(obj.v) ||
|
||||
isStringDecimal(obj.v) ||
|
||||
obj.v.includes('E-')
|
||||
) {
|
||||
const date = excelDateToJSDate(Number(obj.v))
|
||||
|
||||
obj.v = dateToUtcTime(date)
|
||||
}
|
||||
row[element] = obj
|
||||
})
|
||||
})
|
||||
}
|
||||
if (parseParams.dateTimeHeaders.length > 0) {
|
||||
let dateTimeCols: number[] = []
|
||||
parseParams.dateTimeHeaders.forEach((element: any) => {
|
||||
if (headers.indexOf(element) !== -1) {
|
||||
dateTimeCols.push(headers.indexOf(element))
|
||||
}
|
||||
})
|
||||
data.forEach((row: any[]) => {
|
||||
dateTimeCols.forEach((element) => {
|
||||
const obj = row[element]
|
||||
if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
|
||||
const date = excelDateToJSDate(Number(obj.v))
|
||||
obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
|
||||
} else {
|
||||
if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
|
||||
let str = obj.v.substring(0, obj.v.indexOf(':'))
|
||||
str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
|
||||
obj.v = str
|
||||
}
|
||||
const date = new Date(obj.v)
|
||||
if (date.toUTCString() !== 'Invalid Date') {
|
||||
obj.v = dateFormat(date) + ' ' + dateToTime(date)
|
||||
}
|
||||
}
|
||||
row[element] = obj
|
||||
})
|
||||
})
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
private updateXLRuleCols(headers: any, data: any, parseParams: ParseParams) {
|
||||
if (parseParams.xlRules.length > 0) {
|
||||
const xlRuleCols: any = []
|
||||
parseParams.xlRules.forEach((element: any) => {
|
||||
if (headers.indexOf(element.XL_COLUMN) !== -1) {
|
||||
element['index'] = headers.indexOf(element.XL_COLUMN)
|
||||
xlRuleCols.push(element)
|
||||
}
|
||||
})
|
||||
data.forEach((row: any[]) => {
|
||||
xlRuleCols.forEach((element: any) => {
|
||||
const obj = row[element.index]
|
||||
if (element.XL_RULE === 'FORMULA') {
|
||||
if ('f' in obj) {
|
||||
if (obj['t'] === 'n') {
|
||||
obj['v'] = '=' + obj['f']
|
||||
} else {
|
||||
obj['w'] = '=' + obj['f']
|
||||
}
|
||||
}
|
||||
}
|
||||
row[element] = obj
|
||||
})
|
||||
})
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
private toBstr(res: any) {
|
||||
let bytes = new Uint8Array(res)
|
||||
let binary = ''
|
||||
let length = bytes.byteLength
|
||||
for (let i = 0; i < length; i++) {
|
||||
binary += String.fromCharCode(bytes[i])
|
||||
}
|
||||
return binary
|
||||
}
|
||||
}
|
||||
12
client/src/app/spreadsheet.worker.ts
Normal file
12
client/src/app/spreadsheet.worker.ts
Normal file
@ -0,0 +1,12 @@
|
||||
/// <reference lib="webworker" />
|
||||
|
||||
import * as XLSX from '@sheet/crypto'
|
||||
|
||||
addEventListener('message', ({ data }) => {
|
||||
const input = data as {
|
||||
data: any,
|
||||
opts?: XLSX.ParsingOptions | undefined
|
||||
}
|
||||
|
||||
console.log('input', input)
|
||||
});
|
||||
15
client/tsconfig.worker.json
Normal file
15
client/tsconfig.worker.json
Normal file
@ -0,0 +1,15 @@
|
||||
/* To learn more about this file see: https://angular.io/config/tsconfig. */
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./out-tsc/worker",
|
||||
"lib": [
|
||||
"es2018",
|
||||
"webworker"
|
||||
],
|
||||
"types": []
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.worker.ts"
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user