feat(multi load): multiple csv files
This commit is contained in:
Mihajlo Medjedovic
parent
cffeab813d
commit
4d276657b3
@ -7,7 +7,7 @@
|
||||
<button
|
||||
(click)="fileUploadInput.click()"
|
||||
class="btn btn-primary btn-sm"
|
||||
[disabled]="selectedFile !== null"
|
||||
[disabled]="selectedFile !== null || submittingCsv"
|
||||
>
|
||||
Browse file
|
||||
</button>
|
||||
@ -17,7 +17,7 @@
|
||||
id="file-upload"
|
||||
type="file"
|
||||
(change)="onFileChange($event)"
|
||||
appFileSelect
|
||||
multiple
|
||||
/>
|
||||
</div>
|
||||
|
||||
@ -91,7 +91,7 @@
|
||||
</div>
|
||||
|
||||
<div
|
||||
*ngIf="selectedFile === null"
|
||||
*ngIf="selectedFile === null && !submittingCsv"
|
||||
class="no-table-selected pointer-events-none"
|
||||
>
|
||||
<clr-icon
|
||||
@ -104,8 +104,8 @@
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<ng-container *ngIf="selectedFile !== null">
|
||||
<ng-container *ngIf="!parsedDatasets.length">
|
||||
<ng-container *ngIf="selectedFile !== null || submittingCsv">
|
||||
<ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
|
||||
<div class="d-flex clr-justify-content-center mt-15">
|
||||
<div class="dataset-input-wrapper">
|
||||
<p cds-text="secondary regular" class="mb-20">
|
||||
@ -148,9 +148,9 @@
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<ng-container *ngIf="parsedDatasets.length">
|
||||
<ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
|
||||
<div
|
||||
*ngIf="!activeParsedDataset && !activeSubmittedDataset"
|
||||
*ngIf="!activeParsedDataset"
|
||||
class="no-table-selected pointer-events-none"
|
||||
>
|
||||
<clr-icon
|
||||
@ -159,12 +159,7 @@
|
||||
class="is-info icon-dc-fill"
|
||||
></clr-icon>
|
||||
<p class="text-center color-gray mt-10" cds-text="section">
|
||||
Please select a dataset on the left to
|
||||
{{
|
||||
!submittedDatasets.length
|
||||
? 'review data'
|
||||
: 'review submitted results'
|
||||
}}
|
||||
Please select a dataset on the left to review the data
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@ -238,10 +233,26 @@
|
||||
</ng-container>
|
||||
</ng-container>
|
||||
|
||||
<ng-container *ngIf="submittedDatasets.length">
|
||||
<div
|
||||
*ngIf="!activeSubmittedDataset"
|
||||
class="no-table-selected pointer-events-none"
|
||||
>
|
||||
<clr-icon
|
||||
shape="warning-standard"
|
||||
size="40"
|
||||
class="is-info icon-dc-fill"
|
||||
></clr-icon>
|
||||
<p class="text-center color-gray mt-10" cds-text="section">
|
||||
Please select a dataset on the left to review the submit results
|
||||
</p>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
<ng-container *ngIf="activeSubmittedDataset">
|
||||
<div class="d-flex clr-justify-content-between p-10">
|
||||
<div>
|
||||
<p cds-text="secondary regular" class="mb-10">
|
||||
<p *ngIf="activeSubmittedDataset.parseResult" cds-text="secondary regular" class="mb-10">
|
||||
Found in range:
|
||||
<strong
|
||||
>"{{
|
||||
@ -274,6 +285,7 @@
|
||||
|
||||
<div>
|
||||
<button
|
||||
*ngIf="!submittingCsv && activeSubmittedDataset.error"
|
||||
(click)="reSubmitTable(activeSubmittedDataset)"
|
||||
class="btn btn-primary mt-10"
|
||||
[clrLoading]="submitLoading"
|
||||
@ -360,3 +372,18 @@
|
||||
</button>
|
||||
</div>
|
||||
</clr-modal>
|
||||
|
||||
<clr-modal [(clrModalOpen)]="csvSubmitting" [clrModalClosable]="false">
|
||||
<h3 class="modal-title">
|
||||
Submitting {{ csvFiles.length }} CSV {{ csvFiles.length === 1 ? 'file' : 'files'}}
|
||||
</h3>
|
||||
<div class="modal-body">
|
||||
<div class="text-center">
|
||||
<clr-spinner clrMedium></clr-spinner>
|
||||
</div>
|
||||
|
||||
<p cds-text="caption_clean" class="mt-10 text-center">
|
||||
This will take few moments
|
||||
</p>
|
||||
</div>
|
||||
</clr-modal>
|
||||
@ -28,6 +28,7 @@ import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.mod
|
||||
import { CellChange, ChangeSource } from 'handsontable/common'
|
||||
import { baseAfterGetColHeader } from '../shared/utils/hot.utils'
|
||||
import { ColumnSettings } from 'handsontable/settings'
|
||||
import { UploadFile } from '@sasjs/adapter'
|
||||
|
||||
@Component({
|
||||
selector: 'app-multi-dataset',
|
||||
@ -44,6 +45,9 @@ export class MultiDatasetComponent implements OnInit {
|
||||
public hotTableMaxRows =
|
||||
this.licenceState.value.viewer_rows_allowed || Infinity
|
||||
|
||||
public csvFiles: UploadFile[] = []
|
||||
public csvSubmitting: boolean = false
|
||||
|
||||
public selectedFile: File | null = null
|
||||
public parsedDatasets: ParsedDataset[] = []
|
||||
public submittedDatasets: SubmittedDatasetResult[] = []
|
||||
@ -67,10 +71,7 @@ export class MultiDatasetComponent implements OnInit {
|
||||
public submitReasonMessage: string = ''
|
||||
|
||||
public hotUserDatasets: Handsontable.GridSettings = {
|
||||
colHeaders: [
|
||||
'Library',
|
||||
'Table'
|
||||
],
|
||||
colHeaders: ['Library', 'Table'],
|
||||
data: [
|
||||
['', ''],
|
||||
['', ''],
|
||||
@ -122,6 +123,7 @@ export class MultiDatasetComponent implements OnInit {
|
||||
private helperService: HelperService,
|
||||
private sasStoreService: SasStoreService,
|
||||
private spreadsheetService: SpreadsheetService,
|
||||
private sasService: SasService,
|
||||
private cdr: ChangeDetectorRef
|
||||
) {
|
||||
this.hotRegisterer = new HotTableRegisterer()
|
||||
@ -151,12 +153,14 @@ export class MultiDatasetComponent implements OnInit {
|
||||
const libs: string[] = Object.keys(this.libsAndTables)
|
||||
|
||||
if (this.hotUserDatasets?.columns) {
|
||||
(this.hotUserDatasets.columns as ColumnSettings[])[0].source = libs;
|
||||
;(this.hotUserDatasets.columns as ColumnSettings[])[0].source = libs
|
||||
}
|
||||
}
|
||||
|
||||
onFileChange(event: any) {
|
||||
if (!event?.target?.files[0]) {
|
||||
const files = event?.target?.files || []
|
||||
|
||||
if (files.length < 1) {
|
||||
this.eventService.showAbortModal(
|
||||
null,
|
||||
'No file found.',
|
||||
@ -166,11 +170,35 @@ export class MultiDatasetComponent implements OnInit {
|
||||
return
|
||||
}
|
||||
|
||||
const file = event.target.files[0]
|
||||
const fileTitle = file.name
|
||||
const fileExtension = fileTitle.split('.').pop()
|
||||
let matchedExtension = ''
|
||||
|
||||
if (!['xlsx', 'xlsm', 'xlm'].includes(fileExtension)) {
|
||||
for (let file of files) {
|
||||
const fileExtension = file.name.split('.').pop()
|
||||
|
||||
if (!matchedExtension) {
|
||||
matchedExtension = fileExtension
|
||||
}
|
||||
|
||||
if (matchedExtension !== fileExtension) {
|
||||
this.eventService.showInfoModal(
|
||||
'Mixed extensions error',
|
||||
'Please select files with same extension.'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
matchedExtension = fileExtension
|
||||
}
|
||||
|
||||
if (['xlsx', 'xlsm', 'xlm'].includes(matchedExtension)) {
|
||||
// For EXCEL if multiple files, we only take one (the first one)
|
||||
this.selectedFile = event.target.files[0]
|
||||
|
||||
this.initUserInputHot()
|
||||
this.onAutoDetectColumns()
|
||||
} else if (matchedExtension === 'csv') {
|
||||
this.onMultiCsvFiles(event.target.files)
|
||||
} else {
|
||||
this.eventService.showAbortModal(
|
||||
null,
|
||||
'Only excel extensions are allowed. (xlsx)',
|
||||
@ -180,11 +208,37 @@ export class MultiDatasetComponent implements OnInit {
|
||||
return
|
||||
}
|
||||
|
||||
this.selectedFile = event.target.files[0]
|
||||
event.target.value = '' // Reset the upload input
|
||||
}
|
||||
|
||||
this.initUserInputHot()
|
||||
this.onAutoDetectColumns()
|
||||
async onMultiCsvFiles(files: File[]) {
|
||||
for (let file of files) {
|
||||
const fileNameNoExtension = this.parseDatasetFromCsvName(file.name)
|
||||
|
||||
if (
|
||||
this.isValidDatasetFormat(fileNameNoExtension) &&
|
||||
this.isValidDatasetReference(fileNameNoExtension)
|
||||
) {
|
||||
this.csvFiles.push({
|
||||
file: file,
|
||||
fileName: file.name
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.csvFiles.length) {
|
||||
this.eventService.showInfoModal(
|
||||
'CSV Upload',
|
||||
'None of the attached CSV file names matched an actual dataset.'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
this.csvSubmitting = true
|
||||
|
||||
await this.submitCsvFiles()
|
||||
|
||||
this.csvSubmitting = false
|
||||
}
|
||||
|
||||
onDiscardFile() {
|
||||
@ -276,7 +330,10 @@ export class MultiDatasetComponent implements OnInit {
|
||||
if (this.tablesToSubmit.length) {
|
||||
this.showSubmitReasonModal = true
|
||||
} else {
|
||||
this.eventService.showInfoModal('No tables to submit', 'Please include at least one table to proceed.')
|
||||
this.eventService.showInfoModal(
|
||||
'No tables to submit',
|
||||
'Please include at least one table to proceed.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@ -307,43 +364,59 @@ export class MultiDatasetComponent implements OnInit {
|
||||
|
||||
initUserInputHot() {
|
||||
setTimeout(() => {
|
||||
this.hotInstanceUserDataset = this.hotRegisterer.getInstance('hotInstanceUserDataset')
|
||||
this.hotInstanceUserDataset = this.hotRegisterer.getInstance(
|
||||
'hotInstanceUserDataset'
|
||||
)
|
||||
|
||||
this.hotInstanceUserDataset.addHook('beforeChange', (changes: (CellChange | null)[], source: ChangeSource) => {
|
||||
if (changes) {
|
||||
for (let change of changes) {
|
||||
if (change && change[3]) {
|
||||
change[3] = change[3].toUpperCase()
|
||||
this.hotInstanceUserDataset.addHook(
|
||||
'beforeChange',
|
||||
(changes: (CellChange | null)[], source: ChangeSource) => {
|
||||
if (changes) {
|
||||
for (let change of changes) {
|
||||
if (change && change[3]) {
|
||||
change[3] = change[3].toUpperCase()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
this.hotInstanceUserDataset.addHook('afterChange', async (changes: CellChange[] | null, source: ChangeSource) => {
|
||||
if (changes) {
|
||||
if (source === 'edit') {
|
||||
await this.onUserInputDatasetsChange()
|
||||
this.hotInstanceUserDataset.addHook(
|
||||
'afterChange',
|
||||
async (changes: CellChange[] | null, source: ChangeSource) => {
|
||||
if (changes) {
|
||||
if (source === 'edit') {
|
||||
await this.onUserInputDatasetsChange()
|
||||
}
|
||||
|
||||
for (let change of changes) {
|
||||
const row = change[0] as number
|
||||
|
||||
this.markUnmatchedRows(row)
|
||||
}
|
||||
|
||||
this.dynamicCellValidations()
|
||||
|
||||
this.hotInstanceUserDataset.render()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
for (let change of changes) {
|
||||
const row = change[0] as number
|
||||
this.hotInstanceUserDataset.addHook(
|
||||
'afterRemoveRow',
|
||||
async (
|
||||
index: number,
|
||||
amount: number,
|
||||
physicalRows: number[],
|
||||
source?: Handsontable.ChangeSource | undefined
|
||||
) => {
|
||||
await this.onUserInputDatasetsChange()
|
||||
|
||||
for (let row of physicalRows) {
|
||||
this.markUnmatchedRows(row)
|
||||
}
|
||||
|
||||
this.dynamicCellValidations()
|
||||
|
||||
this.hotInstanceUserDataset.render()
|
||||
}
|
||||
})
|
||||
|
||||
this.hotInstanceUserDataset.addHook('afterRemoveRow', async (index: number, amount: number, physicalRows: number[], source?: Handsontable.ChangeSource | undefined) => {
|
||||
await this.onUserInputDatasetsChange()
|
||||
|
||||
for (let row of physicalRows) {
|
||||
this.markUnmatchedRows(row)
|
||||
}
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@ -367,17 +440,32 @@ export class MultiDatasetComponent implements OnInit {
|
||||
|
||||
if (dataAtRow && dataAtRow[0] && dataAtRow[1]) {
|
||||
if (!this.matchedDatasets.includes(dataset)) {
|
||||
cellMetaAtRow.forEach(cellMeta => {
|
||||
this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', 'not-matched')
|
||||
cellMetaAtRow.forEach((cellMeta) => {
|
||||
this.hotInstanceUserDataset.setCellMeta(
|
||||
row,
|
||||
cellMeta.col,
|
||||
'className',
|
||||
'not-matched'
|
||||
)
|
||||
})
|
||||
} else {
|
||||
cellMetaAtRow.forEach(cellMeta => {
|
||||
this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', '')
|
||||
cellMetaAtRow.forEach((cellMeta) => {
|
||||
this.hotInstanceUserDataset.setCellMeta(
|
||||
row,
|
||||
cellMeta.col,
|
||||
'className',
|
||||
''
|
||||
)
|
||||
})
|
||||
}
|
||||
} else {
|
||||
cellMetaAtRow.forEach(cellMeta => {
|
||||
this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', '')
|
||||
cellMetaAtRow.forEach((cellMeta) => {
|
||||
this.hotInstanceUserDataset.setCellMeta(
|
||||
row,
|
||||
cellMeta.col,
|
||||
'className',
|
||||
''
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -444,11 +532,8 @@ export class MultiDatasetComponent implements OnInit {
|
||||
// Set matched datasets to textarea, dataset per row
|
||||
this.userInputDatasets = this.matchedDatasets.join('\n')
|
||||
|
||||
const hotReadyData = this.matchedDatasets.map(matchedDs => {
|
||||
return [
|
||||
matchedDs.split('.')[0],
|
||||
matchedDs.split('.')[1]
|
||||
]
|
||||
const hotReadyData = this.matchedDatasets.map((matchedDs) => {
|
||||
return [matchedDs.split('.')[0], matchedDs.split('.')[1]]
|
||||
})
|
||||
|
||||
// Add empty rows to fill initial number of rows if data has less rows
|
||||
@ -493,10 +578,12 @@ export class MultiDatasetComponent implements OnInit {
|
||||
public get notFoundDatasets(): string[] {
|
||||
const userDatasets = this.getDatasetsFromHot()
|
||||
|
||||
return userDatasets.filter(userDs => !this.matchedDatasets.includes(userDs.trim())).filter(userDs => userDs.length)
|
||||
return userDatasets
|
||||
.filter((userDs) => !this.matchedDatasets.includes(userDs.trim()))
|
||||
.filter((userDs) => userDs.length)
|
||||
}
|
||||
|
||||
public get isHotHidden() :boolean {
|
||||
public get isHotHidden(): boolean {
|
||||
if (!this.hotInstance) return true
|
||||
|
||||
try {
|
||||
@ -509,16 +596,60 @@ export class MultiDatasetComponent implements OnInit {
|
||||
}
|
||||
|
||||
public get tablesToSubmit(): ParsedDataset[] {
|
||||
return this.parsedDatasets.filter(dataset => dataset.datasource && dataset.parseResult && dataset.includeInSubmission)
|
||||
return this.parsedDatasets.filter(
|
||||
(dataset) =>
|
||||
dataset.datasource && dataset.parseResult && dataset.includeInSubmission
|
||||
)
|
||||
}
|
||||
|
||||
public downloadFile(
|
||||
response: any
|
||||
) {
|
||||
public get submittingCsv(): boolean {
|
||||
return this.csvFiles.length > 0
|
||||
}
|
||||
|
||||
public downloadFile(response: any) {
|
||||
const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log`
|
||||
this.helperService.downloadTextFile(filename, JSON.stringify(response))
|
||||
}
|
||||
|
||||
/**
|
||||
* Submits attached CSVs which are matched with existing datasets
|
||||
*/
|
||||
async submitCsvFiles() {
|
||||
let requestsResults: SubmittedDatasetResult[] = []
|
||||
|
||||
for (let file of this.csvFiles) {
|
||||
const libds = this.parseDatasetFromCsvName(file.fileName)
|
||||
|
||||
let error
|
||||
let success
|
||||
|
||||
await this.sasService
|
||||
.uploadFile('services/editors/loadfile', [file], { table: libds })
|
||||
.then(
|
||||
(res: any) => {
|
||||
if (typeof res.sasjsAbort !== 'undefined') {
|
||||
error = res.sasjsAbort
|
||||
} else {
|
||||
success = res
|
||||
}
|
||||
},
|
||||
(err: any) => {
|
||||
console.error('err', err)
|
||||
|
||||
error = err
|
||||
}
|
||||
)
|
||||
|
||||
requestsResults.push({
|
||||
success,
|
||||
error,
|
||||
libds: libds
|
||||
})
|
||||
}
|
||||
|
||||
this.submittedDatasets = requestsResults
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the table for given datasets params LIBRARY.TABLE
|
||||
*/
|
||||
@ -557,7 +688,9 @@ export class MultiDatasetComponent implements OnInit {
|
||||
|
||||
this.submitLoading = true
|
||||
|
||||
let requestsResults: SubmittedDatasetResult[] = explicitDatasets ? this.submittedDatasets : []
|
||||
let requestsResults: SubmittedDatasetResult[] = explicitDatasets
|
||||
? this.submittedDatasets
|
||||
: []
|
||||
|
||||
for (let table of this.parsedDatasets) {
|
||||
// Skip the table if no data inside
|
||||
@ -629,7 +762,9 @@ export class MultiDatasetComponent implements OnInit {
|
||||
// If explicit datasets are set don't just push to th array
|
||||
// instead replace if result already exist from before (this might be re-submit)
|
||||
if (explicitDatasets) {
|
||||
const existingResultIndex = requestsResults.findIndex(result => result.libds === table.libds)
|
||||
const existingResultIndex = requestsResults.findIndex(
|
||||
(result) => result.libds === table.libds
|
||||
)
|
||||
|
||||
if (existingResultIndex > -1) {
|
||||
requestsResults[existingResultIndex] = requestResult
|
||||
@ -653,10 +788,18 @@ export class MultiDatasetComponent implements OnInit {
|
||||
await this.submitTables([activeSubmittedDataset.libds])
|
||||
|
||||
// Activate new resubmitted table
|
||||
const newSubmittedDataset = this.submittedDatasets.find(sd => sd.libds === activeSubmittedDataset.libds)
|
||||
const newSubmittedDataset = this.submittedDatasets.find(
|
||||
(sd) => sd.libds === activeSubmittedDataset.libds
|
||||
)
|
||||
if (newSubmittedDataset) newSubmittedDataset.active = true
|
||||
}
|
||||
|
||||
private parseDatasetFromCsvName(fileName: string) {
|
||||
const fileNameArr = fileName.split('.')
|
||||
fileNameArr.pop()
|
||||
return fileNameArr.join('.')
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns list of strings containing datasets in the HOT user input
|
||||
@ -667,7 +810,9 @@ export class MultiDatasetComponent implements OnInit {
|
||||
|
||||
const hotData = this.hotInstanceUserDataset.getData()
|
||||
|
||||
return hotData.filter(row => row[0]?.length && row[1]?.length).map(row => row ? `${row[0]}.${row[1]}` : '')
|
||||
return hotData
|
||||
.filter((row) => row[0]?.length && row[1]?.length)
|
||||
.map((row) => (row ? `${row[0]}.${row[1]}` : ''))
|
||||
}
|
||||
|
||||
private parseExcelSheetNames(): Promise<string[]> {
|
||||
@ -718,9 +863,9 @@ export class MultiDatasetComponent implements OnInit {
|
||||
*
|
||||
* example: LIB123.TABLE_123
|
||||
*/
|
||||
private isValidDatasetFormat(sheetName: string) {
|
||||
private isValidDatasetFormat(name: string) {
|
||||
const regex = /^\w{1,8}\.\w{1,32}$/gim
|
||||
const correctFormat = regex.test(sheetName)
|
||||
const correctFormat = regex.test(name)
|
||||
|
||||
return correctFormat
|
||||
}
|
||||
@ -867,6 +1012,6 @@ export interface SubmittedDatasetResult {
|
||||
libds: string
|
||||
success: EditorsStageDataSASResponse | undefined
|
||||
error: any
|
||||
parseResult: ParseResult
|
||||
parseResult?: ParseResult
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user