dc/dc
dc/dc
3
1
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 35 Wiki Activity

feat(multi load): multiple csv files

Browse Source
This commit is contained in:
Mihajlo Medjedovic 2024-06-24 15:36:06 +02:00
parent cffeab813d
commit 4d276657b3
2 changed files with 250 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
client/src/app/multi-dataset/multi-dataset.component.html
View File

@ -7,7 +7,7 @@
<button
(click)="fileUploadInput.click()"
class="btn btn-primary btn-sm"
[disabled]="selectedFile !== null"
[disabled]="selectedFile !== null || submittingCsv"
>
Browse file
</button>
@ -17,7 +17,7 @@
id="file-upload"
type="file"
(change)="onFileChange($event)"
appFileSelect
multiple
/>
</div>
@ -91,7 +91,7 @@
</div>
<div
*ngIf="selectedFile === null"
*ngIf="selectedFile === null && !submittingCsv"
class="no-table-selected pointer-events-none"
>
<clr-icon
@ -104,8 +104,8 @@
</p>
</div>
<ng-container *ngIf="selectedFile !== null">
<ng-container *ngIf="!parsedDatasets.length">
<ng-container *ngIf="selectedFile !== null || submittingCsv">
<ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
<div class="d-flex clr-justify-content-center mt-15">
<div class="dataset-input-wrapper">
<p cds-text="secondary regular" class="mb-20">
@ -148,9 +148,9 @@
</div>
</ng-container>
<ng-container *ngIf="parsedDatasets.length">
<ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
<div
*ngIf="!activeParsedDataset && !activeSubmittedDataset"
*ngIf="!activeParsedDataset"
class="no-table-selected pointer-events-none"
>
<clr-icon
@ -159,12 +159,7 @@
class="is-info icon-dc-fill"
></clr-icon>
<p class="text-center color-gray mt-10" cds-text="section">
Please select a dataset on the left to
{{
!submittedDatasets.length
? 'review data'
: 'review submitted results'
}}
Please select a dataset on the left to review the data
</p>
</div>
@ -238,10 +233,26 @@
</ng-container>
</ng-container>
<ng-container *ngIf="submittedDatasets.length">
<div
*ngIf="!activeSubmittedDataset"
class="no-table-selected pointer-events-none"
>
<clr-icon
shape="warning-standard"
size="40"
class="is-info icon-dc-fill"
></clr-icon>
<p class="text-center color-gray mt-10" cds-text="section">
Please select a dataset on the left to review the submit results
</p>
</div>
</ng-container>
<ng-container *ngIf="activeSubmittedDataset">
<div class="d-flex clr-justify-content-between p-10">
<div>
<p cds-text="secondary regular" class="mb-10">
<p *ngIf="activeSubmittedDataset.parseResult" cds-text="secondary regular" class="mb-10">
Found in range:
<strong
>"{{
@ -274,6 +285,7 @@
<div>
<button
*ngIf="!submittingCsv && activeSubmittedDataset.error"
(click)="reSubmitTable(activeSubmittedDataset)"
class="btn btn-primary mt-10"
[clrLoading]="submitLoading"
@ -360,3 +372,18 @@
</button>
</div>
</clr-modal>
<clr-modal [(clrModalOpen)]="csvSubmitting" [clrModalClosable]="false">
<h3 class="modal-title">
Submitting {{ csvFiles.length }} CSV {{ csvFiles.length === 1 ? 'file' : 'files'}}
</h3>
<div class="modal-body">
<div class="text-center">
<clr-spinner clrMedium></clr-spinner>
</div>
<p cds-text="caption_clean" class="mt-10 text-center">
This will take few moments
</p>
</div>
</clr-modal>

273
client/src/app/multi-dataset/multi-dataset.component.ts
View File

@ -28,6 +28,7 @@ import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.mod
import { CellChange, ChangeSource } from 'handsontable/common'
import { baseAfterGetColHeader } from '../shared/utils/hot.utils'
import { ColumnSettings } from 'handsontable/settings'
import { UploadFile } from '@sasjs/adapter'
@Component({
selector: 'app-multi-dataset',
@ -44,6 +45,9 @@ export class MultiDatasetComponent implements OnInit {
public hotTableMaxRows =
this.licenceState.value.viewer_rows_allowed || Infinity
public csvFiles: UploadFile[] = []
public csvSubmitting: boolean = false
public selectedFile: File | null = null
public parsedDatasets: ParsedDataset[] = []
public submittedDatasets: SubmittedDatasetResult[] = []
@ -67,10 +71,7 @@ export class MultiDatasetComponent implements OnInit {
public submitReasonMessage: string = ''
public hotUserDatasets: Handsontable.GridSettings = {
colHeaders: [
'Library',
'Table'
],
colHeaders: ['Library', 'Table'],
data: [
['', ''],
['', ''],
@ -122,6 +123,7 @@ export class MultiDatasetComponent implements OnInit {
private helperService: HelperService,
private sasStoreService: SasStoreService,
private spreadsheetService: SpreadsheetService,
private sasService: SasService,
private cdr: ChangeDetectorRef
) {
this.hotRegisterer = new HotTableRegisterer()
@ -151,12 +153,14 @@ export class MultiDatasetComponent implements OnInit {
const libs: string[] = Object.keys(this.libsAndTables)
if (this.hotUserDatasets?.columns) {
(this.hotUserDatasets.columns as ColumnSettings[])[0].source = libs;
;(this.hotUserDatasets.columns as ColumnSettings[])[0].source = libs
}
}
onFileChange(event: any) {
if (!event?.target?.files[0]) {
const files = event?.target?.files || []
if (files.length < 1) {
this.eventService.showAbortModal(
null,
'No file found.',
@ -166,11 +170,35 @@ export class MultiDatasetComponent implements OnInit {
return
}
const file = event.target.files[0]
const fileTitle = file.name
const fileExtension = fileTitle.split('.').pop()
let matchedExtension = ''
if (!['xlsx', 'xlsm', 'xlm'].includes(fileExtension)) {
for (let file of files) {
const fileExtension = file.name.split('.').pop()
if (!matchedExtension) {
matchedExtension = fileExtension
}
if (matchedExtension !== fileExtension) {
this.eventService.showInfoModal(
'Mixed extensions error',
'Please select files with same extension.'
)
return
}
matchedExtension = fileExtension
}
if (['xlsx', 'xlsm', 'xlm'].includes(matchedExtension)) {
// For EXCEL if multiple files, we only take one (the first one)
this.selectedFile = event.target.files[0]
this.initUserInputHot()
this.onAutoDetectColumns()
} else if (matchedExtension === 'csv') {
this.onMultiCsvFiles(event.target.files)
} else {
this.eventService.showAbortModal(
null,
'Only excel extensions are allowed. (xlsx)',
@ -180,11 +208,37 @@ export class MultiDatasetComponent implements OnInit {
return
}
this.selectedFile = event.target.files[0]
event.target.value = '' // Reset the upload input
}
this.initUserInputHot()
this.onAutoDetectColumns()
async onMultiCsvFiles(files: File[]) {
for (let file of files) {
const fileNameNoExtension = this.parseDatasetFromCsvName(file.name)
if (
this.isValidDatasetFormat(fileNameNoExtension) &&
this.isValidDatasetReference(fileNameNoExtension)
) {
this.csvFiles.push({
file: file,
fileName: file.name
})
}
}
if (!this.csvFiles.length) {
this.eventService.showInfoModal(
'CSV Upload',
'None of the attached CSV file names matched an actual dataset.'
)
return
}
this.csvSubmitting = true
await this.submitCsvFiles()
this.csvSubmitting = false
}
onDiscardFile() {
@ -276,7 +330,10 @@ export class MultiDatasetComponent implements OnInit {
if (this.tablesToSubmit.length) {
this.showSubmitReasonModal = true
} else {
this.eventService.showInfoModal('No tables to submit', 'Please include at least one table to proceed.')
this.eventService.showInfoModal(
'No tables to submit',
'Please include at least one table to proceed.'
)
}
}
@ -307,43 +364,59 @@ export class MultiDatasetComponent implements OnInit {
initUserInputHot() {
setTimeout(() => {
this.hotInstanceUserDataset = this.hotRegisterer.getInstance('hotInstanceUserDataset')
this.hotInstanceUserDataset = this.hotRegisterer.getInstance(
'hotInstanceUserDataset'
)
this.hotInstanceUserDataset.addHook('beforeChange', (changes: (CellChange | null)[], source: ChangeSource) => {
if (changes) {
for (let change of changes) {
if (change && change[3]) {
change[3] = change[3].toUpperCase()
this.hotInstanceUserDataset.addHook(
'beforeChange',
(changes: (CellChange | null)[], source: ChangeSource) => {
if (changes) {
for (let change of changes) {
if (change && change[3]) {
change[3] = change[3].toUpperCase()
}
}
}
}
})
)
this.hotInstanceUserDataset.addHook('afterChange', async (changes: CellChange[] | null, source: ChangeSource) => {
if (changes) {
if (source === 'edit') {
await this.onUserInputDatasetsChange()
this.hotInstanceUserDataset.addHook(
'afterChange',
async (changes: CellChange[] | null, source: ChangeSource) => {
if (changes) {
if (source === 'edit') {
await this.onUserInputDatasetsChange()
}
for (let change of changes) {
const row = change[0] as number
this.markUnmatchedRows(row)
}
this.dynamicCellValidations()
this.hotInstanceUserDataset.render()
}
}
)
for (let change of changes) {
const row = change[0] as number
this.hotInstanceUserDataset.addHook(
'afterRemoveRow',
async (
index: number,
amount: number,
physicalRows: number[],
source?: Handsontable.ChangeSource | undefined
) => {
await this.onUserInputDatasetsChange()
for (let row of physicalRows) {
this.markUnmatchedRows(row)
}
this.dynamicCellValidations()
this.hotInstanceUserDataset.render()
}
})
this.hotInstanceUserDataset.addHook('afterRemoveRow', async (index: number, amount: number, physicalRows: number[], source?: Handsontable.ChangeSource | undefined) => {
await this.onUserInputDatasetsChange()
for (let row of physicalRows) {
this.markUnmatchedRows(row)
}
})
)
})
}
@ -367,17 +440,32 @@ export class MultiDatasetComponent implements OnInit {
if (dataAtRow && dataAtRow[0] && dataAtRow[1]) {
if (!this.matchedDatasets.includes(dataset)) {
cellMetaAtRow.forEach(cellMeta => {
this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', 'not-matched')
cellMetaAtRow.forEach((cellMeta) => {
this.hotInstanceUserDataset.setCellMeta(
row,
cellMeta.col,
'className',
'not-matched'
)
})
} else {
cellMetaAtRow.forEach(cellMeta => {
this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', '')
cellMetaAtRow.forEach((cellMeta) => {
this.hotInstanceUserDataset.setCellMeta(
row,
cellMeta.col,
'className',
''
)
})
}
} else {
cellMetaAtRow.forEach(cellMeta => {
this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', '')
cellMetaAtRow.forEach((cellMeta) => {
this.hotInstanceUserDataset.setCellMeta(
row,
cellMeta.col,
'className',
''
)
})
}
}
@ -444,11 +532,8 @@ export class MultiDatasetComponent implements OnInit {
// Set matched datasets to textarea, dataset per row
this.userInputDatasets = this.matchedDatasets.join('\n')
const hotReadyData = this.matchedDatasets.map(matchedDs => {
return [
matchedDs.split('.')[0],
matchedDs.split('.')[1]
]
const hotReadyData = this.matchedDatasets.map((matchedDs) => {
return [matchedDs.split('.')[0], matchedDs.split('.')[1]]
})
// Add empty rows to fill initial number of rows if data has less rows
@ -493,10 +578,12 @@ export class MultiDatasetComponent implements OnInit {
public get notFoundDatasets(): string[] {
const userDatasets = this.getDatasetsFromHot()
return userDatasets.filter(userDs => !this.matchedDatasets.includes(userDs.trim())).filter(userDs => userDs.length)
return userDatasets
.filter((userDs) => !this.matchedDatasets.includes(userDs.trim()))
.filter((userDs) => userDs.length)
}
public get isHotHidden() :boolean {
public get isHotHidden(): boolean {
if (!this.hotInstance) return true
try {
@ -509,16 +596,60 @@ export class MultiDatasetComponent implements OnInit {
}
public get tablesToSubmit(): ParsedDataset[] {
return this.parsedDatasets.filter(dataset => dataset.datasource && dataset.parseResult && dataset.includeInSubmission)
return this.parsedDatasets.filter(
(dataset) =>
dataset.datasource && dataset.parseResult && dataset.includeInSubmission
)
}
public downloadFile(
response: any
) {
public get submittingCsv(): boolean {
return this.csvFiles.length > 0
}
public downloadFile(response: any) {
const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log`
this.helperService.downloadTextFile(filename, JSON.stringify(response))
}
/**
* Submits attached CSVs which are matched with existing datasets
*/
async submitCsvFiles() {
let requestsResults: SubmittedDatasetResult[] = []
for (let file of this.csvFiles) {
const libds = this.parseDatasetFromCsvName(file.fileName)
let error
let success
await this.sasService
.uploadFile('services/editors/loadfile', [file], { table: libds })
.then(
(res: any) => {
if (typeof res.sasjsAbort !== 'undefined') {
error = res.sasjsAbort
} else {
success = res
}
},
(err: any) => {
console.error('err', err)
error = err
}
)
requestsResults.push({
success,
error,
libds: libds
})
}
this.submittedDatasets = requestsResults
}
/**
* Fetches the table for given datasets params LIBRARY.TABLE
*/
@ -557,7 +688,9 @@ export class MultiDatasetComponent implements OnInit {
this.submitLoading = true
let requestsResults: SubmittedDatasetResult[] = explicitDatasets ? this.submittedDatasets : []
let requestsResults: SubmittedDatasetResult[] = explicitDatasets
? this.submittedDatasets
: []
for (let table of this.parsedDatasets) {
// Skip the table if no data inside
@ -629,7 +762,9 @@ export class MultiDatasetComponent implements OnInit {
// If explicit datasets are set don't just push to th array
// instead replace if result already exist from before (this might be re-submit)
if (explicitDatasets) {
const existingResultIndex = requestsResults.findIndex(result => result.libds === table.libds)
const existingResultIndex = requestsResults.findIndex(
(result) => result.libds === table.libds
)
if (existingResultIndex > -1) {
requestsResults[existingResultIndex] = requestResult
@ -653,10 +788,18 @@ export class MultiDatasetComponent implements OnInit {
await this.submitTables([activeSubmittedDataset.libds])
// Activate new resubmitted table
const newSubmittedDataset = this.submittedDatasets.find(sd => sd.libds === activeSubmittedDataset.libds)
const newSubmittedDataset = this.submittedDatasets.find(
(sd) => sd.libds === activeSubmittedDataset.libds
)
if (newSubmittedDataset) newSubmittedDataset.active = true
}
private parseDatasetFromCsvName(fileName: string) {
const fileNameArr = fileName.split('.')
fileNameArr.pop()
return fileNameArr.join('.')
}
/**
*
* @returns list of strings containing datasets in the HOT user input
@ -667,7 +810,9 @@ export class MultiDatasetComponent implements OnInit {
const hotData = this.hotInstanceUserDataset.getData()
return hotData.filter(row => row[0]?.length && row[1]?.length).map(row => row ? `${row[0]}.${row[1]}` : '')
return hotData
.filter((row) => row[0]?.length && row[1]?.length)
.map((row) => (row ? `${row[0]}.${row[1]}` : ''))
}
private parseExcelSheetNames(): Promise<string[]> {
@ -718,9 +863,9 @@ export class MultiDatasetComponent implements OnInit {
*
* example: LIB123.TABLE_123
*/
private isValidDatasetFormat(sheetName: string) {
private isValidDatasetFormat(name: string) {
const regex = /^\w{1,8}\.\w{1,32}$/gim
const correctFormat = regex.test(sheetName)
const correctFormat = regex.test(name)
return correctFormat
}
@ -867,6 +1012,6 @@ export interface SubmittedDatasetResult {
libds: string
success: EditorsStageDataSASResponse | undefined
error: any
parseResult: ParseResult
parseResult?: ParseResult
active?: boolean
}