feat(multi load): multiple csv files

client/src/app/multi-dataset/multi-dataset.component.html

@@ -7,7 +7,7 @@
     <button
       (click)="fileUploadInput.click()"
       class="btn btn-primary btn-sm"
-      [disabled]="selectedFile !== null"
+      [disabled]="selectedFile !== null || submittingCsv"
     >
       Browse file
     </button>
@@ -17,7 +17,7 @@
       id="file-upload"
       type="file"
       (change)="onFileChange($event)"
-      appFileSelect
+      multiple
     />
   </div>
 
@@ -91,7 +91,7 @@
     </div>
 
     <div
-      *ngIf="selectedFile === null"
+      *ngIf="selectedFile === null && !submittingCsv"
       class="no-table-selected pointer-events-none"
     >
       <clr-icon
@@ -104,8 +104,8 @@
       </p>
     </div>
 
-    <ng-container *ngIf="selectedFile !== null">
+    <ng-container *ngIf="selectedFile !== null || submittingCsv">
-      <ng-container *ngIf="!parsedDatasets.length">
+      <ng-container *ngIf="!parsedDatasets.length && selectedFile !== null">
         <div class="d-flex clr-justify-content-center mt-15">
           <div class="dataset-input-wrapper">
             <p cds-text="secondary regular" class="mb-20">
@@ -148,9 +148,9 @@
         </div>
       </ng-container>
 
-      <ng-container *ngIf="parsedDatasets.length">
+      <ng-container *ngIf="parsedDatasets.length && !submittedDatasets.length">
         <div
-          *ngIf="!activeParsedDataset && !activeSubmittedDataset"
+          *ngIf="!activeParsedDataset"
           class="no-table-selected pointer-events-none"
         >
           <clr-icon
@@ -159,12 +159,7 @@
             class="is-info icon-dc-fill"
           ></clr-icon>
           <p class="text-center color-gray mt-10" cds-text="section">
-            Please select a dataset on the left to
+            Please select a dataset on the left to review the data
-            {{
-              !submittedDatasets.length
-                ? 'review data'
-                : 'review submitted results'
-            }}
           </p>
         </div>
 
@@ -238,10 +233,26 @@
         </ng-container>
       </ng-container>
 
+      <ng-container *ngIf="submittedDatasets.length">
+        <div
+          *ngIf="!activeSubmittedDataset"
+          class="no-table-selected pointer-events-none"
+        >
+        <clr-icon
+          shape="warning-standard"
+          size="40"
+          class="is-info icon-dc-fill"
+        ></clr-icon>
+        <p class="text-center color-gray mt-10" cds-text="section">
+          Please select a dataset on the left to review the submit results
+        </p>
+      </div>
+      </ng-container>
+
       <ng-container *ngIf="activeSubmittedDataset">
         <div class="d-flex clr-justify-content-between p-10">
           <div>
-            <p cds-text="secondary regular" class="mb-10">
+            <p *ngIf="activeSubmittedDataset.parseResult" cds-text="secondary regular" class="mb-10">
               Found in range:
               <strong
                 >"{{
@@ -274,6 +285,7 @@
 
           <div>
             <button
+            *ngIf="!submittingCsv && activeSubmittedDataset.error"
             (click)="reSubmitTable(activeSubmittedDataset)"
             class="btn btn-primary mt-10"
             [clrLoading]="submitLoading"
@@ -360,3 +372,18 @@
     </button>
   </div>
 </clr-modal>
+
+<clr-modal [(clrModalOpen)]="csvSubmitting" [clrModalClosable]="false">
+  <h3 class="modal-title">
+    Submitting {{ csvFiles.length }} CSV {{ csvFiles.length === 1 ? 'file' : 'files'}}
+  </h3>
+  <div class="modal-body">
+    <div class="text-center">
+      <clr-spinner clrMedium></clr-spinner>
+    </div>
+
+    <p cds-text="caption_clean" class="mt-10 text-center">
+      This will take few moments
+    </p>
+  </div>
+</clr-modal>

client/src/app/multi-dataset/multi-dataset.component.ts

@@ -28,6 +28,7 @@ import { EditorsStageDataSASResponse } from '../models/sas/editors-stagedata.mod
 import { CellChange, ChangeSource } from 'handsontable/common'
 import { baseAfterGetColHeader } from '../shared/utils/hot.utils'
 import { ColumnSettings } from 'handsontable/settings'
+import { UploadFile } from '@sasjs/adapter'
 
 @Component({
   selector: 'app-multi-dataset',
@@ -44,6 +45,9 @@ export class MultiDatasetComponent implements OnInit {
   public hotTableMaxRows =
     this.licenceState.value.viewer_rows_allowed || Infinity
 
+  public csvFiles: UploadFile[] = []
+  public csvSubmitting: boolean = false
+
   public selectedFile: File | null = null
   public parsedDatasets: ParsedDataset[] = []
   public submittedDatasets: SubmittedDatasetResult[] = []
@@ -67,10 +71,7 @@ export class MultiDatasetComponent implements OnInit {
   public submitReasonMessage: string = ''
 
   public hotUserDatasets: Handsontable.GridSettings = {
-    colHeaders: [
+    colHeaders: ['Library', 'Table'],
-      'Library',
-      'Table'
-    ],
     data: [
       ['', ''],
       ['', ''],
@@ -122,6 +123,7 @@ export class MultiDatasetComponent implements OnInit {
     private helperService: HelperService,
     private sasStoreService: SasStoreService,
     private spreadsheetService: SpreadsheetService,
+    private sasService: SasService,
     private cdr: ChangeDetectorRef
   ) {
     this.hotRegisterer = new HotTableRegisterer()
@@ -151,12 +153,14 @@ export class MultiDatasetComponent implements OnInit {
     const libs: string[] = Object.keys(this.libsAndTables)
 
     if (this.hotUserDatasets?.columns) {
-      (this.hotUserDatasets.columns as ColumnSettings[])[0].source = libs;
+      ;(this.hotUserDatasets.columns as ColumnSettings[])[0].source = libs
     }
   }
 
   onFileChange(event: any) {
-    if (!event?.target?.files[0]) {
+    const files = event?.target?.files || []
+
+    if (files.length < 1) {
       this.eventService.showAbortModal(
         null,
         'No file found.',
@@ -166,11 +170,35 @@ export class MultiDatasetComponent implements OnInit {
       return
     }
 
-    const file = event.target.files[0]
+    let matchedExtension = ''
-    const fileTitle = file.name
-    const fileExtension = fileTitle.split('.').pop()
 
-    if (!['xlsx', 'xlsm', 'xlm'].includes(fileExtension)) {
+    for (let file of files) {
+      const fileExtension = file.name.split('.').pop()
+
+      if (!matchedExtension) {
+        matchedExtension = fileExtension
+      }
+
+      if (matchedExtension !== fileExtension) {
+        this.eventService.showInfoModal(
+          'Mixed extensions error',
+          'Please select files with same extension.'
+        )
+        return
+      }
+
+      matchedExtension = fileExtension
+    }
+
+    if (['xlsx', 'xlsm', 'xlm'].includes(matchedExtension)) {
+      // For EXCEL if multiple files, we only take one (the first one)
+      this.selectedFile = event.target.files[0]
+
+      this.initUserInputHot()
+      this.onAutoDetectColumns()
+    } else if (matchedExtension === 'csv') {
+      this.onMultiCsvFiles(event.target.files)
+    } else {
       this.eventService.showAbortModal(
         null,
         'Only excel extensions are allowed. (xlsx)',
@@ -180,11 +208,37 @@ export class MultiDatasetComponent implements OnInit {
       return
     }
 
-    this.selectedFile = event.target.files[0]
     event.target.value = '' // Reset the upload input
+  }
 
-    this.initUserInputHot()
+  async onMultiCsvFiles(files: File[]) {
-    this.onAutoDetectColumns()
+    for (let file of files) {
+      const fileNameNoExtension = this.parseDatasetFromCsvName(file.name)
+
+      if (
+        this.isValidDatasetFormat(fileNameNoExtension) &&
+        this.isValidDatasetReference(fileNameNoExtension)
+      ) {
+        this.csvFiles.push({
+          file: file,
+          fileName: file.name
+        })
+      }
+    }
+
+    if (!this.csvFiles.length) {
+      this.eventService.showInfoModal(
+        'CSV Upload',
+        'None of the attached CSV file names matched an actual dataset.'
+      )
+      return
+    }
+
+    this.csvSubmitting = true
+
+    await this.submitCsvFiles()
+
+    this.csvSubmitting = false
   }
 
   onDiscardFile() {
@@ -276,7 +330,10 @@ export class MultiDatasetComponent implements OnInit {
     if (this.tablesToSubmit.length) {
       this.showSubmitReasonModal = true
     } else {
-      this.eventService.showInfoModal('No tables to submit', 'Please include at least one table to proceed.')
+      this.eventService.showInfoModal(
+        'No tables to submit',
+        'Please include at least one table to proceed.'
+      )
     }
   }
 
@@ -307,9 +364,13 @@ export class MultiDatasetComponent implements OnInit {
 
   initUserInputHot() {
     setTimeout(() => {
-      this.hotInstanceUserDataset = this.hotRegisterer.getInstance('hotInstanceUserDataset')
+      this.hotInstanceUserDataset = this.hotRegisterer.getInstance(
+        'hotInstanceUserDataset'
+      )
 
-      this.hotInstanceUserDataset.addHook('beforeChange', (changes: (CellChange | null)[], source: ChangeSource) => {
+      this.hotInstanceUserDataset.addHook(
+        'beforeChange',
+        (changes: (CellChange | null)[], source: ChangeSource) => {
           if (changes) {
             for (let change of changes) {
               if (change && change[3]) {
@@ -317,9 +378,12 @@ export class MultiDatasetComponent implements OnInit {
               }
             }
           }
-      })
+        }
+      )
 
-      this.hotInstanceUserDataset.addHook('afterChange', async (changes: CellChange[] | null, source: ChangeSource) => {
+      this.hotInstanceUserDataset.addHook(
+        'afterChange',
+        async (changes: CellChange[] | null, source: ChangeSource) => {
           if (changes) {
             if (source === 'edit') {
               await this.onUserInputDatasetsChange()
@@ -335,15 +399,24 @@ export class MultiDatasetComponent implements OnInit {
 
             this.hotInstanceUserDataset.render()
           }
-      })
+        }
+      )
 
-      this.hotInstanceUserDataset.addHook('afterRemoveRow', async (index: number, amount: number, physicalRows: number[], source?: Handsontable.ChangeSource | undefined) => {
+      this.hotInstanceUserDataset.addHook(
+        'afterRemoveRow',
+        async (
+          index: number,
+          amount: number,
+          physicalRows: number[],
+          source?: Handsontable.ChangeSource | undefined
+        ) => {
           await this.onUserInputDatasetsChange()
 
           for (let row of physicalRows) {
             this.markUnmatchedRows(row)
           }
-      })
+        }
+      )
     })
   }
 
@@ -367,17 +440,32 @@ export class MultiDatasetComponent implements OnInit {
 
     if (dataAtRow && dataAtRow[0] && dataAtRow[1]) {
       if (!this.matchedDatasets.includes(dataset)) {
-        cellMetaAtRow.forEach(cellMeta => {
+        cellMetaAtRow.forEach((cellMeta) => {
-          this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', 'not-matched')
+          this.hotInstanceUserDataset.setCellMeta(
+            row,
+            cellMeta.col,
+            'className',
+            'not-matched'
+          )
         })
       } else {
-        cellMetaAtRow.forEach(cellMeta => {
+        cellMetaAtRow.forEach((cellMeta) => {
-          this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', '')
+          this.hotInstanceUserDataset.setCellMeta(
+            row,
+            cellMeta.col,
+            'className',
+            ''
+          )
         })
       }
     } else {
-      cellMetaAtRow.forEach(cellMeta => {
+      cellMetaAtRow.forEach((cellMeta) => {
-        this.hotInstanceUserDataset.setCellMeta(row, cellMeta.col, 'className', '')
+        this.hotInstanceUserDataset.setCellMeta(
+          row,
+          cellMeta.col,
+          'className',
+          ''
+        )
       })
     }
   }
@@ -444,11 +532,8 @@ export class MultiDatasetComponent implements OnInit {
     // Set matched datasets to textarea, dataset per row
     this.userInputDatasets = this.matchedDatasets.join('\n')
 
-    const hotReadyData = this.matchedDatasets.map(matchedDs => {
+    const hotReadyData = this.matchedDatasets.map((matchedDs) => {
-      return [
+      return [matchedDs.split('.')[0], matchedDs.split('.')[1]]
-        matchedDs.split('.')[0],
-        matchedDs.split('.')[1]
-      ]
     })
 
     // Add empty rows to fill initial number of rows if data has less rows
@@ -493,7 +578,9 @@ export class MultiDatasetComponent implements OnInit {
   public get notFoundDatasets(): string[] {
     const userDatasets = this.getDatasetsFromHot()
 
-    return userDatasets.filter(userDs => !this.matchedDatasets.includes(userDs.trim())).filter(userDs => userDs.length)
+    return userDatasets
+      .filter((userDs) => !this.matchedDatasets.includes(userDs.trim()))
+      .filter((userDs) => userDs.length)
   }
 
   public get isHotHidden(): boolean {
@@ -509,16 +596,60 @@ export class MultiDatasetComponent implements OnInit {
   }
 
   public get tablesToSubmit(): ParsedDataset[] {
-    return this.parsedDatasets.filter(dataset => dataset.datasource && dataset.parseResult && dataset.includeInSubmission)
+    return this.parsedDatasets.filter(
+      (dataset) =>
+        dataset.datasource && dataset.parseResult && dataset.includeInSubmission
+    )
   }
 
-  public downloadFile(
+  public get submittingCsv(): boolean {
-    response: any
+    return this.csvFiles.length > 0
-  ) {
+  }
+
+  public downloadFile(response: any) {
     const filename = `stagedata-${this.activeSubmittedDataset?.libds}-log`
     this.helperService.downloadTextFile(filename, JSON.stringify(response))
   }
 
+  /**
+   * Submits attached CSVs which are matched with existing datasets
+   */
+  async submitCsvFiles() {
+    let requestsResults: SubmittedDatasetResult[] = []
+
+    for (let file of this.csvFiles) {
+      const libds = this.parseDatasetFromCsvName(file.fileName)
+
+      let error
+      let success
+
+      await this.sasService
+      .uploadFile('services/editors/loadfile', [file], { table: libds })
+      .then(
+        (res: any) => {
+          if (typeof res.sasjsAbort !== 'undefined') {
+            error = res.sasjsAbort
+          } else {
+            success = res
+          }
+        },
+        (err: any) => {
+          console.error('err', err)
+
+          error = err
+        }
+      )
+
+      requestsResults.push({
+        success,
+        error,
+        libds: libds
+      })
+    }
+
+    this.submittedDatasets = requestsResults
+  }
+
   /**
    * Fetches the table for given datasets params LIBRARY.TABLE
    */
@@ -557,7 +688,9 @@ export class MultiDatasetComponent implements OnInit {
 
     this.submitLoading = true
 
-    let requestsResults: SubmittedDatasetResult[] = explicitDatasets ? this.submittedDatasets : []
+    let requestsResults: SubmittedDatasetResult[] = explicitDatasets
+      ? this.submittedDatasets
+      : []
 
     for (let table of this.parsedDatasets) {
       // Skip the table if no data inside
@@ -629,7 +762,9 @@ export class MultiDatasetComponent implements OnInit {
         // If explicit datasets are set don't just push to th array
         // instead replace if result already exist from before (this might be re-submit)
         if (explicitDatasets) {
-          const existingResultIndex = requestsResults.findIndex(result => result.libds === table.libds)
+          const existingResultIndex = requestsResults.findIndex(
+            (result) => result.libds === table.libds
+          )
 
           if (existingResultIndex > -1) {
             requestsResults[existingResultIndex] = requestResult
@@ -653,10 +788,18 @@ export class MultiDatasetComponent implements OnInit {
     await this.submitTables([activeSubmittedDataset.libds])
 
     // Activate new resubmitted table
-    const newSubmittedDataset = this.submittedDatasets.find(sd => sd.libds === activeSubmittedDataset.libds)
+    const newSubmittedDataset = this.submittedDatasets.find(
+      (sd) => sd.libds === activeSubmittedDataset.libds
+    )
     if (newSubmittedDataset) newSubmittedDataset.active = true
   }
 
+  private parseDatasetFromCsvName(fileName: string) {
+    const fileNameArr = fileName.split('.')
+    fileNameArr.pop()
+    return fileNameArr.join('.')
+  }
+
   /**
    *
    * @returns list of strings containing datasets in the HOT user input
@@ -667,7 +810,9 @@ export class MultiDatasetComponent implements OnInit {
 
     const hotData = this.hotInstanceUserDataset.getData()
 
-    return hotData.filter(row => row[0]?.length && row[1]?.length).map(row => row ? `${row[0]}.${row[1]}` : '')
+    return hotData
+      .filter((row) => row[0]?.length && row[1]?.length)
+      .map((row) => (row ? `${row[0]}.${row[1]}` : ''))
   }
 
   private parseExcelSheetNames(): Promise<string[]> {
@@ -718,9 +863,9 @@ export class MultiDatasetComponent implements OnInit {
    *
    *  example: LIB123.TABLE_123
    */
-  private isValidDatasetFormat(sheetName: string) {
+  private isValidDatasetFormat(name: string) {
     const regex = /^\w{1,8}\.\w{1,32}$/gim
-    const correctFormat = regex.test(sheetName)
+    const correctFormat = regex.test(name)
 
     return correctFormat
   }
@@ -867,6 +1012,6 @@ export interface SubmittedDatasetResult {
   libds: string
   success: EditorsStageDataSASResponse | undefined
   error: any
-  parseResult: ParseResult
+  parseResult?: ParseResult
   active?: boolean
 }