Merge pull request 'fix: allow CSV uploads with licence row limit' (#219) from fix/213-csv-license-row-limit into main

Reviewed-on: #219

.gitea/workflows/build.yaml

@@ -126,7 +126,7 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

.gitea/workflows/release.yaml

@@ -136,7 +136,7 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

client/cypress/e2e/csv-limited.cy.ts

@@ -0,0 +1,95 @@
+const username = Cypress.env('username')
+const password = Cypress.env('password')
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+const serverType = Cypress.env('serverType')
+const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
+const fixturePath = 'csvs/'
+
+context('csv file upload restriction (free tier): ', function () {
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    cy.get('body').then(($body) => {
+      const usernameInput = $body.find('input.username')[0]
+
+      if (usernameInput && !Cypress.dom.isHidden(usernameInput)) {
+        cy.get('input.username').type(username)
+        cy.get('input.password').type(password)
+        cy.get('.login-group button').click()
+      }
+    })
+
+    cy.get('.app-loading', { timeout: longerCommandTimeout }).should(
+      'not.exist'
+    )
+
+    // Skip licensing page if presented - continue with free tier
+    cy.url().then((url) => {
+      if (url.includes('licensing')) {
+        cy.get('button').contains('Continue with free tier').click()
+      }
+    })
+
+    visitPage('home')
+  })
+
+  it('1 | File upload is restricted on free tier', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    // Click upload button - should show feature locked modal
+    cy.get('.buttonBar button:last-child').should('exist').click()
+
+    cy.get('.modal-title').should('contain', 'Locked Feature (File Upload)')
+  })
+})
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let targetLib
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            targetLib = node
+            break
+          }
+        }
+
+        cy.get(targetLib).within(() => {
+          cy.get('.clr-tree-node-content-container > button').click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+}
+
+const attachFile = (filename: string, callback?: any) => {
+  cy.get('.buttonBar button:last-child')
+    .should('exist')
+    .click()
+    .then(() => {
+      cy.get('input[type="file"]#file-upload')
+        .attachFile(`/${fixturePath}/${filename}`)
+        .then(() => {
+          if (callback) callback()
+        })
+    })
+}
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}

client/cypress/fixtures/csvs/regular.csv

@@ -4,7 +4,11 @@ PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,SOME_NUM,SOME_DATE,SOME_DATETIME,SOME_
 2,even more dummy data,Option 3,42,12FEB1960,01JAN1960:00:00:42,0:02:22,3,44
 3,"It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told:",Option 2,1613.001,27FEB1961,01JAN1960:00:07:03,0:00:44,3,44
 4,if you can fill the unforgiving minute,Option 1,1613.0011235,02AUG1971,29MAY1973:06:12:03,0:06:52,3,44
-1010,10 bottles of beer on the wall,Option 1,0.9153696885,04MAR1962,01JAN1960:12:47:55,0:01:40,92,76
+1010,"10 bottles of beer
+
+
+
+ on the wall",Option 1,0.9153696885,04MAR1962,01JAN1960:12:47:55,0:01:40,92,76
 1011,11 bottles of beer on the wall,Option 1,0.3531217558,29MAR1960,01JAN1960:03:33:24,0:01:03,80,29
 1012,12 bottles of beer on the wall,Option 1,0.6743748717,02AUG1962,01JAN1960:07:25:59,0:00:10,16,98
 1013,13 bottles of beer on the wall,Option 1,0.1305445992,11SEP1960,01JAN1960:13:51:32,0:00:35,73,15

client/src/app/_globals.ts

@@ -55,6 +55,7 @@ export interface HandsontableStaticConfig {
  * Cached viyaApi collections, search and selected endpoint
  */
 export const globals: {
+  embed: boolean
   rootParam: string
   dcLib: string
   xlmaps: XLMapListItem[]
@@ -69,6 +70,7 @@ export const globals: {
   handsontable: HandsontableStaticConfig
   [key: string]: any
 } = {
+  embed: false,
   rootParam: <string>'',
   dcLib: '',
   xlmaps: [],

client/src/app/app.component.html

@@ -107,7 +107,7 @@
     </div>
   </ng-container>
 
-  <header class="app-header">
+  <header class="app-header" *ngIf="!embed">
     <!-- <button
       *ngIf="
         isMainRoute('view') ||
@@ -213,9 +213,10 @@
   </header>
   <nav
     *ngIf="
-      router.url.includes('submitted') ||
+      !embed &&
+      (router.url.includes('submitted') ||
         router.url.includes('approve') ||
-      router.url.includes('history')
+        router.url.includes('history'))
     "
     class="subnav"
   >

client/src/app/app.component.ts

@@ -70,6 +70,7 @@ export class AppComponent {
 
   public syssite = this.appService.syssite
   public licenceState = this.licenceService.licenceState
+  public embed = globals.embed
 
   constructor(
     private appService: AppService,
@@ -143,6 +144,16 @@ export class AppComponent {
       }
     })
 
+    const hashQuery = window.location.hash.split('?')[1]
+    if (hashQuery) {
+      const embedParam = new URLSearchParams(hashQuery).get('embed')
+      if (embedParam !== null) {
+        const isEmbed = embedParam !== 'false'
+        globals.embed = isEmbed
+        this.embed = isEmbed
+      }
+    }
+
     this.subscribeToShowAbortModal()
     this.subscribeToRequestsModal()
     this.subscribeToStartupData()

client/src/app/editor/editor.component.html

@@ -165,7 +165,7 @@
         class="card-header clr-row buttonBar headerBar clr-flex-md-row clr-justify-content-center clr-justify-content-lg-end"
       >
         <div
-          *ngIf="tableTrue"
+          *ngIf="tableTrue && !embed"
           class="clr-col-12 clr-col-md-3 clr-col-lg-4 backBtn"
         >
           <span

client/src/app/editor/editor.component.ts

@@ -264,6 +264,9 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
   public badEdit = false
   public badEditCause: string | undefined
   public badEditTitle: string | undefined
+  get embed() {
+    return globals.embed
+  }
   public tableTrue: boolean | undefined
   public saveLoading = false
   public approvers: string[] = []

client/src/app/free-tier.config.ts

@@ -30,7 +30,7 @@ export const freeTierConfig: LicenceState = {
   lineage_daily_limit: 3,
   tables_in_library_limit: 35,
   viewbox: true,
-  fileUpload: true,
+  fileUpload: false,
   editRecord: true,
   addRecord: true
 }

client/src/app/shared/spreadsheet-util/spreadsheet-util.ts

@@ -375,38 +375,30 @@ export class SpreadsheetUtil {
     fileType: string
   ): Promise<ParseResult> {
     return new Promise((resolve, reject) => {
-      if (this.licenceState.value.submit_rows_limit !== Infinity) {
+      if (!this.licenceState.value.fileUpload) {
         uploader.queue.pop()
         return reject(
-          'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
+          'File uploads are not enabled for this licence. Please contact support@datacontroller.io'
         )
       }
 
-      if (parseParams.encoding === 'WLATIN1') {
-        let reader = new FileReader()
-        const self = this
-        // Closure to capture the file information.
-        reader.onload = (theFile: any) => {
-          let encoded = iconv.decode(
-            Buffer.from(theFile.target.result),
-            'CP-1252'
-          )
-          let blob = new Blob([encoded], { type: fileType })
-          let encodedFile: File = blobToFile(blob, parseParams.file.name)
+      if (parseParams.encoding !== 'WLATIN1') return resolve({ uploader })
+
+      const reader = new FileReader()
+      reader.onload = (theFile) => {
+        if (!theFile.target?.result) return resolve({ uploader })
+
+        const text = theFile.target.result as string
+        const encoded = iconv.encode(text, 'CP-1252')
+        const blob = new Blob([encoded], { type: fileType })
+        const encodedFile: File = blobToFile(blob, parseParams.file.name)
         uploader.queue.pop()
         uploader.addToQueue([encodedFile])
 
-          return resolve({
-            uploader
-          })
+        return resolve({ uploader })
       }
 
-        reader.readAsArrayBuffer(parseParams.file)
-      } else {
-        return resolve({
-          uploader
-        })
-      }
+      reader.readAsText(parseParams.file)
     })
   }
 

client/src/app/system/system.component.html

@@ -236,7 +236,19 @@
         <div class="admin-action">
           Download Configuration
 
-          <button (click)="downloadConfiguration()" class="btn btn-info btn-sm">
+          <input
+            type="text"
+            class="clr-input libref-input"
+            maxlength="8"
+            [ngModel]="dcLib"
+            (ngModelChange)="targetLibref = $event.toUpperCase()"
+            placeholder="Target Libref"
+          />
+          <button
+            (click)="downloadConfiguration()"
+            [disabled]="targetLibref !== dcLib && !isValidLibref(targetLibref)"
+            class="btn btn-info btn-sm"
+          >
             DOWNLOAD
           </button>
         </div>

client/src/app/system/system.component.scss

@@ -0,0 +1,5 @@
+.libref-input {
+  width: 100px;
+  margin: 0 8px;
+  text-transform: uppercase;
+}

client/src/app/system/system.component.ts

@@ -10,6 +10,7 @@ import { EnvironmentInfo } from './models/environment-info.model'
 import { AppSettingsService } from '../services/app-settings.service'
 import { AppSettings } from '../models/AppSettings'
 import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { globals } from '../_globals'
 
 @Component({
   selector: 'app-system',
@@ -39,6 +40,8 @@ export class SystemComponent implements OnInit {
   responseModal: boolean = false
 
   Infinity = Infinity
+  dcLib: string = globals.dcLib
+  targetLibref: string = globals.dcLib
 
   licenceState = this.licenceService.licenceState
   settings: AppSettings
@@ -71,13 +74,21 @@ export class SystemComponent implements OnInit {
     this.appSettingsService.setAppSettings(this.settings)
   }
 
+  isValidLibref(value: string): boolean {
+    return /^[A-Za-z_]\w{0,7}$/.test(value.trim())
+  }
+
   downloadConfiguration() {
     let sasjsConfig = this.sasService.getSasjsConfig()
     let storage = sasjsConfig.serverUrl
     let metaData = sasjsConfig.appLoc
     let path = this.sasService.getExecutionPath()
+    let lib = this.targetLibref.toUpperCase().trim()
     let downUrl =
       storage + path + '/?_program=' + metaData + '/services/admin/exportconfig'
+    if (lib && lib !== this.dcLib && this.isValidLibref(lib)) {
+      downUrl += '&dclib=' + encodeURIComponent(lib)
+    }
     window.open(downUrl)
   }
 

sas/sasjs/services/public/getrawdata.sas

@@ -16,6 +16,7 @@
   @li mf_existfeature.sas
   @li dc_assignlib.sas
   @li mp_ds2cards.sas
+  @li mp_ds2csv.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_cntlout.sas
@@ -117,10 +118,8 @@ options validvarname=upcase;
   /* cannot proc export excel if PC Files is not licensed */
   %then %do;
     %let outfile=%sysfunc(pathname(work))/&table..csv;
-    PROC EXPORT DATA= staged
-        OUTFILE= "&outfile"
-        DBMS=csv REPLACE;
-    RUN;
+    /* cannot use PROC EXPORT as we need to wrap all char values in quotes */
+    %mp_ds2csv(work.staged,outfile="&outfile",headerformat=NAME)
     %let ext=csv;
     %let mimetype=csv;
   %end;