chore(release): 6.7.0 [skip ci]

# [6.7.0](https://git.datacontroller.io/dc/dc/compare/v6.6.4...v6.7.0) (2024-04-01)

### Features

* numeric values in hot dropdown aligned right ([9635626](963562621d))

.gitea/workflows/build.yaml

@@ -21,8 +21,17 @@ jobs:
       - name: Lint check
         run: npm run lint:check
 
-      - name: Licence checker
+      - name: Install dependencies
         run: |
           cd client
           npm ci
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
+
+      - name: Licence checker
+        run: |
+          cd client
           npm run license-checker

.gitea/workflows/release.yaml

@@ -34,7 +34,14 @@ jobs:
           CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
+          cd client
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
 
       - name: Check audit
       # Audit should fail and stop the CI if critical vulnerability found
@@ -86,7 +93,14 @@ jobs:
           CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
+          cd client
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
 
       # Install pm2 and prepare SASJS server
       - run: npm i -g pm2
@@ -185,6 +199,11 @@ jobs:
         run: |
           cd client
           npm ci
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
           npm run build
 
       - name: Build SAS9 EBI Release

CHANGELOG.md

@@ -1,3 +1,33 @@
+# [6.7.0](https://git.datacontroller.io/dc/dc/compare/v6.6.4...v6.7.0) (2024-04-01)
+
+
+### Features
+
+* numeric values in hot dropdown aligned right ([9635626](https://git.datacontroller.io/dc/dc/commit/963562621ddf0e8d24a29a8481c5e6da1b040708))
+
+## [6.6.4](https://git.datacontroller.io/dc/dc/compare/v6.6.3...v6.6.4) (2024-04-01)
+
+
+### Bug Fixes
+
+* ordering SOFTSELECT numerically in dropdown ([f522038](https://git.datacontroller.io/dc/dc/commit/f522038b8ddb1da14b8adbf8346d0a4539a94cc8)), closes [#85](https://git.datacontroller.io/dc/dc/issues/85)
+* reverting col ([fbbcf90](https://git.datacontroller.io/dc/dc/commit/fbbcf90956bf538b032b0107c07b8576d20353b9))
+* typo ([31d4e5c](https://git.datacontroller.io/dc/dc/commit/31d4e5c727f790d428fb2ea8da60dca929561805))
+
+## [6.6.3](https://git.datacontroller.io/dc/dc/compare/v6.6.2...v6.6.3) (2024-02-26)
+
+
+### Bug Fixes
+
+* allow empty clause value when NE or CONTAINS ([432450a](https://git.datacontroller.io/dc/dc/commit/432450a15b51a269821ba1d430854f5d1dd04703))
+
+## [6.6.2](https://git.datacontroller.io/dc/dc/compare/v6.6.1...v6.6.2) (2024-02-22)
+
+
+### Bug Fixes
+
+* excel with commas getting wrapped in quotes ([3860134](https://git.datacontroller.io/dc/dc/commit/38601346a529cfe3787bb286a639e0293c365020))
+
 ## [6.6.1](https://git.datacontroller.io/dc/dc/compare/v6.6.0...v6.6.1) (2024-02-19)
 
 

client/package-lock.json

@@ -23,7 +23,7 @@
         "@handsontable/angular": "^13.1.0",
         "@sasjs/adapter": "4.10.2",
         "@sasjs/utils": "^3.4.0",
-        "@sheet/crypto": "1.20211122.1",
+        "@sheet/crypto": "file:../../../RESOURCES/libs/0fae52415ec93a6dd3d5bd33c73b6dc124a07496be568a7aba6d033ec65df60b.tgz",
         "@types/d3-graphviz": "^2.6.7",
         "@types/text-encoding": "0.0.35",
         "base64-arraybuffer": "^0.2.0",
@@ -4652,7 +4652,7 @@
     },
     "node_modules/@sheet/crypto": {
       "version": "1.20211122.1",
-      "resolved": "https://pylon.sheetjs.com:54111/@sheet%2fcrypto/-/crypto-1.20211122.1.tgz",
+      "resolved": "file:../../../RESOURCES/libs/0fae52415ec93a6dd3d5bd33c73b6dc124a07496be568a7aba6d033ec65df60b.tgz",
       "integrity": "sha512-G3/HWyzFUYbbVQoQIa+KSeMOhFnK492Ep595FXbzWN9IGZSwuvFl4saEyMl8R8pE2Al5YgSZuR9MpDpx3f7Izg==",
       "bin": {
         "xlsx": "bin/xlsx.njs"
@@ -23681,8 +23681,7 @@
       }
     },
     "@sheet/crypto": {
-      "version": "1.20211122.1",
-      "resolved": "https://pylon.sheetjs.com:54111/@sheet%2fcrypto/-/crypto-1.20211122.1.tgz",
+      "version": "file:../../../RESOURCES/libs/0fae52415ec93a6dd3d5bd33c73b6dc124a07496be568a7aba6d033ec65df60b.tgz",
       "integrity": "sha512-G3/HWyzFUYbbVQoQIa+KSeMOhFnK492Ep595FXbzWN9IGZSwuvFl4saEyMl8R8pE2Al5YgSZuR9MpDpx3f7Izg=="
     },
     "@sideway/address": {

client/package.json

@@ -51,7 +51,6 @@
     "@handsontable/angular": "^13.1.0",
     "@sasjs/adapter": "4.10.2",
     "@sasjs/utils": "^3.4.0",
-    "@sheet/crypto": "1.20211122.1",
     "@types/d3-graphviz": "^2.6.7",
     "@types/text-encoding": "0.0.35",
     "base64-arraybuffer": "^0.2.0",

client/src/app/editor/components/edit-record/edit-record.component.html

@@ -112,7 +112,7 @@
 
                 <div
                   *ngIf="
-                    ['autocomplete'].includes(
+                    ['autocomplete', 'autocomplete.custom'].includes(
                       $any(currentRecordValidator?.getRule(col.key)?.editor)
                     )
                   "
@@ -163,7 +163,7 @@
 
                 <div
                   *ngIf="
-                    ['autocomplete'].includes(
+                    ['autocomplete', 'autocomplete.custom'].includes(
                       $any(currentRecordValidator?.getRule(col.key)?.editor)
                     )
                   "

client/src/app/editor/editor.component.ts

@@ -940,13 +940,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
             return row.map((col: any, index: number) => {
               if (!col && col !== 0) col = ''
 
-              if (isNaN(col)) {
-                col = col.replace(/"/g, '""')
+              /**
+               * Keeping this for the reference
+               * Code below used to convert JSON to CSV
+               * now the XLSX is converting to CSV
+               */
+              // if (isNaN(col)) {
+              //   // Match and replace the double quotes, ignore the first and last char
+              //   // in case they are double quotes already
+              //   col = col.replace(/(?<!^)"(?!$)/g, '""')
 
-                if (col.search(/,/g) > -1) {
-                  col = '"' + col + '"'
-                }
-              }
+              //   if (col.search(/,/g) > -1 ||
+              //       col.search(/\r|\n/g) > -1
+              //   ) {
+              //     // Missing quotes at the end
+              //     if (col.search(/"$/g) < 0) {
+              //       col = col + '"' // So we add them
+              //     }
+
+              //     // Missing quotes at the start
+              //     if (col.search(/^"/g) < 0) {
+              //       col = '"' + col // So we add them
+              //     }
+              //   }
+              // }
 
               const colName = this.headerShow[index]
               const colRule = this.dcValidator?.getRule(colName)
@@ -961,20 +978,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
           this.data = csvArrayData
 
-          let csvContent = csvArrayHeaders.join(',') + '\n'
-          // Apply licence rows limitation if exists
-          csvContent += csvArrayData
-            .slice(0, this.licenceState.value.submit_rows_limit)
-            .map((e) => e.join(','))
-            .join('\n')
+          // Apply licence rows limitation if exists, it is only affecting data
+          // which will be send to SAS
+          const strippedCsvArrayData = csvArrayData.slice(
+            0,
+            this.licenceState.value.submit_rows_limit
+          )
+          // To submit to sas service, we need clean version of CSV of file
+          // attached. XLSX will do the parsing and heavy lifting
+          // First we create worksheet of json (data we extracted)
+          let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+            skipHeader: true
+          })
+          // create CSV to be uploaded from worksheet
+          let csvContentClean = XLSX.utils.sheet_to_csv(ws)
+          // Prepend headers
+          csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
 
           if (this.encoding === 'WLATIN1') {
-            let encoded = iconv.decode(Buffer.from(csvContent), 'CP-1252')
+            let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
             let blob = new Blob([encoded], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           } else {
-            let blob = new Blob([csvContent], { type: 'application/csv' })
+            let blob = new Blob([csvContentClean], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           }
@@ -1929,13 +1956,13 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
         if (entry.values.length > 0) {
           hot.setCellMeta(entry.row, entry.col, 'renderer', 'autocomplete')
-          hot.setCellMeta(entry.row, entry.col, 'editor', 'autocomplete')
+          hot.setCellMeta(entry.row, entry.col, 'editor', 'autocomplete.custom')
           hot.setCellMeta(entry.row, entry.col, 'strict', entry.strict)
           hot.setCellMeta(entry.row, entry.col, 'filter', false)
 
           this.currentEditRecordValidator?.updateRule(entry.col, {
             renderer: 'autocomplete',
-            editor: 'autocomplete',
+            editor: 'autocomplete.custom',
             strict: entry.strict,
             filter: false
           })
@@ -2030,13 +2057,13 @@ export class EditorComponent implements OnInit, AfterViewInit {
       }
 
       hot.setCellMeta(row, cellCol, 'renderer', 'autocomplete')
-      hot.setCellMeta(row, cellCol, 'editor', 'autocomplete')
+      hot.setCellMeta(row, cellCol, 'editor', 'autocomplete.custom')
       hot.setCellMeta(row, cellCol, 'strict', cellValidationEntry.strict)
       hot.setCellMeta(row, cellCol, 'filter', false)
 
       this.currentEditRecordValidator?.updateRule(cellCol, {
         renderer: 'autocomplete',
-        editor: 'autocomplete',
+        editor: 'autocomplete.custom',
         strict: cellValidationEntry.strict,
         filter: false
       })
@@ -2679,13 +2706,13 @@ export class EditorComponent implements OnInit, AfterViewInit {
           const strict = this.cellValidationSource[validationSourceIndex].strict
 
           hot.setCellMeta(row, column, 'renderer', 'autocomplete')
-          hot.setCellMeta(row, column, 'editor', 'autocomplete')
+          hot.setCellMeta(row, column, 'editor', 'autocomplete.custom')
           hot.setCellMeta(row, column, 'strict', strict)
           hot.setCellMeta(row, column, 'filter', false)
 
           this.currentEditRecordValidator?.updateRule(column, {
             renderer: 'autocomplete',
-            editor: 'autocomplete',
+            editor: 'autocomplete.custom',
             strict: strict,
             filter: false
           })
@@ -2919,6 +2946,37 @@ export class EditorComponent implements OnInit, AfterViewInit {
     )
   }
 
+  /**
+   * Function checks if selected hot cell is solo cell selected
+   * and if it is, set the `filter` property based on filter param.
+   *
+   * @param filter
+   */
+  private setCellFilter(filter: boolean) {
+    const hotSelected = this.hotInstance.getSelected()
+    const selection = hotSelected ? hotSelected[0] : hotSelected
+
+    // When we open a dropdown we want filter disabled so value in cell
+    // don't filter out items, since we want to see them all.
+    // But when we start typing we want to be able to start filtering values
+    // again
+    if (selection) {
+      const startRow = selection[0]
+      const endRow = selection[2]
+      const startCell = selection[1]
+      const endCell = selection[3]
+
+      if (startRow === endRow && startCell === endCell) {
+        const cellMeta = this.hotInstance.getCellMeta(startRow, startCell)
+
+        // If filter is not already set at the value in the param, set it
+        if (cellMeta && cellMeta.filter === !filter) {
+          this.hotInstance.setCellMeta(startRow, startCell, 'filter', filter)
+        }
+      }
+    }
+  }
+
   async ngOnInit() {
     this.licenceService.hot_license_key.subscribe(
       (hot_license_key: string | undefined) => {
@@ -3274,28 +3332,16 @@ export class EditorComponent implements OnInit, AfterViewInit {
       }
     )
 
-    hot.addHook('beforeKeyDown', (e: any) => {
-      const hotSelected = this.hotInstance.getSelected()
-      const selection = hotSelected ? hotSelected[0] : hotSelected
-
+    hot.addHook('afterBeginEditing', () => {
       // When we open a dropdown we want filter disabled so value in cell
       // don't filter out items, since we want to see them all.
+      this.setCellFilter(false)
+    })
+
+    hot.addHook('beforeKeyDown', () => {
       // When we start typing, we are enabling the filter since we want to find
       // values faster.
-      if (selection) {
-        const startRow = selection[0]
-        const endRow = selection[2]
-        const startCell = selection[1]
-        const endCell = selection[3]
-
-        if (startRow === endRow && startCell === endCell) {
-          const cellMeta = this.hotInstance.getCellMeta(startRow, startCell)
-
-          if (cellMeta && cellMeta.filter === false) {
-            this.hotInstance.setCellMeta(startRow, startCell, 'filter', true)
-          }
-        }
-      }
+      this.setCellFilter(true)
     })
 
     hot.addHook('afterChange', (source: any, change: any) => {

client/src/app/query/query.component.ts

@@ -878,17 +878,25 @@ export class QueryComponent
    */
   public hasInvalidCluase(clauses: any): boolean {
     for (let clause of clauses) {
+      clause['invalidClause'] = false
+
       if (
-        clause.variable === null ||
-        clause.operator === null ||
-        clause.value === null ||
-        clause.value === ''
+        clause.value === '' &&
+        !(clause.operator === 'NE' || clause.operator === 'CONTAINS')
+      ) {
+        clause['invalidClause'] = true
+
+        return true
+      }
+
+      if (
+        clause.variable === null ||
+        clause.operator === null ||
+        clause.value === null
       ) {
         clause['invalidClause'] = true
 
         return true
-      } else {
-        clause['invalidClause'] = false
       }
     }
 

client/src/app/shared/dc-validator/dc-validator.ts

@@ -20,6 +20,7 @@ import { parseColType } from './utils/parseColType'
 import { dqValidate } from './validations/dq-validation'
 import { specialMissingNumericValidator } from './validations/hot-custom-validators'
 import { applyNumericFormats } from './utils/applyNumericFormats'
+import { CustomAutocompleteEditor } from './editors/numericAutocomplete'
 
 export class DcValidator {
   private rules: DcValidation[] = []
@@ -38,6 +39,8 @@ export class DcValidator {
     dqData: DQData[],
     hotInstance?: Handsontable
   ) {
+    this.registerCustomEditors()
+
     this.sasparams = sasparams
     this.hotInstance = hotInstance
     this.rules = parseColType(sasparams.COLTYPE)
@@ -51,6 +54,13 @@ export class DcValidator {
     this.setupValidations()
   }
 
+  registerCustomEditors() {
+    Handsontable.editors.registerEditor(
+      'autocomplete.custom',
+      CustomAutocompleteEditor
+    )
+  }
+
   getRules(): DcValidation[] {
     return this.rules
   }
@@ -262,6 +272,7 @@ export class DcValidator {
         if (source.length > 0) {
           this.rules[i].source = source
           this.rules[i].type = 'autocomplete'
+          this.rules[i].editor = 'autocomplete.custom'
           this.rules[i].filter = false
         }
 
@@ -315,7 +326,10 @@ export class DcValidator {
 
       // Because of dynamic cell validation, that will change the type of cell to dropdown
       // `rules[i].colType` could be different type (eg. numeric). So we check if current cell is dropdown, to call HOT native dropdown validator
-      if (this.editor === 'autocomplete') {
+      if (
+        this.editor === 'autocomplete' ||
+        this.editor === 'autocomplete.custom'
+      ) {
         self
           .getHandsontableValidator('autocomplete')
           .call(this, value, (valid: boolean) => {

client/src/app/shared/dc-validator/editors/numericAutocomplete.ts

@@ -0,0 +1,28 @@
+import Handsontable from 'handsontable'
+import Core from 'handsontable/core'
+
+export class CustomAutocompleteEditor extends Handsontable.editors
+  .AutocompleteEditor {
+  constructor(instance: Core) {
+    super(instance)
+  }
+
+  createElements() {
+    super.createElements()
+  }
+
+  // Listbox open
+  open(event?: Event | undefined): void {
+    super.open(event)
+
+    if (this.isCellNumeric()) {
+      this.htContainer.classList.add('numericListbox')
+    } else {
+      this.htContainer.classList.remove('numericListbox')
+    }
+  }
+
+  isCellNumeric() {
+    return this.cellProperties?.className?.includes('htNumeric')
+  }
+}

client/src/styles.scss

@@ -713,6 +713,11 @@ clr-icon.is-info {
   border: 1px solid red !important;
   color: #ffffff !important;
 }
+
+.handsontable .numericListbox {
+  text-align: right;
+}
+
 .margin-top-20 {
   margin-top: 20px;
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "6.6.1",
+  "version": "6.7.0",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/package-lock.json

@@ -229,12 +229,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/tough-cookie": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
-      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
-      "peer": true
-    },
     "node_modules/abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -1933,12 +1927,6 @@
         "@types/node": "*"
       }
     },
-    "@types/tough-cookie": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
-      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
-      "peer": true
-    },
     "abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -2965,8 +2953,7 @@
     "ws": {
       "version": "8.13.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
-      "requires": {}
+      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA=="
     },
     "xml": {
       "version": "1.0.1",

sas/sasjs/macros/mpe_makedata.sas

@@ -1874,6 +1874,16 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/columns_in_libds"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* test softselect on numeric var (should be ordered numerically) */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_X_TEST"
+    ,base_col="SOME_BESTNUM"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="&lib..MPE_X_TEST.SOME_BESTNUM"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"

sas/sasjs/services/editors/getdata.sas

@@ -631,9 +631,14 @@ create table dqdata as
       select distinct "&&base_col&x" as base_col length=32
         ,"&source" as rule_value length=74
         ,cats(&col) as rule_data length=1000
-        ,0 as selectbox_order
+        ,&col as tmp_order
       from &lib..&ds
-      order by 1;
+      order by tmp_order;
+    /* ensure both numerics and char vals are ordered correctly */
+    data work.dqdata&x (drop=tmp_order);
+      set work.dqdata&x;
+      selectbox_order=_n_;
+    run;
     %mp_abort(iftrue= (&syscc ne 0)
       ,mac=&_program
       ,msg=%str(syscc=&syscc when selecting &&base_col&x from &orig_libds)