chore(release): 6.7.0 [skip ci]

# [6.7.0](https://git.datacontroller.io/dc/dc/compare/v6.6.4...v6.7.0) (2024-04-01)

### Features

* numeric values in hot dropdown aligned right ([9635626](963562621d))

.gitea/workflows/build.yaml

@@ -21,8 +21,17 @@ jobs:
       - name: Lint check
         run: npm run lint:check
 
-      - name: Licence checker
+      - name: Install dependencies
         run: |
           cd client
           npm ci
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
+
+      - name: Licence checker
+        run: |
+          cd client
           npm run license-checker

.gitea/workflows/release.yaml

@@ -34,7 +34,14 @@ jobs:
           CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
+          cd client
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
 
       - name: Check audit
       # Audit should fail and stop the CI if critical vulnerability found
@@ -86,7 +93,14 @@ jobs:
           CYPRESS_CREDS: ${{ secrets.CYPRESS_CREDS }}
 
       - name: Install dependencies
-        run: npm ci
+        run: |
+          npm ci
+          cd client
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
 
       # Install pm2 and prepare SASJS server
       - run: npm i -g pm2
@@ -185,6 +199,11 @@ jobs:
         run: |
           cd client
           npm ci
+          # Install sheet
+          wget ${{ secrets.SHEETLINK }}
+          mv ${{ secrets.SHEETNAME }} ${{ secrets.SHEETNAME }}.tgz
+          npm i ${{ secrets.SHEETNAME }}.tgz
+          # End
           npm run build
 
       - name: Build SAS9 EBI Release

CHANGELOG.md

@@ -1,3 +1,53 @@
+# [6.7.0](https://git.datacontroller.io/dc/dc/compare/v6.6.4...v6.7.0) (2024-04-01)
+
+
+### Features
+
+* numeric values in hot dropdown aligned right ([9635626](https://git.datacontroller.io/dc/dc/commit/963562621ddf0e8d24a29a8481c5e6da1b040708))
+
+## [6.6.4](https://git.datacontroller.io/dc/dc/compare/v6.6.3...v6.6.4) (2024-04-01)
+
+
+### Bug Fixes
+
+* ordering SOFTSELECT numerically in dropdown ([f522038](https://git.datacontroller.io/dc/dc/commit/f522038b8ddb1da14b8adbf8346d0a4539a94cc8)), closes [#85](https://git.datacontroller.io/dc/dc/issues/85)
+* reverting col ([fbbcf90](https://git.datacontroller.io/dc/dc/commit/fbbcf90956bf538b032b0107c07b8576d20353b9))
+* typo ([31d4e5c](https://git.datacontroller.io/dc/dc/commit/31d4e5c727f790d428fb2ea8da60dca929561805))
+
+## [6.6.3](https://git.datacontroller.io/dc/dc/compare/v6.6.2...v6.6.3) (2024-02-26)
+
+
+### Bug Fixes
+
+* allow empty clause value when NE or CONTAINS ([432450a](https://git.datacontroller.io/dc/dc/commit/432450a15b51a269821ba1d430854f5d1dd04703))
+
+## [6.6.2](https://git.datacontroller.io/dc/dc/compare/v6.6.1...v6.6.2) (2024-02-22)
+
+
+### Bug Fixes
+
+* excel with commas getting wrapped in quotes ([3860134](https://git.datacontroller.io/dc/dc/commit/38601346a529cfe3787bb286a639e0293c365020))
+
+## [6.6.1](https://git.datacontroller.io/dc/dc/compare/v6.6.0...v6.6.1) (2024-02-19)
+
+
+### Bug Fixes
+
+* **client:** bumped @sasjs/adapter with fixed redirected login ([eb1c09d](https://git.datacontroller.io/dc/dc/commit/eb1c09d7909ba07faf763da261545dc1efaec1b3))
+
+# [6.6.0](https://git.datacontroller.io/dc/dc/compare/v6.5.2...v6.6.0) (2024-02-12)
+
+
+### Bug Fixes
+
+* adjust the col numbers in extracted data ([cff5989](https://git.datacontroller.io/dc/dc/commit/cff598955930d2581349e5c6e8b2dd3f9ac96b4c))
+
+
+### Features
+
+* extra table metadata for [#75](https://git.datacontroller.io/dc/dc/issues/75) ([837821f](https://git.datacontroller.io/dc/dc/commit/837821fd01477d340524dfdaf8dd3d3758cf3095))
+* show dsnote on hover title ([6565834](https://git.datacontroller.io/dc/dc/commit/6565834ad4089ecf2de39967e6ed6f217ee4a0a5))
+
 ## [6.5.2](https://git.datacontroller.io/dc/dc/compare/v6.5.1...v6.5.2) (2024-02-06)
 
 

client/package-lock.json

@@ -21,9 +21,9 @@
         "@clr/icons": "^13.0.2",
         "@clr/ui": "^13.17.0",
         "@handsontable/angular": "^13.1.0",
-        "@sasjs/adapter": "4.10.1",
+        "@sasjs/adapter": "4.10.2",
         "@sasjs/utils": "^3.4.0",
-        "@sheet/crypto": "1.20211122.1",
+        "@sheet/crypto": "file:../../../RESOURCES/libs/0fae52415ec93a6dd3d5bd33c73b6dc124a07496be568a7aba6d033ec65df60b.tgz",
         "@types/d3-graphviz": "^2.6.7",
         "@types/text-encoding": "0.0.35",
         "base64-arraybuffer": "^0.2.0",
@@ -4389,9 +4389,9 @@
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.1.tgz",
-      "integrity": "sha512-/z6eR+3nNaLPyycK8YmpF+GAWNy0zgdl8n4cv4r45hjVBulPHVop7oj57JM/0uIPVOTT2V9IwrMCT/sFPq++vw==",
+      "version": "4.10.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.2.tgz",
+      "integrity": "sha512-IAEbstlfnAckkV1mMhgcJNuOAry55Zhj6OIM7RZiKxiWO5i/q8OLvKMb3Q9KLRT4cS+yB3sRnGe/RQ8mps0fXg==",
       "hasInstallScript": true,
       "dependencies": {
         "@sasjs/utils": "2.52.0",
@@ -4652,7 +4652,7 @@
     },
     "node_modules/@sheet/crypto": {
       "version": "1.20211122.1",
-      "resolved": "https://pylon.sheetjs.com:54111/@sheet%2fcrypto/-/crypto-1.20211122.1.tgz",
+      "resolved": "file:../../../RESOURCES/libs/0fae52415ec93a6dd3d5bd33c73b6dc124a07496be568a7aba6d033ec65df60b.tgz",
       "integrity": "sha512-G3/HWyzFUYbbVQoQIa+KSeMOhFnK492Ep595FXbzWN9IGZSwuvFl4saEyMl8R8pE2Al5YgSZuR9MpDpx3f7Izg==",
       "bin": {
         "xlsx": "bin/xlsx.njs"
@@ -23481,9 +23481,9 @@
       "optional": true
     },
     "@sasjs/adapter": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.1.tgz",
-      "integrity": "sha512-/z6eR+3nNaLPyycK8YmpF+GAWNy0zgdl8n4cv4r45hjVBulPHVop7oj57JM/0uIPVOTT2V9IwrMCT/sFPq++vw==",
+      "version": "4.10.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.2.tgz",
+      "integrity": "sha512-IAEbstlfnAckkV1mMhgcJNuOAry55Zhj6OIM7RZiKxiWO5i/q8OLvKMb3Q9KLRT4cS+yB3sRnGe/RQ8mps0fXg==",
       "requires": {
         "@sasjs/utils": "2.52.0",
         "axios": "0.27.2",
@@ -23681,8 +23681,7 @@
       }
     },
     "@sheet/crypto": {
-      "version": "1.20211122.1",
-      "resolved": "https://pylon.sheetjs.com:54111/@sheet%2fcrypto/-/crypto-1.20211122.1.tgz",
+      "version": "file:../../../RESOURCES/libs/0fae52415ec93a6dd3d5bd33c73b6dc124a07496be568a7aba6d033ec65df60b.tgz",
       "integrity": "sha512-G3/HWyzFUYbbVQoQIa+KSeMOhFnK492Ep595FXbzWN9IGZSwuvFl4saEyMl8R8pE2Al5YgSZuR9MpDpx3f7Izg=="
     },
     "@sideway/address": {

client/package.json

@@ -49,9 +49,8 @@
     "@clr/icons": "^13.0.2",
     "@clr/ui": "^13.17.0",
     "@handsontable/angular": "^13.1.0",
-    "@sasjs/adapter": "4.10.1",
+    "@sasjs/adapter": "4.10.2",
     "@sasjs/utils": "^3.4.0",
-    "@sheet/crypto": "1.20211122.1",
     "@types/d3-graphviz": "^2.6.7",
     "@types/text-encoding": "0.0.35",
     "base64-arraybuffer": "^0.2.0",

client/src/app/editor/components/edit-record/edit-record.component.html

@@ -112,7 +112,7 @@
 
                 <div
                   *ngIf="
-                    ['autocomplete'].includes(
+                    ['autocomplete', 'autocomplete.custom'].includes(
                       $any(currentRecordValidator?.getRule(col.key)?.editor)
                     )
                   "
@@ -163,7 +163,7 @@
 
                 <div
                   *ngIf="
-                    ['autocomplete'].includes(
+                    ['autocomplete', 'autocomplete.custom'].includes(
                       $any(currentRecordValidator?.getRule(col.key)?.editor)
                     )
                   "

client/src/app/editor/editor.component.html

@@ -186,24 +186,37 @@
             } as libdsParsed"
             class="editor-title text-center mt-0-i"
           >
-            <clr-icon
-              (click)="datasetInfo = true"
-              shape="info-circle"
-              class="is-highlight cursor-pointer"
-              size="24"
-            ></clr-icon>
+            <clr-tooltip>
+              <clr-icon
+                clrTooltipTrigger
+                (click)="datasetInfo = true"
+                shape="info-circle"
+                class="is-highlight cursor-pointer"
+                size="24"
+              ></clr-icon>
 
-            <clr-icon
-              *ngIf="libdsParsed.tableName.includes('-FC')"
-              shape="bolt"
-              class="color-yellow"
-            ></clr-icon>
+              <clr-icon
+                *ngIf="libdsParsed.tableName.includes('-FC')"
+                shape="bolt"
+                class="color-yellow"
+              ></clr-icon>
+
+              <span clrTooltipTrigger>
+                {{ libdsParsed.libName }}.<a
+                  class="mr-10"
+                  [routerLink]="'/view/data/' + libds!"
+                  >{{ libdsParsed.tableName.replace('-FC', '') }}</a
+                >
+              </span>
+              <clr-tooltip-content
+                clrPosition="bottom-left"
+                clrSize="lg"
+                *clrIfOpen
+              >
+                {{ this.dsNote }}
+              </clr-tooltip-content>
+            </clr-tooltip>
 
-            {{ libdsParsed.libName }}.<a
-              class="mr-10"
-              [routerLink]="'/view/data/' + libds!"
-              >{{ libdsParsed.tableName.replace('-FC', '') }}</a
-            >
             <ng-container *ngIf="dataSource">
               <ng-container *ngIf="!zeroFilterRows">
                 ({{ dataSource.length | thousandSeparator: ',' }}

client/src/app/editor/editor.component.ts

@@ -121,6 +121,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
   datasetInfo: boolean = false
   dsmeta: DSMeta[] = []
+  dsNote = ''
 
   viewboxes: boolean = false
 
@@ -939,13 +940,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
             return row.map((col: any, index: number) => {
               if (!col && col !== 0) col = ''
 
-              if (isNaN(col)) {
-                col = col.replace(/"/g, '""')
+              /**
+               * Keeping this for the reference
+               * Code below used to convert JSON to CSV
+               * now the XLSX is converting to CSV
+               */
+              // if (isNaN(col)) {
+              //   // Match and replace the double quotes, ignore the first and last char
+              //   // in case they are double quotes already
+              //   col = col.replace(/(?<!^)"(?!$)/g, '""')
 
-                if (col.search(/,/g) > -1) {
-                  col = '"' + col + '"'
-                }
-              }
+              //   if (col.search(/,/g) > -1 ||
+              //       col.search(/\r|\n/g) > -1
+              //   ) {
+              //     // Missing quotes at the end
+              //     if (col.search(/"$/g) < 0) {
+              //       col = col + '"' // So we add them
+              //     }
+
+              //     // Missing quotes at the start
+              //     if (col.search(/^"/g) < 0) {
+              //       col = '"' + col // So we add them
+              //     }
+              //   }
+              // }
 
               const colName = this.headerShow[index]
               const colRule = this.dcValidator?.getRule(colName)
@@ -960,20 +978,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
           this.data = csvArrayData
 
-          let csvContent = csvArrayHeaders.join(',') + '\n'
-          // Apply licence rows limitation if exists
-          csvContent += csvArrayData
-            .slice(0, this.licenceState.value.submit_rows_limit)
-            .map((e) => e.join(','))
-            .join('\n')
+          // Apply licence rows limitation if exists, it is only affecting data
+          // which will be send to SAS
+          const strippedCsvArrayData = csvArrayData.slice(
+            0,
+            this.licenceState.value.submit_rows_limit
+          )
+          // To submit to sas service, we need clean version of CSV of file
+          // attached. XLSX will do the parsing and heavy lifting
+          // First we create worksheet of json (data we extracted)
+          let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+            skipHeader: true
+          })
+          // create CSV to be uploaded from worksheet
+          let csvContentClean = XLSX.utils.sheet_to_csv(ws)
+          // Prepend headers
+          csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
 
           if (this.encoding === 'WLATIN1') {
-            let encoded = iconv.decode(Buffer.from(csvContent), 'CP-1252')
+            let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
             let blob = new Blob([encoded], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           } else {
-            let blob = new Blob([csvContent], { type: 'application/csv' })
+            let blob = new Blob([csvContentClean], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           }
@@ -1928,13 +1956,13 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
         if (entry.values.length > 0) {
           hot.setCellMeta(entry.row, entry.col, 'renderer', 'autocomplete')
-          hot.setCellMeta(entry.row, entry.col, 'editor', 'autocomplete')
+          hot.setCellMeta(entry.row, entry.col, 'editor', 'autocomplete.custom')
           hot.setCellMeta(entry.row, entry.col, 'strict', entry.strict)
           hot.setCellMeta(entry.row, entry.col, 'filter', false)
 
           this.currentEditRecordValidator?.updateRule(entry.col, {
             renderer: 'autocomplete',
-            editor: 'autocomplete',
+            editor: 'autocomplete.custom',
             strict: entry.strict,
             filter: false
           })
@@ -2029,13 +2057,13 @@ export class EditorComponent implements OnInit, AfterViewInit {
       }
 
       hot.setCellMeta(row, cellCol, 'renderer', 'autocomplete')
-      hot.setCellMeta(row, cellCol, 'editor', 'autocomplete')
+      hot.setCellMeta(row, cellCol, 'editor', 'autocomplete.custom')
       hot.setCellMeta(row, cellCol, 'strict', cellValidationEntry.strict)
       hot.setCellMeta(row, cellCol, 'filter', false)
 
       this.currentEditRecordValidator?.updateRule(cellCol, {
         renderer: 'autocomplete',
-        editor: 'autocomplete',
+        editor: 'autocomplete.custom',
         strict: cellValidationEntry.strict,
         filter: false
       })
@@ -2678,13 +2706,13 @@ export class EditorComponent implements OnInit, AfterViewInit {
           const strict = this.cellValidationSource[validationSourceIndex].strict
 
           hot.setCellMeta(row, column, 'renderer', 'autocomplete')
-          hot.setCellMeta(row, column, 'editor', 'autocomplete')
+          hot.setCellMeta(row, column, 'editor', 'autocomplete.custom')
           hot.setCellMeta(row, column, 'strict', strict)
           hot.setCellMeta(row, column, 'filter', false)
 
           this.currentEditRecordValidator?.updateRule(column, {
             renderer: 'autocomplete',
-            editor: 'autocomplete',
+            editor: 'autocomplete.custom',
             strict: strict,
             filter: false
           })
@@ -2918,6 +2946,37 @@ export class EditorComponent implements OnInit, AfterViewInit {
     )
   }
 
+  /**
+   * Function checks if selected hot cell is solo cell selected
+   * and if it is, set the `filter` property based on filter param.
+   *
+   * @param filter
+   */
+  private setCellFilter(filter: boolean) {
+    const hotSelected = this.hotInstance.getSelected()
+    const selection = hotSelected ? hotSelected[0] : hotSelected
+
+    // When we open a dropdown we want filter disabled so value in cell
+    // don't filter out items, since we want to see them all.
+    // But when we start typing we want to be able to start filtering values
+    // again
+    if (selection) {
+      const startRow = selection[0]
+      const endRow = selection[2]
+      const startCell = selection[1]
+      const endCell = selection[3]
+
+      if (startRow === endRow && startCell === endCell) {
+        const cellMeta = this.hotInstance.getCellMeta(startRow, startCell)
+
+        // If filter is not already set at the value in the param, set it
+        if (cellMeta && cellMeta.filter === !filter) {
+          this.hotInstance.setCellMeta(startRow, startCell, 'filter', filter)
+        }
+      }
+    }
+  }
+
   async ngOnInit() {
     this.licenceService.hot_license_key.subscribe(
       (hot_license_key: string | undefined) => {
@@ -2986,6 +3045,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
     this.cols = response.data.cols
     this.dsmeta = response.data.dsmeta
 
+    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
+    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
+    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')
+
+    if (notes && notes.VALUE) {
+      this.dsNote = notes.VALUE
+    } else if (longDesc && longDesc.VALUE) {
+      this.dsNote = longDesc.VALUE
+    } else if (shortDesc && shortDesc.VALUE) {
+      this.dsNote = shortDesc.VALUE
+    } else {
+      this.dsNote = ''
+    }
+
     const hot: Handsontable = this.hotInstance
 
     const approvers: Approver[] = response.data.approvers
@@ -3259,28 +3332,16 @@ export class EditorComponent implements OnInit, AfterViewInit {
       }
     )
 
-    hot.addHook('beforeKeyDown', (e: any) => {
-      const hotSelected = this.hotInstance.getSelected()
-      const selection = hotSelected ? hotSelected[0] : hotSelected
-
+    hot.addHook('afterBeginEditing', () => {
       // When we open a dropdown we want filter disabled so value in cell
       // don't filter out items, since we want to see them all.
+      this.setCellFilter(false)
+    })
+
+    hot.addHook('beforeKeyDown', () => {
       // When we start typing, we are enabling the filter since we want to find
       // values faster.
-      if (selection) {
-        const startRow = selection[0]
-        const endRow = selection[2]
-        const startCell = selection[1]
-        const endCell = selection[3]
-
-        if (startRow === endRow && startCell === endCell) {
-          const cellMeta = this.hotInstance.getCellMeta(startRow, startCell)
-
-          if (cellMeta && cellMeta.filter === false) {
-            this.hotInstance.setCellMeta(startRow, startCell, 'filter', true)
-          }
-        }
-      }
+      this.setCellFilter(true)
     })
 
     hot.addHook('afterChange', (source: any, change: any) => {

client/src/app/query/query.component.ts

@@ -878,17 +878,25 @@ export class QueryComponent
    */
   public hasInvalidCluase(clauses: any): boolean {
     for (let clause of clauses) {
+      clause['invalidClause'] = false
+
       if (
-        clause.variable === null ||
-        clause.operator === null ||
-        clause.value === null ||
-        clause.value === ''
+        clause.value === '' &&
+        !(clause.operator === 'NE' || clause.operator === 'CONTAINS')
+      ) {
+        clause['invalidClause'] = true
+
+        return true
+      }
+
+      if (
+        clause.variable === null ||
+        clause.operator === null ||
+        clause.value === null
       ) {
         clause['invalidClause'] = true
 
         return true
-      } else {
-        clause['invalidClause'] = false
       }
     }
 

client/src/app/shared/dc-validator/dc-validator.ts

@@ -20,6 +20,7 @@ import { parseColType } from './utils/parseColType'
 import { dqValidate } from './validations/dq-validation'
 import { specialMissingNumericValidator } from './validations/hot-custom-validators'
 import { applyNumericFormats } from './utils/applyNumericFormats'
+import { CustomAutocompleteEditor } from './editors/numericAutocomplete'
 
 export class DcValidator {
   private rules: DcValidation[] = []
@@ -38,6 +39,8 @@ export class DcValidator {
     dqData: DQData[],
     hotInstance?: Handsontable
   ) {
+    this.registerCustomEditors()
+
     this.sasparams = sasparams
     this.hotInstance = hotInstance
     this.rules = parseColType(sasparams.COLTYPE)
@@ -51,6 +54,13 @@ export class DcValidator {
     this.setupValidations()
   }
 
+  registerCustomEditors() {
+    Handsontable.editors.registerEditor(
+      'autocomplete.custom',
+      CustomAutocompleteEditor
+    )
+  }
+
   getRules(): DcValidation[] {
     return this.rules
   }
@@ -262,6 +272,7 @@ export class DcValidator {
         if (source.length > 0) {
           this.rules[i].source = source
           this.rules[i].type = 'autocomplete'
+          this.rules[i].editor = 'autocomplete.custom'
           this.rules[i].filter = false
         }
 
@@ -315,7 +326,10 @@ export class DcValidator {
 
       // Because of dynamic cell validation, that will change the type of cell to dropdown
       // `rules[i].colType` could be different type (eg. numeric). So we check if current cell is dropdown, to call HOT native dropdown validator
-      if (this.editor === 'autocomplete') {
+      if (
+        this.editor === 'autocomplete' ||
+        this.editor === 'autocomplete.custom'
+      ) {
         self
           .getHandsontableValidator('autocomplete')
           .call(this, value, (valid: boolean) => {

client/src/app/shared/dc-validator/editors/numericAutocomplete.ts

@@ -0,0 +1,28 @@
+import Handsontable from 'handsontable'
+import Core from 'handsontable/core'
+
+export class CustomAutocompleteEditor extends Handsontable.editors
+  .AutocompleteEditor {
+  constructor(instance: Core) {
+    super(instance)
+  }
+
+  createElements() {
+    super.createElements()
+  }
+
+  // Listbox open
+  open(event?: Event | undefined): void {
+    super.open(event)
+
+    if (this.isCellNumeric()) {
+      this.htContainer.classList.add('numericListbox')
+    } else {
+      this.htContainer.classList.remove('numericListbox')
+    }
+  }
+
+  isCellNumeric() {
+    return this.cellProperties?.className?.includes('htNumeric')
+  }
+}

client/src/app/viewer/viewer.component.html

@@ -358,36 +358,49 @@
       </section>
 
       <div class="title-col clr-col-auto clr-flex-column clr-flex-sm-row">
-        <clr-icon
-          (click)="datasetInfo = true"
-          shape="info-circle"
-          class="is-highlight cursor-pointer"
-          size="24"
-        ></clr-icon>
-
-        <clr-icon
-          *ngIf="tableTitle?.includes('-FC')"
-          shape="bolt"
-          class="color-yellow mt-5 mr-5"
-        ></clr-icon>
-
         <h3
-          *ngIf="tableTitle && tableTitle.length > 0"
-          class="viewerTitle clr-flex-column d-flex clr-flex-sm-row clr-align-items-center"
+          class="viewerTitle clr-flex-column d-flex clr-flex-sm-row clr-align-items-center clr-justify-content-center"
         >
-          {{ tableTitle?.replace('-FC', '') }}
+          <clr-tooltip class="d-flex">
+            <clr-icon
+              clrTooltipTrigger
+              (click)="datasetInfo = true"
+              shape="info-circle"
+              class="is-highlight cursor-pointer"
+              size="24"
+            ></clr-icon>
 
-          <span *ngIf="numberOfRows !== null">
-            ({{ numberOfRows | thousandSeparator: ',' }}
-            {{ numberOfRows! === 1 ? 'row' : 'rows' }}, {{ filterCols.length
-            }}{{ filterCols.length === 1 ? ' col' : ' cols' }})
-          </span>
+            <clr-icon
+              *ngIf="tableTitle?.includes('-FC')"
+              shape="bolt"
+              class="color-yellow mt-5 mr-5"
+            ></clr-icon>
 
-          <clr-icon
-            (click)="reloadTableData()"
-            class="refresh-table"
-            shape="refresh"
-          ></clr-icon>
+            <span clrTooltipTrigger *ngIf="tableTitle && tableTitle.length > 0">
+              {{ tableTitle?.replace('-FC', '') }}
+            </span>
+            <clr-tooltip-content
+              clrPosition="bottom-left"
+              clrSize="lg"
+              *clrIfOpen
+            >
+              {{ this.dsNote }}
+            </clr-tooltip-content>
+          </clr-tooltip>
+
+          <ng-container *ngIf="tableTitle && tableTitle.length > 0">
+            <span *ngIf="numberOfRows !== null">
+              ({{ numberOfRows | thousandSeparator: ',' }}
+              {{ numberOfRows! === 1 ? 'row' : 'rows' }}, {{ filterCols.length
+              }}{{ filterCols.length === 1 ? ' col' : ' cols' }})
+            </span>
+
+            <clr-icon
+              (click)="reloadTableData()"
+              class="refresh-table"
+              shape="refresh"
+            ></clr-icon>
+          </ng-container>
         </h3>
       </div>
 

client/src/app/viewer/viewer.component.ts

@@ -95,6 +95,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
   public $dataFormats: $DataFormats | null = null
   public datasetInfo: boolean = false
   public dsmeta: DSMeta[] = []
+  public dsNote = ''
 
   public licenceState = this.licenceService.licenceState
   public Infinity = Infinity
@@ -246,6 +247,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
         this.hotTable.data = res.viewdata
         this.$dataFormats = res.$viewdata
         this.dsmeta = res.dsmeta
+        this.setDSNote()
         this.numberOfRows = res.sasparams[0].NOBS
         this.queryText = res.sasparams[0].FILTER_TEXT
         this.headerPks = res.sasparams[0].PK_FIELDS.split(' ')
@@ -803,6 +805,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
           this.hotTable.data = res.viewdata
           this.$dataFormats = res.$viewdata
           this.dsmeta = res.dsmeta
+          this.setDSNote()
           this.queryText = res.sasparams[0].FILTER_TEXT
           let columns: any[] = []
           let colArr = []
@@ -1016,6 +1019,22 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
     this.sasStoreService.removeClause()
   }
 
+  private setDSNote() {
+    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
+    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
+    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')
+
+    if (notes && notes.VALUE) {
+      this.dsNote = notes.VALUE
+    } else if (longDesc && longDesc.VALUE) {
+      this.dsNote = longDesc.VALUE
+    } else if (shortDesc && shortDesc.VALUE) {
+      this.dsNote = shortDesc.VALUE
+    } else {
+      this.dsNote = ''
+    }
+  }
+
   private setupHot() {
     setTimeout(() => {
       if (!this.loadingTableView && this.libDataset) {

client/src/app/xlmap/xlmap.component.ts

@@ -385,34 +385,39 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
       const start = getCellAddress(rule.XLMAP_START, arrayOfObjects)
       const finish = getFinishingCell(start, rule.XLMAP_FINISH, arrayOfObjects)
 
-      const range = `${start}:${finish}`
+      const a1Range = `${start}:${finish}`
+
+      const range = XLSX.utils.decode_range(a1Range)
 
       const rangedData = <any[]>XLSX.utils.sheet_to_json(sheet, {
         raw: true,
-        range: range,
+        range: a1Range,
         header: 'A',
         blankrows: true
       })
 
       for (let i = 0; i < rangedData.length; i++) {
         const row = rangedData[i]
-        // Get the keys of the object (excluding '__rowNum__')
-        const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
 
-        for (let j = 0; j < keys.length; j++) {
-          const key = keys[j]
-          const val = row[key]
+        // `range.s.c` is the index of first column in the range
+        // `range.e.c` is the index of last column in the range
+        // we'll iterate from first column to last column and
+        // extract value where defined and push to extracted data array
+        for (let j = range.s.c, x = 0; j <= range.e.c; j++, x++) {
+          const col = XLSX.utils.encode_col(j)
 
-          // in excel's R1C1 notation indexing starts from 1 but in JS it starts from 0
-          // therefore, we'll have to add 1 to rows and cols
-          extractedData.push({
-            LOAD_REF: '0',
-            XLMAP_ID: rule.XLMAP_ID,
-            XLMAP_RANGE_ID: rule.XLMAP_RANGE_ID,
-            ROW_NO: i + 1,
-            COL_NO: j + 1,
-            VALUE_TXT: val
-          })
+          if (col in row) {
+            // in excel's R1C1 notation indexing starts from 1 but in JS it starts from 0
+            // therefore, we'll have to add 1 to rows and cols
+            extractedData.push({
+              LOAD_REF: '0',
+              XLMAP_ID: rule.XLMAP_ID,
+              XLMAP_RANGE_ID: rule.XLMAP_RANGE_ID,
+              ROW_NO: i + 1,
+              COL_NO: x + 1,
+              VALUE_TXT: row[col]
+            })
+          }
         }
       }
     })

client/src/environments/_eula.ts

@@ -18,4 +18,4 @@ In any case, you must not make any such use of this software as to develop softw
 UNLESS EXPRESSLY AGREED OTHERWISE, 4GL APPS PROVIDES THIS SOFTWARE ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, AND IN NO EVENT AND UNDER NO LEGAL THEORY, SHALL 4GL APPS BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING FROM USE OR INABILITY TO USE THIS SOFTWARE.
 
 
-`
+`

client/src/styles.scss

@@ -713,6 +713,11 @@ clr-icon.is-info {
   border: 1px solid red !important;
   color: #ffffff !important;
 }
+
+.handsontable .numericListbox {
+  text-align: right;
+}
+
 .margin-top-20 {
   margin-top: 20px;
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "6.5.2",
+  "version": "6.7.0",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/package-lock.json

@@ -229,12 +229,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/tough-cookie": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
-      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
-      "peer": true
-    },
     "node_modules/abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -1933,12 +1927,6 @@
         "@types/node": "*"
       }
     },
-    "@types/tough-cookie": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
-      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
-      "peer": true
-    },
     "abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -2965,8 +2953,7 @@
     "ws": {
       "version": "8.13.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
-      "requires": {}
+      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA=="
     },
     "xml": {
       "version": "1.0.1",

sas/sasjs/macros/mpe_dsmeta.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief Gets table metadata
+  @details Runs mp_dsmeta and adds datadictionary info
+
+  <h4> SAS Macros </h4>
+  @li mp_dsmeta.sas
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_dsmeta(libds, outds=dsmeta);
+  %local ddsd ddld notes lenstmt;
+  %let lenstmt=length ods_table $18 name $100 value $1000;
+  %let libds=%upcase(&libds);
+  %mp_dsmeta(&libds, outds=&outds)
+
+  data _null_;
+    set &mpelib..mpe_datadictionary;
+    where &dc_dttmtfmt < tx_to & dd_source=%upcase("&libds") & dd_type='TABLE';
+    call symputx('ddsd',dd_shortdesc,'l');
+    call symputx('ddld',dd_longdesc,'l');
+  run;
+
+  data &outds;
+    &lenstmt;
+    if last then do;
+      ODS_TABLE='MPE_DATADICTIONARY';
+      NAME='DD_SHORTDESC';
+      VALUE="&ddsd";
+      output;
+      NAME='DD_LONGDESC';
+      VALUE="&ddld";
+      output;
+    end;
+    set &outds end=last;
+    output;
+  run;
+
+  data _data_;
+    set &mpelib..mpe_tables;
+    where libref="%scan(&libds,1,.)"
+        & dsn="%scan(&libds,2,.)"
+        & &dc_dttmtfmt<tx_to;
+    &lenstmt;
+    ODS_TABLE='MPE_TABLES';
+    array c _character_;
+    array n _numeric_;
+    do over c;
+      name=upcase(vname(c));
+      value=c;
+      output;
+    end;
+    do over n;
+      name=upcase(vname(n));
+      value=cats(n);
+      output;
+    end;
+    keep ods_table name value;
+  run;
+
+  proc append base=&outds data=&syslast;
+  run;
+
+
+%mend mpe_dsmeta;

sas/sasjs/macros/mpe_dsmeta.test.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Testing mpe_dsmeta macro
+  @details Checking functionality of mpe_dsmeta.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mpe_dsmeta.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* run the macro*/
+%mp_assertscope(SNAPSHOT)
+%mpe_dsmeta(&mpelib..mpe_security, outds=test1)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot
+)
+
+data work.test1;
+  set work.test1;
+  where ods_table in ('MPE_DATADICTIONARY','MPE_TABLES');
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.test1,
+  desc=Test 1 - 27 records returned,
+  test=EQUALS 27,
+  outds=work.test_results
+)
+
+

sas/sasjs/macros/mpe_makedata.sas

@@ -1874,6 +1874,16 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/columns_in_libds"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* test softselect on numeric var (should be ordered numerically) */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_X_TEST"
+    ,base_col="SOME_BESTNUM"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="&lib..MPE_X_TEST.SOME_BESTNUM"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"

sas/sasjs/services/editors/getdata.sas

@@ -14,7 +14,7 @@
   <h5> sasdata </h5>
   <h5> sasparams </h5>
   Contains info on the request.  One row is returned.
-  @li CLS_FLG - set to 0 if there are no CLS rules (everything should be editable)
+  @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
     else set to 1 (CLS rules exist)
   @li ISMAP - set to 1 if the target DS is an excel map target, else 0
 
@@ -56,12 +56,12 @@
   @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
   @li mp_cntlout.sas
-  @li mp_dsmeta.sas
   @li mp_getcols.sas
   @li mp_getmaxvarlengths.sas
   @li mp_validatecol.sas
   @li mpe_accesscheck.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_dsmeta.sas
   @li mpe_getlabels.sas
   @li mpe_filtermaster.sas
   @li mpe_runhook.sas
@@ -631,9 +631,14 @@ create table dqdata as
       select distinct "&&base_col&x" as base_col length=32
         ,"&source" as rule_value length=74
         ,cats(&col) as rule_data length=1000
-        ,0 as selectbox_order
+        ,&col as tmp_order
       from &lib..&ds
-      order by 1;
+      order by tmp_order;
+    /* ensure both numerics and char vals are ordered correctly */
+    data work.dqdata&x (drop=tmp_order);
+      set work.dqdata&x;
+      selectbox_order=_n_;
+    run;
     %mp_abort(iftrue= (&syscc ne 0)
       ,mac=&_program
       ,msg=%str(syscc=&syscc when selecting &&base_col&x from &orig_libds)
@@ -665,7 +670,7 @@ data xl_rules;
   keep xl_column xl_rule;
 run;
 
-%mp_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&libds, outds=dsmeta)
 
 /* send to the client */
 %webout(OPEN)

sas/sasjs/services/editors/getxlmaps.test.sas

@@ -21,7 +21,7 @@
 data _null_;
   file &f1 termstr=crlf;
   put 'XLMAP_ID:$char12.';
-  put "Sample";
+  put "BASEL-KM1";
 run;
 
 %mx_testservice(&_program,
@@ -38,7 +38,7 @@ run;
 
 %mp_assertdsobs(work.xlmaprules,
   test=ATLEAST 2,
-  desc=Checking successful return of at least 2 rules for the Sample map,
+  desc=Checking successful return of at least 2 rules for the BASEL-KM1 map,
   outds=work.test_results
 )
 

sas/sasjs/services/public/viewdata.sas

@@ -44,13 +44,13 @@
   @li mf_verifymacvars.sas
   @li mp_abort.sas
   @li mp_cntlout.sas
-  @li mp_dsmeta.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
   @li mp_jsonout.sas
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_dsmeta.sas
   @li mpe_filtermaster.sas
 
 
@@ -351,7 +351,7 @@ run;
 
 %mp_getcols(&libds, outds=cols)
 
-%mp_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&libds, outds=dsmeta)
 
 %webout(OPEN)
 %webout(OBJ,cls_rules)