chore(release): 6.6.3 [skip ci]

## [6.6.3](https://git.datacontroller.io/dc/dc/compare/v6.6.2...v6.6.3) (2024-02-26)

### Bug Fixes

* allow empty clause value when NE or CONTAINS ([432450a](432450a15b))

CHANGELOG.md

@@ -1,3 +1,44 @@
+## [6.6.3](https://git.datacontroller.io/dc/dc/compare/v6.6.2...v6.6.3) (2024-02-26)
+
+
+### Bug Fixes
+
+* allow empty clause value when NE or CONTAINS ([432450a](https://git.datacontroller.io/dc/dc/commit/432450a15b51a269821ba1d430854f5d1dd04703))
+
+## [6.6.2](https://git.datacontroller.io/dc/dc/compare/v6.6.1...v6.6.2) (2024-02-22)
+
+
+### Bug Fixes
+
+* excel with commas getting wrapped in quotes ([3860134](https://git.datacontroller.io/dc/dc/commit/38601346a529cfe3787bb286a639e0293c365020))
+
+## [6.6.1](https://git.datacontroller.io/dc/dc/compare/v6.6.0...v6.6.1) (2024-02-19)
+
+
+### Bug Fixes
+
+* **client:** bumped @sasjs/adapter with fixed redirected login ([eb1c09d](https://git.datacontroller.io/dc/dc/commit/eb1c09d7909ba07faf763da261545dc1efaec1b3))
+
+# [6.6.0](https://git.datacontroller.io/dc/dc/compare/v6.5.2...v6.6.0) (2024-02-12)
+
+
+### Bug Fixes
+
+* adjust the col numbers in extracted data ([cff5989](https://git.datacontroller.io/dc/dc/commit/cff598955930d2581349e5c6e8b2dd3f9ac96b4c))
+
+
+### Features
+
+* extra table metadata for [#75](https://git.datacontroller.io/dc/dc/issues/75) ([837821f](https://git.datacontroller.io/dc/dc/commit/837821fd01477d340524dfdaf8dd3d3758cf3095))
+* show dsnote on hover title ([6565834](https://git.datacontroller.io/dc/dc/commit/6565834ad4089ecf2de39967e6ed6f217ee4a0a5))
+
+## [6.5.2](https://git.datacontroller.io/dc/dc/compare/v6.5.1...v6.5.2) (2024-02-06)
+
+
+### Bug Fixes
+
+* ordering mpe_selectbox data by the data values after selectbox_order ([2b54034](https://git.datacontroller.io/dc/dc/commit/2b5403497317632a4be8a00f21455c036f1e6461))
+
 ## [6.5.1](https://git.datacontroller.io/dc/dc/compare/v6.5.0...v6.5.1) (2024-02-02)
 
 

client/package-lock.json

@@ -21,7 +21,7 @@
         "@clr/icons": "^13.0.2",
         "@clr/ui": "^13.17.0",
         "@handsontable/angular": "^13.1.0",
-        "@sasjs/adapter": "4.10.1",
+        "@sasjs/adapter": "4.10.2",
         "@sasjs/utils": "^3.4.0",
         "@sheet/crypto": "1.20211122.1",
         "@types/d3-graphviz": "^2.6.7",
@@ -4389,9 +4389,9 @@
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.1.tgz",
-      "integrity": "sha512-/z6eR+3nNaLPyycK8YmpF+GAWNy0zgdl8n4cv4r45hjVBulPHVop7oj57JM/0uIPVOTT2V9IwrMCT/sFPq++vw==",
+      "version": "4.10.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.2.tgz",
+      "integrity": "sha512-IAEbstlfnAckkV1mMhgcJNuOAry55Zhj6OIM7RZiKxiWO5i/q8OLvKMb3Q9KLRT4cS+yB3sRnGe/RQ8mps0fXg==",
       "hasInstallScript": true,
       "dependencies": {
         "@sasjs/utils": "2.52.0",
@@ -23481,9 +23481,9 @@
       "optional": true
     },
     "@sasjs/adapter": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.1.tgz",
-      "integrity": "sha512-/z6eR+3nNaLPyycK8YmpF+GAWNy0zgdl8n4cv4r45hjVBulPHVop7oj57JM/0uIPVOTT2V9IwrMCT/sFPq++vw==",
+      "version": "4.10.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.2.tgz",
+      "integrity": "sha512-IAEbstlfnAckkV1mMhgcJNuOAry55Zhj6OIM7RZiKxiWO5i/q8OLvKMb3Q9KLRT4cS+yB3sRnGe/RQ8mps0fXg==",
       "requires": {
         "@sasjs/utils": "2.52.0",
         "axios": "0.27.2",

client/package.json

@@ -49,7 +49,7 @@
     "@clr/icons": "^13.0.2",
     "@clr/ui": "^13.17.0",
     "@handsontable/angular": "^13.1.0",
-    "@sasjs/adapter": "4.10.1",
+    "@sasjs/adapter": "4.10.2",
     "@sasjs/utils": "^3.4.0",
     "@sheet/crypto": "1.20211122.1",
     "@types/d3-graphviz": "^2.6.7",

client/src/app/editor/editor.component.html

@@ -186,7 +186,9 @@
             } as libdsParsed"
             class="editor-title text-center mt-0-i"
           >
+            <clr-tooltip>
               <clr-icon
+                clrTooltipTrigger
                 (click)="datasetInfo = true"
                 shape="info-circle"
                 class="is-highlight cursor-pointer"
@@ -199,11 +201,22 @@
                 class="color-yellow"
               ></clr-icon>
 
+              <span clrTooltipTrigger>
                 {{ libdsParsed.libName }}.<a
                   class="mr-10"
                   [routerLink]="'/view/data/' + libds!"
                   >{{ libdsParsed.tableName.replace('-FC', '') }}</a
                 >
+              </span>
+              <clr-tooltip-content
+                clrPosition="bottom-left"
+                clrSize="lg"
+                *clrIfOpen
+              >
+                {{ this.dsNote }}
+              </clr-tooltip-content>
+            </clr-tooltip>
+
             <ng-container *ngIf="dataSource">
               <ng-container *ngIf="!zeroFilterRows">
                 ({{ dataSource.length | thousandSeparator: ',' }}

client/src/app/editor/editor.component.ts

@@ -121,6 +121,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
   datasetInfo: boolean = false
   dsmeta: DSMeta[] = []
+  dsNote = ''
 
   viewboxes: boolean = false
 
@@ -939,13 +940,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
             return row.map((col: any, index: number) => {
               if (!col && col !== 0) col = ''
 
-              if (isNaN(col)) {
-                col = col.replace(/"/g, '""')
+              /**
+               * Keeping this for the reference
+               * Code below used to convert JSON to CSV
+               * now the XLSX is converting to CSV
+               */
+              // if (isNaN(col)) {
+              //   // Match and replace the double quotes, ignore the first and last char
+              //   // in case they are double quotes already
+              //   col = col.replace(/(?<!^)"(?!$)/g, '""')
 
-                if (col.search(/,/g) > -1) {
-                  col = '"' + col + '"'
-                }
-              }
+              //   if (col.search(/,/g) > -1 ||
+              //       col.search(/\r|\n/g) > -1
+              //   ) {
+              //     // Missing quotes at the end
+              //     if (col.search(/"$/g) < 0) {
+              //       col = col + '"' // So we add them
+              //     }
+
+              //     // Missing quotes at the start
+              //     if (col.search(/^"/g) < 0) {
+              //       col = '"' + col // So we add them
+              //     }
+              //   }
+              // }
 
               const colName = this.headerShow[index]
               const colRule = this.dcValidator?.getRule(colName)
@@ -960,20 +978,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
           this.data = csvArrayData
 
-          let csvContent = csvArrayHeaders.join(',') + '\n'
-          // Apply licence rows limitation if exists
-          csvContent += csvArrayData
-            .slice(0, this.licenceState.value.submit_rows_limit)
-            .map((e) => e.join(','))
-            .join('\n')
+          // Apply licence rows limitation if exists, it is only affecting data
+          // which will be send to SAS
+          const strippedCsvArrayData = csvArrayData.slice(
+            0,
+            this.licenceState.value.submit_rows_limit
+          )
+          // To submit to sas service, we need clean version of CSV of file
+          // attached. XLSX will do the parsing and heavy lifting
+          // First we create worksheet of json (data we extracted)
+          let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+            skipHeader: true
+          })
+          // create CSV to be uploaded from worksheet
+          let csvContentClean = XLSX.utils.sheet_to_csv(ws)
+          // Prepend headers
+          csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
 
           if (this.encoding === 'WLATIN1') {
-            let encoded = iconv.decode(Buffer.from(csvContent), 'CP-1252')
+            let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
             let blob = new Blob([encoded], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           } else {
-            let blob = new Blob([csvContent], { type: 'application/csv' })
+            let blob = new Blob([csvContentClean], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           }
@@ -2986,6 +3014,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
     this.cols = response.data.cols
     this.dsmeta = response.data.dsmeta
 
+    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
+    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
+    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')
+
+    if (notes && notes.VALUE) {
+      this.dsNote = notes.VALUE
+    } else if (longDesc && longDesc.VALUE) {
+      this.dsNote = longDesc.VALUE
+    } else if (shortDesc && shortDesc.VALUE) {
+      this.dsNote = shortDesc.VALUE
+    } else {
+      this.dsNote = ''
+    }
+
     const hot: Handsontable = this.hotInstance
 
     const approvers: Approver[] = response.data.approvers

client/src/app/query/query.component.ts

@@ -878,17 +878,25 @@ export class QueryComponent
    */
   public hasInvalidCluase(clauses: any): boolean {
     for (let clause of clauses) {
+      clause['invalidClause'] = false
+
       if (
-        clause.variable === null ||
-        clause.operator === null ||
-        clause.value === null ||
-        clause.value === ''
+        clause.value === '' &&
+        !(clause.operator === 'NE' || clause.operator === 'CONTAINS')
+      ) {
+        clause['invalidClause'] = true
+
+        return true
+      }
+
+      if (
+        clause.variable === null ||
+        clause.operator === null ||
+        clause.value === null
       ) {
         clause['invalidClause'] = true
 
         return true
-      } else {
-        clause['invalidClause'] = false
       }
     }
 

client/src/app/viewer/viewer.component.html

@@ -358,7 +358,12 @@
       </section>
 
       <div class="title-col clr-col-auto clr-flex-column clr-flex-sm-row">
+        <h3
+          class="viewerTitle clr-flex-column d-flex clr-flex-sm-row clr-align-items-center clr-justify-content-center"
+        >
+          <clr-tooltip class="d-flex">
             <clr-icon
+              clrTooltipTrigger
               (click)="datasetInfo = true"
               shape="info-circle"
               class="is-highlight cursor-pointer"
@@ -371,12 +376,19 @@
               class="color-yellow mt-5 mr-5"
             ></clr-icon>
 
-        <h3
-          *ngIf="tableTitle && tableTitle.length > 0"
-          class="viewerTitle clr-flex-column d-flex clr-flex-sm-row clr-align-items-center"
-        >
+            <span clrTooltipTrigger *ngIf="tableTitle && tableTitle.length > 0">
               {{ tableTitle?.replace('-FC', '') }}
+            </span>
+            <clr-tooltip-content
+              clrPosition="bottom-left"
+              clrSize="lg"
+              *clrIfOpen
+            >
+              {{ this.dsNote }}
+            </clr-tooltip-content>
+          </clr-tooltip>
 
+          <ng-container *ngIf="tableTitle && tableTitle.length > 0">
             <span *ngIf="numberOfRows !== null">
               ({{ numberOfRows | thousandSeparator: ',' }}
               {{ numberOfRows! === 1 ? 'row' : 'rows' }}, {{ filterCols.length
@@ -388,6 +400,7 @@
               class="refresh-table"
               shape="refresh"
             ></clr-icon>
+          </ng-container>
         </h3>
       </div>
 

client/src/app/viewer/viewer.component.ts

@@ -95,6 +95,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
   public $dataFormats: $DataFormats | null = null
   public datasetInfo: boolean = false
   public dsmeta: DSMeta[] = []
+  public dsNote = ''
 
   public licenceState = this.licenceService.licenceState
   public Infinity = Infinity
@@ -246,6 +247,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
         this.hotTable.data = res.viewdata
         this.$dataFormats = res.$viewdata
         this.dsmeta = res.dsmeta
+        this.setDSNote()
         this.numberOfRows = res.sasparams[0].NOBS
         this.queryText = res.sasparams[0].FILTER_TEXT
         this.headerPks = res.sasparams[0].PK_FIELDS.split(' ')
@@ -803,6 +805,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
           this.hotTable.data = res.viewdata
           this.$dataFormats = res.$viewdata
           this.dsmeta = res.dsmeta
+          this.setDSNote()
           this.queryText = res.sasparams[0].FILTER_TEXT
           let columns: any[] = []
           let colArr = []
@@ -1016,6 +1019,22 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
     this.sasStoreService.removeClause()
   }
 
+  private setDSNote() {
+    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
+    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
+    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')
+
+    if (notes && notes.VALUE) {
+      this.dsNote = notes.VALUE
+    } else if (longDesc && longDesc.VALUE) {
+      this.dsNote = longDesc.VALUE
+    } else if (shortDesc && shortDesc.VALUE) {
+      this.dsNote = shortDesc.VALUE
+    } else {
+      this.dsNote = ''
+    }
+  }
+
   private setupHot() {
     setTimeout(() => {
       if (!this.loadingTableView && this.libDataset) {

client/src/app/xlmap/xlmap.component.ts

@@ -385,24 +385,28 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
       const start = getCellAddress(rule.XLMAP_START, arrayOfObjects)
       const finish = getFinishingCell(start, rule.XLMAP_FINISH, arrayOfObjects)
 
-      const range = `${start}:${finish}`
+      const a1Range = `${start}:${finish}`
+
+      const range = XLSX.utils.decode_range(a1Range)
 
       const rangedData = <any[]>XLSX.utils.sheet_to_json(sheet, {
         raw: true,
-        range: range,
+        range: a1Range,
         header: 'A',
         blankrows: true
       })
 
       for (let i = 0; i < rangedData.length; i++) {
         const row = rangedData[i]
-        // Get the keys of the object (excluding '__rowNum__')
-        const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
 
-        for (let j = 0; j < keys.length; j++) {
-          const key = keys[j]
-          const val = row[key]
+        // `range.s.c` is the index of first column in the range
+        // `range.e.c` is the index of last column in the range
+        // we'll iterate from first column to last column and
+        // extract value where defined and push to extracted data array
+        for (let j = range.s.c, x = 0; j <= range.e.c; j++, x++) {
+          const col = XLSX.utils.encode_col(j)
 
+          if (col in row) {
             // in excel's R1C1 notation indexing starts from 1 but in JS it starts from 0
             // therefore, we'll have to add 1 to rows and cols
             extractedData.push({
@@ -410,11 +414,12 @@ export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
               XLMAP_ID: rule.XLMAP_ID,
               XLMAP_RANGE_ID: rule.XLMAP_RANGE_ID,
               ROW_NO: i + 1,
-            COL_NO: j + 1,
-            VALUE_TXT: val
+              COL_NO: x + 1,
+              VALUE_TXT: row[col]
             })
           }
         }
+      }
     })
 
     this.status = Status.ReadyToSubmit

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "6.5.1",
+  "version": "6.6.3",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/sasjs/macros/mpe_dsmeta.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief Gets table metadata
+  @details Runs mp_dsmeta and adds datadictionary info
+
+  <h4> SAS Macros </h4>
+  @li mp_dsmeta.sas
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_dsmeta(libds, outds=dsmeta);
+  %local ddsd ddld notes lenstmt;
+  %let lenstmt=length ods_table $18 name $100 value $1000;
+  %let libds=%upcase(&libds);
+  %mp_dsmeta(&libds, outds=&outds)
+
+  data _null_;
+    set &mpelib..mpe_datadictionary;
+    where &dc_dttmtfmt < tx_to & dd_source=%upcase("&libds") & dd_type='TABLE';
+    call symputx('ddsd',dd_shortdesc,'l');
+    call symputx('ddld',dd_longdesc,'l');
+  run;
+
+  data &outds;
+    &lenstmt;
+    if last then do;
+      ODS_TABLE='MPE_DATADICTIONARY';
+      NAME='DD_SHORTDESC';
+      VALUE="&ddsd";
+      output;
+      NAME='DD_LONGDESC';
+      VALUE="&ddld";
+      output;
+    end;
+    set &outds end=last;
+    output;
+  run;
+
+  data _data_;
+    set &mpelib..mpe_tables;
+    where libref="%scan(&libds,1,.)"
+        & dsn="%scan(&libds,2,.)"
+        & &dc_dttmtfmt<tx_to;
+    &lenstmt;
+    ODS_TABLE='MPE_TABLES';
+    array c _character_;
+    array n _numeric_;
+    do over c;
+      name=upcase(vname(c));
+      value=c;
+      output;
+    end;
+    do over n;
+      name=upcase(vname(n));
+      value=cats(n);
+      output;
+    end;
+    keep ods_table name value;
+  run;
+
+  proc append base=&outds data=&syslast;
+  run;
+
+
+%mend mpe_dsmeta;

sas/sasjs/macros/mpe_dsmeta.test.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Testing mpe_dsmeta macro
+  @details Checking functionality of mpe_dsmeta.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mpe_dsmeta.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* run the macro*/
+%mp_assertscope(SNAPSHOT)
+%mpe_dsmeta(&mpelib..mpe_security, outds=test1)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot
+)
+
+data work.test1;
+  set work.test1;
+  where ods_table in ('MPE_DATADICTIONARY','MPE_TABLES');
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.test1,
+  desc=Test 1 - 27 records returned,
+  test=EQUALS 27,
+  outds=work.test_results
+)
+
+

sas/sasjs/services/editors/getdata.sas

@@ -14,7 +14,7 @@
   <h5> sasdata </h5>
   <h5> sasparams </h5>
   Contains info on the request.  One row is returned.
-  @li CLS_FLG - set to 0 if there are no CLS rules (everything should be editable)
+  @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
     else set to 1 (CLS rules exist)
   @li ISMAP - set to 1 if the target DS is an excel map target, else 0
 
@@ -56,12 +56,12 @@
   @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
   @li mp_cntlout.sas
-  @li mp_dsmeta.sas
   @li mp_getcols.sas
   @li mp_getmaxvarlengths.sas
   @li mp_validatecol.sas
   @li mpe_accesscheck.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_dsmeta.sas
   @li mpe_getlabels.sas
   @li mpe_filtermaster.sas
   @li mpe_runhook.sas
@@ -645,7 +645,8 @@ create table dqdata as
 %dq_selects()
 
 proc sort data=dqdata;
-  by base_col selectbox_order;
+  /* order by selectbox_order then the value */
+  by base_col selectbox_order rule_data;
 run;
 
 %mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)
@@ -664,7 +665,7 @@ data xl_rules;
   keep xl_column xl_rule;
 run;
 
-%mp_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&libds, outds=dsmeta)
 
 /* send to the client */
 %webout(OPEN)

sas/sasjs/services/editors/getxlmaps.test.sas

@@ -21,7 +21,7 @@
 data _null_;
   file &f1 termstr=crlf;
   put 'XLMAP_ID:$char12.';
-  put "Sample";
+  put "BASEL-KM1";
 run;
 
 %mx_testservice(&_program,
@@ -38,7 +38,7 @@ run;
 
 %mp_assertdsobs(work.xlmaprules,
   test=ATLEAST 2,
-  desc=Checking successful return of at least 2 rules for the Sample map,
+  desc=Checking successful return of at least 2 rules for the BASEL-KM1 map,
   outds=work.test_results
 )
 

sas/sasjs/services/public/viewdata.sas

@@ -44,13 +44,13 @@
   @li mf_verifymacvars.sas
   @li mp_abort.sas
   @li mp_cntlout.sas
-  @li mp_dsmeta.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
   @li mp_jsonout.sas
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_dsmeta.sas
   @li mpe_filtermaster.sas
 
 
@@ -351,7 +351,7 @@ run;
 
 %mp_getcols(&libds, outds=cols)
 
-%mp_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&libds, outds=dsmeta)
 
 %webout(OPEN)
 %webout(OBJ,cls_rules)