chore(release): 6.6.2 [skip ci]

## [6.6.2](https://git.datacontroller.io/dc/dc/compare/v6.6.1...v6.6.2) (2024-02-22)

### Bug Fixes

* excel with commas getting wrapped in quotes ([3860134](38601346a5))

.gitea/workflows/release.yaml

@@ -150,7 +150,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Write .npmrc file
         run: |

.vscode/settings.json

@@ -1,18 +1,19 @@
 {
-    "cSpell.words": [
+  "cSpell.words": [
-        "SYSERRORTEXT",
+    "Licence",
-        "SYSWARNINGTEXT"
+    "SYSERRORTEXT",
-    ],
+    "SYSWARNINGTEXT",
-    "editor.rulers": [
+    "xlmaprules",
-        80
+    "xlmaps"
-    ],
+  ],
-    "files.trimTrailingWhitespace": true,
+  "editor.rulers": [80],
-    "[markdown]": {
+  "files.trimTrailingWhitespace": true,
-        "files.trimTrailingWhitespace": false
+  "[markdown]": {
-    },
+    "files.trimTrailingWhitespace": false
-    "workbench.colorCustomizations": {
+  },
-        "titleBar.activeForeground": "#ebe8e8",
+  "workbench.colorCustomizations": {
-        "titleBar.activeBackground": "#95ff0053",
+    "titleBar.activeForeground": "#ebe8e8",
-    },
+    "titleBar.activeBackground": "#95ff0053"
-    "terminal.integrated.wordSeparators": " ()[]{}',\"`─‘’"
+  },
-}
+  "terminal.integrated.wordSeparators": " ()[]{}',\"`─‘’"
+}

CHANGELOG.md

@@ -1,3 +1,79 @@
+## [6.6.2](https://git.datacontroller.io/dc/dc/compare/v6.6.1...v6.6.2) (2024-02-22)
+
+
+### Bug Fixes
+
+* excel with commas getting wrapped in quotes ([3860134](https://git.datacontroller.io/dc/dc/commit/38601346a529cfe3787bb286a639e0293c365020))
+
+## [6.6.1](https://git.datacontroller.io/dc/dc/compare/v6.6.0...v6.6.1) (2024-02-19)
+
+
+### Bug Fixes
+
+* **client:** bumped @sasjs/adapter with fixed redirected login ([eb1c09d](https://git.datacontroller.io/dc/dc/commit/eb1c09d7909ba07faf763da261545dc1efaec1b3))
+
+# [6.6.0](https://git.datacontroller.io/dc/dc/compare/v6.5.2...v6.6.0) (2024-02-12)
+
+
+### Bug Fixes
+
+* adjust the col numbers in extracted data ([cff5989](https://git.datacontroller.io/dc/dc/commit/cff598955930d2581349e5c6e8b2dd3f9ac96b4c))
+
+
+### Features
+
+* extra table metadata for [#75](https://git.datacontroller.io/dc/dc/issues/75) ([837821f](https://git.datacontroller.io/dc/dc/commit/837821fd01477d340524dfdaf8dd3d3758cf3095))
+* show dsnote on hover title ([6565834](https://git.datacontroller.io/dc/dc/commit/6565834ad4089ecf2de39967e6ed6f217ee4a0a5))
+
+## [6.5.2](https://git.datacontroller.io/dc/dc/compare/v6.5.1...v6.5.2) (2024-02-06)
+
+
+### Bug Fixes
+
+* ordering mpe_selectbox data by the data values after selectbox_order ([2b54034](https://git.datacontroller.io/dc/dc/commit/2b5403497317632a4be8a00f21455c036f1e6461))
+
+## [6.5.1](https://git.datacontroller.io/dc/dc/compare/v6.5.0...v6.5.1) (2024-02-02)
+
+
+### Bug Fixes
+
+* ensuring submitter email can be pulled from mpe_emails ([eac0104](https://git.datacontroller.io/dc/dc/commit/eac0104d7aebaf98ff1d1c504c1ce3b25d4a0ce8))
+
+# [6.5.0](https://git.datacontroller.io/dc/dc/compare/v6.4.0...v6.5.0) (2024-01-26)
+
+
+### Features
+
+* filtering by reference to Variables as well as Values ([6eb1aa8](https://git.datacontroller.io/dc/dc/commit/6eb1aa85d29294d63e6af377e622fbed7fd1fab8))
+
+# [6.4.0](https://git.datacontroller.io/dc/dc/compare/v6.3.1...v6.4.0) (2024-01-24)
+
+
+### Bug Fixes
+
+* add dcLib to globals ([5d93346](https://git.datacontroller.io/dc/dc/commit/5d93346b52eda27c2829770e96686a713296d373))
+* add service to get xlmap rules and fixed interface name ([9ffa30a](https://git.datacontroller.io/dc/dc/commit/9ffa30ab747f5b62acbd452431a5e6e440afcb80))
+* increasing length of mpe_excel_map cols to ([2d4d068](https://git.datacontroller.io/dc/dc/commit/2d4d068413dcdac98581f08939e74bde65b73428))
+* providing info on mapids to FE ([fd94945](https://git.datacontroller.io/dc/dc/commit/fd94945466c1a797ddc89815258a65624a9cb0cf))
+* removing tables from EDIT menu that are in xlmaps ([9550ae4](https://git.datacontroller.io/dc/dc/commit/9550ae4d1154a0272f8a2427ac9d2afdfd699c96))
+* removing XLMAP_TARGETLIBDS from mpe_xlmaps_rules table ([93702c6](https://git.datacontroller.io/dc/dc/commit/93702c63dc280cdba1e46f0fd8fe0deaec879611))
+* renaming TABLE macvar to LOAD_REF in postdata.sas ([01915a2](https://git.datacontroller.io/dc/dc/commit/01915a2db9a4dfb94e4e8213e2c32181da36d349))
+* reverting xlmap in getdata change ([2d6e747](https://git.datacontroller.io/dc/dc/commit/2d6e747db9b84e9fb0dfcf9102a2f7dd2cb51891))
+* update edit tab to load ([516e5a2](https://git.datacontroller.io/dc/dc/commit/516e5a206216f79ab1dce9f4eab0d31115743160))
+
+
+### Features
+
+* adding ability to define the target table for excel maps ([c86fba9](https://git.datacontroller.io/dc/dc/commit/c86fba9dc75ddc6033132f469ad1c31b9131b12e))
+* adding ismap attribute to getdata response (and fixing test) ([2702bb3](https://git.datacontroller.io/dc/dc/commit/2702bb3c84c45903def1aa2b8cc20a6dd080281b))
+* Complex Excel Uploads ([cf19381](https://git.datacontroller.io/dc/dc/commit/cf193810606f287b8d6f864c4eb64d43c5ab5f3c)), closes [#69](https://git.datacontroller.io/dc/dc/issues/69)
+* Create Tables / Files dropdown under load tab ([b473b19](https://git.datacontroller.io/dc/dc/commit/b473b198a61f468dff74cd8e64692e7847084a80))
+* display list of maps in sidebar ([5aec024](https://git.datacontroller.io/dc/dc/commit/5aec0242429942f8a989b5fb79f8d3865e9de01a))
+* implemented the logic for xlmap component ([50696bb](https://git.datacontroller.io/dc/dc/commit/50696bb926dd00472db65a008771a4b6352871be))
+* model changes for [#69](https://git.datacontroller.io/dc/dc/issues/69) ([271543a](https://git.datacontroller.io/dc/dc/commit/271543a446a2116718f99f0540e3cd911f9f5fe7))
+* new getxlmaps service to return rules for a particular xlmap_id ([56264ec](https://git.datacontroller.io/dc/dc/commit/56264ecc6908bf6c8e3e666dfeba7068d6195df8))
+* validating the excel map after stage (adding load-ref) ([a485c3b](https://git.datacontroller.io/dc/dc/commit/a485c3b78724a36f7bacb264fb02140cc62d6512))
+
 ## [6.3.1](https://git.datacontroller.io/dc/dc/compare/v6.3.0...v6.3.1) (2024-01-01)
 
 

client/package-lock.json

@@ -21,7 +21,7 @@
         "@clr/icons": "^13.0.2",
         "@clr/ui": "^13.17.0",
         "@handsontable/angular": "^13.1.0",
-        "@sasjs/adapter": "4.10.1",
+        "@sasjs/adapter": "4.10.2",
         "@sasjs/utils": "^3.4.0",
         "@sheet/crypto": "1.20211122.1",
         "@types/d3-graphviz": "^2.6.7",
@@ -4389,9 +4389,9 @@
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "4.10.1",
+      "version": "4.10.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.2.tgz",
-      "integrity": "sha512-/z6eR+3nNaLPyycK8YmpF+GAWNy0zgdl8n4cv4r45hjVBulPHVop7oj57JM/0uIPVOTT2V9IwrMCT/sFPq++vw==",
+      "integrity": "sha512-IAEbstlfnAckkV1mMhgcJNuOAry55Zhj6OIM7RZiKxiWO5i/q8OLvKMb3Q9KLRT4cS+yB3sRnGe/RQ8mps0fXg==",
       "hasInstallScript": true,
       "dependencies": {
         "@sasjs/utils": "2.52.0",
@@ -23481,9 +23481,9 @@
       "optional": true
     },
     "@sasjs/adapter": {
-      "version": "4.10.1",
+      "version": "4.10.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.10.2.tgz",
-      "integrity": "sha512-/z6eR+3nNaLPyycK8YmpF+GAWNy0zgdl8n4cv4r45hjVBulPHVop7oj57JM/0uIPVOTT2V9IwrMCT/sFPq++vw==",
+      "integrity": "sha512-IAEbstlfnAckkV1mMhgcJNuOAry55Zhj6OIM7RZiKxiWO5i/q8OLvKMb3Q9KLRT4cS+yB3sRnGe/RQ8mps0fXg==",
       "requires": {
         "@sasjs/utils": "2.52.0",
         "axios": "0.27.2",

client/package.json

@@ -49,7 +49,7 @@
     "@clr/icons": "^13.0.2",
     "@clr/ui": "^13.17.0",
     "@handsontable/angular": "^13.1.0",
-    "@sasjs/adapter": "4.10.1",
+    "@sasjs/adapter": "4.10.2",
     "@sasjs/utils": "^3.4.0",
     "@sheet/crypto": "1.20211122.1",
     "@types/d3-graphviz": "^2.6.7",

client/src/app/_globals.ts

@@ -37,6 +37,12 @@ export const initFilter: { filter: FilterCache } = {
   }
 }
 
+export interface XLMapListItem {
+  id: string
+  description: string
+  targetDS: string
+}
+
 /**
  * Cached filtering values across whole app (editor, viewer, viewboxes)
  * Cached lineage libraries, tables
@@ -46,6 +52,8 @@ export const initFilter: { filter: FilterCache } = {
  */
 export const globals: {
   rootParam: string
+  dcLib: string
+  xlmaps: XLMapListItem[]
   editor: any
   viewer: any
   viewboxes: ViewboxCache
@@ -57,11 +65,13 @@ export const globals: {
   [key: string]: any
 } = {
   rootParam: <string>'',
+  dcLib: '',
+  xlmaps: [],
   editor: {
     startupSet: <boolean>false,
     treeNodeLibraries: <any[] | null>[],
     libsAndTables: <any[]>[],
-    libraries: <String[] | undefined>[],
+    libraries: <string[] | undefined>[],
     library: <string>'',
     table: <string>'',
     filter: <FilterCache>{

client/src/app/app.component.html

@@ -168,7 +168,7 @@
           </button>
           <clr-dropdown-menu *clrIfOpen clrPosition="bottom-left">
             <a [routerLink]="['/view']" clrDropdownItem>VIEW</a>
-            <a [routerLink]="['/home']" clrDropdownItem>EDIT</a>
+            <a [routerLink]="['/home']" clrDropdownItem>LOAD</a>
             <a [routerLink]="['/review/submitted']" clrDropdownItem>REVIEW</a>
           </clr-dropdown-menu>
         </clr-dropdown>
@@ -189,7 +189,7 @@
             router.url.includes('edit-record') ||
             router.url.includes('home')
           "
-          >EDIT</a
+          >LOAD</a
         >
         <a
           [routerLink]="['/review/submitted']"

client/src/app/app.routing.ts

@@ -4,19 +4,19 @@
  * The full license information can be found in LICENSE in the root directory of this project.
  */
 import { ModuleWithProviders } from '@angular/core'
-import { Routes, RouterModule } from '@angular/router'
+import { RouterModule, Routes } from '@angular/router'
 
-import { HomeComponent } from './home/home.component'
 import { NotFoundComponent } from './not-found/not-found.component'
 
+import { DeployModule } from './deploy/deploy.module'
+import { EditorModule } from './editor/editor.module'
+import { HomeModule } from './home/home.module'
+import { LicensingModule } from './licensing/licensing.module'
+import { ReviewModule } from './review/review.module'
 import { ReviewRouteComponent } from './routes/review-route/review-route.component'
 import { StageModule } from './stage/stage.module'
-import { EditorModule } from './editor/editor.module'
-import { ViewerModule } from './viewer/viewer.module'
-import { ReviewModule } from './review/review.module'
-import { DeployModule } from './deploy/deploy.module'
-import { LicensingModule } from './licensing/licensing.module'
 import { SystemModule } from './system/system.module'
+import { ViewerModule } from './viewer/viewer.module'
 
 /**
  * Defining routes
@@ -45,7 +45,7 @@ export const ROUTES: Routes = [
     path: 'licensing',
     loadChildren: () => LicensingModule
   },
-  { path: 'home', component: HomeComponent },
+  { path: 'home', loadChildren: () => HomeModule },
   {
     /**
      * Load editor module with subroutes

client/src/app/editor/editor.component.html

@@ -186,24 +186,37 @@
             } as libdsParsed"
             class="editor-title text-center mt-0-i"
           >
-            <clr-icon
+            <clr-tooltip>
-              (click)="datasetInfo = true"
+              <clr-icon
-              shape="info-circle"
+                clrTooltipTrigger
-              class="is-highlight cursor-pointer"
+                (click)="datasetInfo = true"
-              size="24"
+                shape="info-circle"
-            ></clr-icon>
+                class="is-highlight cursor-pointer"
+                size="24"
+              ></clr-icon>
 
-            <clr-icon
+              <clr-icon
-              *ngIf="libdsParsed.tableName.includes('-FC')"
+                *ngIf="libdsParsed.tableName.includes('-FC')"
-              shape="bolt"
+                shape="bolt"
-              class="color-yellow"
+                class="color-yellow"
-            ></clr-icon>
+              ></clr-icon>
+
+              <span clrTooltipTrigger>
+                {{ libdsParsed.libName }}.<a
+                  class="mr-10"
+                  [routerLink]="'/view/data/' + libds!"
+                  >{{ libdsParsed.tableName.replace('-FC', '') }}</a
+                >
+              </span>
+              <clr-tooltip-content
+                clrPosition="bottom-left"
+                clrSize="lg"
+                *clrIfOpen
+              >
+                {{ this.dsNote }}
+              </clr-tooltip-content>
+            </clr-tooltip>
 
-            {{ libdsParsed.libName }}.<a
-              class="mr-10"
-              [routerLink]="'/view/data/' + libds!"
-              >{{ libdsParsed.tableName.replace('-FC', '') }}</a
-            >
             <ng-container *ngIf="dataSource">
               <ng-container *ngIf="!zeroFilterRows">
                 ({{ dataSource.length | thousandSeparator: ',' }}

client/src/app/editor/editor.component.ts

@@ -38,7 +38,7 @@ import { HotTableInterface } from '../models/HotTable.interface'
 import {
   $DataFormats,
   DSMeta,
-  EditorsGetdataServiceResponse
+  EditorsGetDataServiceResponse
 } from '../models/sas/editors-getdata.model'
 import { DataFormat } from '../models/sas/common/DateFormat'
 import SheetInfo from '../models/SheetInfo'
@@ -121,6 +121,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
   datasetInfo: boolean = false
   dsmeta: DSMeta[] = []
+  dsNote = ''
 
   viewboxes: boolean = false
 
@@ -939,13 +940,30 @@ export class EditorComponent implements OnInit, AfterViewInit {
             return row.map((col: any, index: number) => {
               if (!col && col !== 0) col = ''
 
-              if (isNaN(col)) {
+              /**
-                col = col.replace(/"/g, '""')
+               * Keeping this for the reference
+               * Code below used to convert JSON to CSV
+               * now the XLSX is converting to CSV
+               */
+              // if (isNaN(col)) {
+              //   // Match and replace the double quotes, ignore the first and last char
+              //   // in case they are double quotes already
+              //   col = col.replace(/(?<!^)"(?!$)/g, '""')
 
-                if (col.search(/,/g) > -1) {
+              //   if (col.search(/,/g) > -1 ||
-                  col = '"' + col + '"'
+              //       col.search(/\r|\n/g) > -1
-                }
+              //   ) {
-              }
+              //     // Missing quotes at the end
+              //     if (col.search(/"$/g) < 0) {
+              //       col = col + '"' // So we add them
+              //     }
+
+              //     // Missing quotes at the start
+              //     if (col.search(/^"/g) < 0) {
+              //       col = '"' + col // So we add them
+              //     }
+              //   }
+              // }
 
               const colName = this.headerShow[index]
               const colRule = this.dcValidator?.getRule(colName)
@@ -960,20 +978,27 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
           this.data = csvArrayData
 
-          let csvContent = csvArrayHeaders.join(',') + '\n'
+          // Apply licence rows limitation if exists, it is only affecting data
-          // Apply licence rows limitation if exists
+          // which will be send to SAS
-          csvContent += csvArrayData
+          const strippedCsvArrayData = csvArrayData.slice(0, this.licenceState.value.submit_rows_limit)
-            .slice(0, this.licenceState.value.submit_rows_limit)
+          // To submit to sas service, we need clean version of CSV of file
-            .map((e) => e.join(','))
+          // attached. XLSX will do the parsing and heavy lifting
-            .join('\n')
+          // First we create worksheet of json (data we extracted)
+          let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
+            skipHeader: true
+          });
+          // create CSV to be uploaded from worksheet
+          let csvContentClean = XLSX.utils.sheet_to_csv(ws);
+          // Prepend headers
+          csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean
 
           if (this.encoding === 'WLATIN1') {
-            let encoded = iconv.decode(Buffer.from(csvContent), 'CP-1252')
+            let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
             let blob = new Blob([encoded], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           } else {
-            let blob = new Blob([csvContent], { type: 'application/csv' })
+            let blob = new Blob([csvContentClean], { type: 'application/csv' })
             let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
             this.uploader.addToQueue([newCSVFile])
           }
@@ -2964,7 +2989,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
       await this.sasStoreService
         .callService(myParams, 'SASControlTable', 'editors/getdata', this.libds)
-        .then((res: EditorsGetdataServiceResponse) => {
+        .then((res: EditorsGetDataServiceResponse) => {
           this.initSetup(res)
         })
         .catch((err: any) => {
@@ -2976,7 +3001,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
   ngAfterViewInit() {}
 
-  initSetup(response: EditorsGetdataServiceResponse) {
+  initSetup(response: EditorsGetDataServiceResponse) {
     this.hotInstance = this.hotRegisterer.getInstance('hotInstance')
 
     if (this.getdataError) return
@@ -2986,6 +3011,20 @@ export class EditorComponent implements OnInit, AfterViewInit {
     this.cols = response.data.cols
     this.dsmeta = response.data.dsmeta
 
+    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
+    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
+    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')
+
+    if (notes && notes.VALUE) {
+      this.dsNote = notes.VALUE
+    } else if (longDesc && longDesc.VALUE) {
+      this.dsNote = longDesc.VALUE
+    } else if (shortDesc && shortDesc.VALUE) {
+      this.dsNote = shortDesc.VALUE
+    } else {
+      this.dsNote = ''
+    }
+
     const hot: Handsontable = this.hotInstance
 
     const approvers: Approver[] = response.data.approvers

client/src/app/home/home-routing.module.ts

@@ -0,0 +1,23 @@
+import { NgModule } from '@angular/core'
+import { RouterModule, Routes } from '@angular/router'
+import { HomeRouteComponent } from '../routes/home-route/home-route.component'
+import { HomeComponent } from './home.component'
+import { XLMapModule } from '../xlmap/xlmap.module'
+
+const routes: Routes = [
+  {
+    path: '',
+    component: HomeRouteComponent,
+    children: [
+      { path: '', pathMatch: 'full', redirectTo: 'tables' },
+      { path: 'tables', component: HomeComponent },
+      { path: 'files', loadChildren: () => XLMapModule }
+    ]
+  }
+]
+
+@NgModule({
+  imports: [RouterModule.forChild(routes)],
+  exports: [RouterModule]
+})
+export class HomeRoutingModule {}

client/src/app/home/home.module.ts

@@ -1,15 +1,18 @@
-import { NgModule } from '@angular/core'
 import { CommonModule } from '@angular/common'
-import { HomeComponent } from './home.component'
+import { NgModule } from '@angular/core'
-import { ClarityModule } from '@clr/angular'
 import { FormsModule } from '@angular/forms'
+import { ClarityModule } from '@clr/angular'
 import { AppSharedModule } from '../app-shared.module'
-import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
 import { DirectivesModule } from '../directives/directives.module'
+import { HomeRouteComponent } from '../routes/home-route/home-route.component'
+import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
+import { HomeRoutingModule } from './home-routing.module'
+import { HomeComponent } from './home.component'
 
 @NgModule({
-  declarations: [HomeComponent],
+  declarations: [HomeComponent, HomeRouteComponent],
   imports: [
+    HomeRoutingModule,
     FormsModule,
     ClarityModule,
     AppSharedModule,

client/src/app/lineage/lineage.component.ts

@@ -656,11 +656,10 @@ export class LineageComponent {
           this.flatdata = res.flatdata
 
           if (this.libraryList) {
-            let libraryToSelect = this.libraryList.find(
+            let libraryToSelect = this.libraryList.find((library: any) =>
-              (library: any) =>
+              res.info[0]?.LIBURI?.toUpperCase()?.includes(
-                res.info[0]?.LIBURI?.toUpperCase()?.includes(
+                library?.LIBRARYID?.toUpperCase()
-                  library?.LIBRARYID?.toUpperCase()
+              )
-                )
             )
 
             let tableToSelect: any

client/src/app/models/FilterQuery.ts

@@ -6,6 +6,7 @@ export interface FilterClause {
   operators: string[]
   type: string
   value: any
+  valueVariable: boolean
   values: { formatted: string; unformatted: any }[]
   variable: string
 }

client/src/app/models/sas/editors-getdata.model.ts

@@ -4,12 +4,12 @@ import { DQData, SASParam } from '../TableData'
 import { BaseSASResponse } from './common/BaseSASResponse'
 import { DataFormat } from './common/DateFormat'
 
-export interface EditorsGetdataServiceResponse {
+export interface EditorsGetDataServiceResponse {
-  data: EditorsGetdataSASResponse
+  data: EditorsGetDataSASResponse
   libds: string
 }
 
-export interface EditorsGetdataSASResponse extends BaseSASResponse {
+export interface EditorsGetDataSASResponse extends BaseSASResponse {
   $sasdata: $DataFormats
   sasdata: Sasdata[]
   sasparams: SASParam[]

client/src/app/query/query.component.html

@@ -413,7 +413,10 @@
 >
   <app-soft-select
     label="Value"
+    [secondLabel]="'Variable'"
+    [emitOnlySelected]="query.valueVariable"
     [inputId]="'vals_' + queryIndex + '_' + clauseIndex"
+    (selectedLabelChange)="selectedLabelChange($event, query)"
     [(value)]="query.value"
     [enableLoadMore]="query.nobs > query.values.length"
     (onInputEvent)="
@@ -423,9 +426,19 @@
       onAutocompleteLoadingMore($event, query.variable, queryIndex, clauseIndex)
     "
   >
-    <option [value]="column.unformatted" *ngFor="let column of query.values">
+    <div *ngIf="!query.valueVariable">
-      {{ column.formatted.trim() }}
+      <option [value]="column.unformatted" *ngFor="let column of query.values">
-    </option>
+        {{ column.formatted.trim() }}
+      </option>
+    </div>
+
+    <div *ngIf="query.valueVariable">
+      <ng-container *ngFor="let column of cols">
+        <option [value]="column.NAME" *ngIf="column.TYPE === query.type">
+          {{ column.NAME }}
+        </option>
+      </ng-container>
+    </div>
   </app-soft-select>
 </ng-template>
 

client/src/app/query/query.component.ts

@@ -95,6 +95,7 @@ export class QueryComponent
         variable: null,
         operator: null,
         value: null,
+        valueVariable: false,
         startrow: 0,
         rows: 0,
         nobs: 0,
@@ -193,6 +194,20 @@ export class QueryComponent
    */
   usePickersChange() {
     this.queryDateTime = []
+    if (this.usePickers) {
+      this.clauses.queryObj.forEach((queryObj: any) => {
+        queryObj.elements.forEach((element: any) => {
+          const isDateOrTime = ['DATETIME', 'TIME', 'DATE'].includes(
+            element.ddtype
+          )
+
+          if (isDateOrTime && element.valueVariable) {
+            element.value = ''
+            element.valueVariable = false
+          }
+        })
+      })
+    }
   }
 
   /**
@@ -253,8 +268,6 @@ export class QueryComponent
       get(globals, objPath).filter.libds = this.libds
     }
     get(globals, objPath).filter.clauses = this.clauses
-
-    console.log('globals', globals)
   }
 
   /**
@@ -750,6 +763,12 @@ export class QueryComponent
     )
   }
 
+  public selectedLabelChange(label: string, query: any) {
+    query.valueVariable = label === 'Variable'
+    query.value = ''
+    this.whereClauseFn()
+  }
+
   public variableInputChange(
     queryVariable: any,
     index: number,

client/src/app/routes/home-route/home-route.component.html

@@ -0,0 +1 @@
+<router-outlet></router-outlet>

client/src/app/routes/home-route/home-route.component.ts

@@ -0,0 +1,17 @@
+import { Component, OnInit, OnDestroy } from '@angular/core'
+
+@Component({
+  selector: 'app-home-route',
+  templateUrl: './home-route.component.html',
+  styleUrls: ['./home-route.component.scss'],
+  host: {
+    class: 'content-container'
+  }
+})
+export class HomeRouteComponent implements OnInit, OnDestroy {
+  constructor() {}
+
+  ngOnInit() {}
+
+  ngOnDestroy() {}
+}

client/src/app/routes/xlmap-route/xlmap-route.component.html

@@ -0,0 +1 @@
+<router-outlet></router-outlet>

client/src/app/routes/xlmap-route/xlmap-route.component.ts

@@ -0,0 +1,17 @@
+import { Component, OnInit, OnDestroy } from '@angular/core'
+
+@Component({
+  selector: 'app-xlmap-route',
+  templateUrl: './xlmap-route.component.html',
+  styleUrls: ['./xlmap-route.component.scss'],
+  host: {
+    class: 'content-container'
+  }
+})
+export class XLMapRouteComponent implements OnInit, OnDestroy {
+  constructor() {}
+
+  ngOnInit() {}
+
+  ngOnDestroy() {}
+}

client/src/app/services/app.service.ts

@@ -74,6 +74,7 @@ export class AppService {
           missingProps.push('Globvars')
         if (!res.sasdatasets) missingProps.push('Sasdatasets')
         if (!res.saslibs) missingProps.push('Saslibs')
+        if (!res.xlmaps) missingProps.push('XLMaps')
 
         if (missingProps.length > 0) {
           startupServiceError = true
@@ -135,10 +136,17 @@ export class AppService {
           globals.editor.libsAndTables = libsAndTables
         }
 
+        globals.xlmaps = res.xlmaps.map((xlmap: any) => ({
+          id: xlmap[0],
+          description: xlmap[1],
+          targetDS: xlmap[2]
+        }))
         globals.editor.treeNodeLibraries = treeNodeLibraries
         globals.editor.libraries = libraries
         globals.editor.startupSet = true
 
+        globals.dcLib = res.globvars[0].DCLIB
+
         await this.licenceService.activation(res)
       })
       .catch((err: any) => {

client/src/app/services/sas-store.service.ts

@@ -10,8 +10,8 @@ import { globals } from '../_globals'
 import { FilterClause, FilterGroup, FilterQuery } from '../models/FilterQuery'
 import {
   $DataFormats,
-  EditorsGetdataSASResponse,
+  EditorsGetDataSASResponse,
-  EditorsGetdataServiceResponse
+  EditorsGetDataServiceResponse
 } from '../models/sas/editors-getdata.model'
 import { LoggerService } from './logger.service'
 import { isSpecialMissing } from '@sasjs/utils/input/validators'
@@ -57,13 +57,13 @@ export class SasStoreService {
     libds: string
   ) {
     this.libds = libds
-    let tables: any = {}
+    const tables: any = {}
     tables[tableName] = [tableData]
-    let res: EditorsGetdataSASResponse = await this.sasService.request(
+    const res: EditorsGetDataSASResponse = await this.sasService.request(
       program,
       tables
     )
-    let response: EditorsGetdataServiceResponse = {
+    const response: EditorsGetDataServiceResponse = {
       data: res,
       libds: this.libds
     }
@@ -209,6 +209,14 @@ export class SasStoreService {
     return res
   }
 
+  public async getXLMapRules(id: string) {
+    const tables = {
+      getxlmaps_in: [{ XLMAP_ID: id }]
+    }
+    const res: any = await this.sasService.request('editors/getxlmaps', tables)
+    return res
+  }
+
   public async getDetails(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]
@@ -408,14 +416,18 @@ export class SasStoreService {
     for (let index = 0; index < clauses.queryObj.length; index++) {
       let string = ''
       let clause = clauses.queryObj[index]
+
       for (let ind = 0; ind < clause.elements.length; ind++) {
         let query = clause.elements[ind]
+
         if (ind < clause.elements.length - 1) {
           opr = clause.clauseLogic
         } else {
           opr = ''
         }
+
         let val: any
+
         for (let k = 0; k < query.values.length; k++) {
           if (
             typeof query.value === 'string' &&
@@ -486,6 +498,8 @@ export class SasStoreService {
         }
 
         let type = query.type
+        //if the value is variable, omit quotes in the 'where' string
+        const isValueVariable = query.valueVariable
         let variable = query.variable === null ? '' : query.variable
         let oper = query.operator === null ? '' : query.operator
         // let value = val === null ? "''" : val;
@@ -499,10 +513,14 @@ export class SasStoreService {
         if (type === 'char' && oper !== 'IN' && oper !== 'NOT IN') {
           if (typeof value === 'undefined') {
             value = ''
-            value = " '" + value + "' "
-          } else {
-            value = " '" + value + "' "
           }
+
+          if (isValueVariable) {
+            value = ' ' + value + ' ' //without quotes, with spaces
+          } else {
+            value = " '" + value + "' " //with quotes and spaces
+          }
+
           string = string + ' ' + variable + ' ' + oper + value + opr
         } else {
           if (type === 'num' && typeof value === 'undefined') {
@@ -596,7 +614,7 @@ export class SasStoreService {
               rawValue = '.'
             }
           } else {
-            if (filterClause.type === 'char') {
+            if (filterClause.type === 'char' && !filterClause.valueVariable) {
               rawValue = `'${filterClause.value.replace(/'/g, "''")}'`
             }
           }

client/src/app/shared/sidebar/sidebar.component.html

@@ -107,7 +107,29 @@
       </clr-tab-content>
     </clr-tab>
   </clr-tabs>
-  <p *ngIf="isMainRoute('home')" class="page-title">Edit</p>
+
+  <div
+    *ngIf="isMainRoute('home')"
+    class="d-flex justify-content-center sub-dropdown"
+  >
+    <clr-dropdown>
+      <button class="dropdown-toggle btn btn-link" clrDropdownTrigger>
+        {{ getSubPage() }}
+        <clr-icon shape="caret down"></clr-icon>
+      </button>
+      <clr-dropdown-menu *clrIfOpen>
+        <a
+          clrVerticalNavLink
+          routerLink="/home/tables"
+          routerLinkActive="active"
+          >Tables</a
+        >
+        <a clrVerticalNavLink routerLink="/home/files" routerLinkActive="active"
+          >Files</a
+        >
+      </clr-dropdown-menu>
+    </clr-dropdown>
+  </div>
 
   <div class="nav-divider"></div>
 

client/src/app/shared/soft-select/soft-select.component.html

@@ -1,4 +1,22 @@
-<label *ngIf="label" class="clr-control-label">{{ label }}</label>
+<label
+  *ngIf="label"
+  [class.secondLabelActive]="secondLabel && secondLabel.length > 0"
+  class="clr-control-label"
+>
+  <span
+    (click)="onChangeLabel('first')"
+    [class.value-type-selected]="labelSelected === 'first'"
+    >{{ label }}</span
+  >
+  <ng-container *ngIf="secondLabel">
+    /
+    <span
+      (click)="onChangeLabel('second')"
+      [class.value-type-selected]="labelSelected === 'second'"
+      >{{ secondLabel }}</span
+    >
+  </ng-container>
+</label>
 <ng-container [ngSwitch]="type">
   <ng-container *ngSwitchCase="'date'">
     <clr-date-container>

client/src/app/shared/soft-select/soft-select.component.scss

@@ -28,4 +28,12 @@ clr-date-container {
             margin-top: -5px;
         }
     }
+}
+
+label.secondLabelActive span {
+    &:not(.value-type-selected) {
+        text-decoration: line-through;
+        cursor: pointer;
+        opacity: 0.6;
+    }
 }

client/src/app/shared/soft-select/soft-select.component.ts

@@ -18,6 +18,7 @@ import { OnLoadingMoreEvent } from '../autocomplete/autocomplete.component'
 export class SoftSelectComponent implements OnInit, OnChanges {
   @Input() inputId: string = ''
   @Input() label: string | undefined
+  @Input() secondLabel: string | undefined
   @Input() value: Date | string | null = ''
   @Input() disabled: boolean = false
   @Input() type: string = 'text'
@@ -30,20 +31,24 @@ export class SoftSelectComponent implements OnInit, OnChanges {
   @Output() focusinInput: EventEmitter<any> = new EventEmitter()
   @Output() onAutocompleteLoadingMore: EventEmitter<OnLoadingMoreEvent> =
     new EventEmitter()
+  @Output() selectedLabelChange: EventEmitter<string> = new EventEmitter()
 
   @ViewChild('input') inputElement: any
 
   temp: Date | string | null = ''
   inputFocused: boolean = false
 
+  labelSelected: LabelTypes = 'first'
+
   constructor() {}
 
   ngOnChanges(changes: SimpleChanges): void {
     if (
       changes.value &&
       changes.value.currentValue !== changes.value.previousValue
-    )
+    ) {
       this.valueChange.emit(changes.value.currentValue)
+    }
   }
 
   ngOnInit(): void {}
@@ -85,4 +90,14 @@ export class SoftSelectComponent implements OnInit, OnChanges {
   onFocusinInput(event: any) {
     this.focusinInput.emit(event)
   }
+
+  onChangeLabel(label: LabelTypes) {
+    this.labelSelected = label
+
+    const selectedLabelText = label === 'first' ? this.label : this.secondLabel
+
+    this.selectedLabelChange.emit(selectedLabelText)
+  }
 }
+
+export type LabelTypes = 'first' | 'second'

client/src/app/stage/stage.component.ts

@@ -7,6 +7,7 @@ import { EventService } from '../services/event.service'
 import { AppService } from '../services/app.service'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { LicenceService } from '../services/licence.service'
+import { globals } from '../_globals'
 
 @Component({
   selector: 'app-stage',
@@ -55,7 +56,15 @@ export class StageComponent implements OnInit {
   }
 
   public goBack() {
-    this.route.navigateByUrl('/editor/' + this.tableDetails.BASE_TABLE)
+    const xlmap = globals.xlmaps.find(
+      (xlmap) => xlmap.targetDS === this.tableDetails.BASE_TABLE
+    )
+    if (xlmap) {
+      const id = this.hotTable.data[0].XLMAP_ID
+      this.route.navigateByUrl('/home/files/' + id)
+    } else {
+      this.route.navigateByUrl('/editor/' + this.tableDetails.BASE_TABLE)
+    }
   }
 
   public download(id: any) {

client/src/app/viewer/viewer.component.html

@@ -358,36 +358,49 @@
       </section>
 
       <div class="title-col clr-col-auto clr-flex-column clr-flex-sm-row">
-        <clr-icon
-          (click)="datasetInfo = true"
-          shape="info-circle"
-          class="is-highlight cursor-pointer"
-          size="24"
-        ></clr-icon>
-
-        <clr-icon
-          *ngIf="tableTitle?.includes('-FC')"
-          shape="bolt"
-          class="color-yellow mt-5 mr-5"
-        ></clr-icon>
-
         <h3
-          *ngIf="tableTitle && tableTitle.length > 0"
+          class="viewerTitle clr-flex-column d-flex clr-flex-sm-row clr-align-items-center clr-justify-content-center"
-          class="viewerTitle clr-flex-column d-flex clr-flex-sm-row clr-align-items-center"
         >
-          {{ tableTitle?.replace('-FC', '') }}
+          <clr-tooltip class="d-flex">
+            <clr-icon
+              clrTooltipTrigger
+              (click)="datasetInfo = true"
+              shape="info-circle"
+              class="is-highlight cursor-pointer"
+              size="24"
+            ></clr-icon>
 
-          <span *ngIf="numberOfRows !== null">
+            <clr-icon
-            ({{ numberOfRows | thousandSeparator: ',' }}
+              *ngIf="tableTitle?.includes('-FC')"
-            {{ numberOfRows! === 1 ? 'row' : 'rows' }}, {{ filterCols.length
+              shape="bolt"
-            }}{{ filterCols.length === 1 ? ' col' : ' cols' }})
+              class="color-yellow mt-5 mr-5"
-          </span>
+            ></clr-icon>
 
-          <clr-icon
+            <span clrTooltipTrigger *ngIf="tableTitle && tableTitle.length > 0">
-            (click)="reloadTableData()"
+              {{ tableTitle?.replace('-FC', '') }}
-            class="refresh-table"
+            </span>
-            shape="refresh"
+            <clr-tooltip-content
-          ></clr-icon>
+              clrPosition="bottom-left"
+              clrSize="lg"
+              *clrIfOpen
+            >
+              {{ this.dsNote }}
+            </clr-tooltip-content>
+          </clr-tooltip>
+
+          <ng-container *ngIf="tableTitle && tableTitle.length > 0">
+            <span *ngIf="numberOfRows !== null">
+              ({{ numberOfRows | thousandSeparator: ',' }}
+              {{ numberOfRows! === 1 ? 'row' : 'rows' }}, {{ filterCols.length
+              }}{{ filterCols.length === 1 ? ' col' : ' cols' }})
+            </span>
+
+            <clr-icon
+              (click)="reloadTableData()"
+              class="refresh-table"
+              shape="refresh"
+            ></clr-icon>
+          </ng-container>
         </h3>
       </div>
 

client/src/app/viewer/viewer.component.ts

@@ -95,6 +95,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
   public $dataFormats: $DataFormats | null = null
   public datasetInfo: boolean = false
   public dsmeta: DSMeta[] = []
+  public dsNote = ''
 
   public licenceState = this.licenceService.licenceState
   public Infinity = Infinity
@@ -246,6 +247,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
         this.hotTable.data = res.viewdata
         this.$dataFormats = res.$viewdata
         this.dsmeta = res.dsmeta
+        this.setDSNote()
         this.numberOfRows = res.sasparams[0].NOBS
         this.queryText = res.sasparams[0].FILTER_TEXT
         this.headerPks = res.sasparams[0].PK_FIELDS.split(' ')
@@ -803,6 +805,7 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
           this.hotTable.data = res.viewdata
           this.$dataFormats = res.$viewdata
           this.dsmeta = res.dsmeta
+          this.setDSNote()
           this.queryText = res.sasparams[0].FILTER_TEXT
           let columns: any[] = []
           let colArr = []
@@ -1016,6 +1019,22 @@ export class ViewerComponent implements AfterContentInit, AfterViewInit {
     this.sasStoreService.removeClause()
   }
 
+  private setDSNote() {
+    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
+    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
+    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')
+
+    if (notes && notes.VALUE) {
+      this.dsNote = notes.VALUE
+    } else if (longDesc && longDesc.VALUE) {
+      this.dsNote = longDesc.VALUE
+    } else if (shortDesc && shortDesc.VALUE) {
+      this.dsNote = shortDesc.VALUE
+    } else {
+      this.dsNote = ''
+    }
+  }
+
   private setupHot() {
     setTimeout(() => {
       if (!this.loadingTableView && this.libDataset) {

client/src/app/xlmap/tests/xl.utils.spec.ts

@@ -0,0 +1,159 @@
+import {
+  extractRowAndCol,
+  getCellAddress,
+  getFinishingCell,
+  isBlankRow
+} from '../utils/xl.utils'
+
+describe('isBlankRow', () => {
+  it('should return true for a blank row', () => {
+    const blankRow = { __rowNum__: 1 }
+    expect(isBlankRow(blankRow)).toBeTrue()
+  })
+
+  it('should return false for a non-blank row', () => {
+    const nonBlankRow = {
+      B: 3,
+      C: 'some value',
+      D: -203
+    }
+    expect(isBlankRow(nonBlankRow)).toBeFalse()
+  })
+})
+
+describe('extractRowAndCol', () => {
+  it('should extract row and column from "MATCH F R[2]C[0]: CASH BALANCE"', () => {
+    const input = 'MATCH F R[2]C[0]: CASH BALANCE'
+    const result = extractRowAndCol(input)
+    expect(result).toEqual({ row: 2, column: 0 })
+  })
+
+  it('should extract row and column from "RELATIVE R[10]C[6]"', () => {
+    const input = 'RELATIVE R[10]C[6]'
+    const result = extractRowAndCol(input)
+    expect(result).toEqual({ row: 10, column: 6 })
+  })
+
+  it('should return null for invalid input', () => {
+    const invalidInput = 'INVALID INPUT'
+    const result = extractRowAndCol(invalidInput)
+    expect(result).toBeNull()
+  })
+})
+
+describe('getCellAddress', () => {
+  const arrayOfObjects = [
+    { A: 'valueA1', B: 'valueB1' },
+    { A: 'valueA2', B: 'valueB2' }
+  ]
+
+  it('should convert "ABSOLUTE D8" to A1-style address', () => {
+    const input = 'ABSOLUTE D8'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('D8')
+  })
+
+  it('should convert "RELATIVE R[10]C[6]" to A1-style address', () => {
+    const input = 'RELATIVE R[10]C[6]'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('F10')
+  })
+
+  it('should convert "MATCH 1 R[0]C[0]:valueA1" to A1-style address', () => {
+    const input = 'MATCH 1 R[0]C[0]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('A1')
+  })
+
+  it('should convert "MATCH A R[0]C[0]:valueA1" to A1-style address', () => {
+    const input = 'MATCH A R[0]C[0]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('A1')
+  })
+
+  it('should convert "MATCH 1 R[1]C[0]:valueA1" to A1-style address', () => {
+    const input = 'MATCH 1 R[1]C[0]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('A2')
+  })
+
+  it('should convert "MATCH A R[0]C[1]:valueA1" to A1-style address', () => {
+    const input = 'MATCH A R[0]C[1]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('B1')
+  })
+
+  it('should convert "MATCH 1 R[1]C[1]:valueA1" to A1-style address', () => {
+    const input = 'MATCH 1 R[1]C[1]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('B2')
+  })
+
+  it('should convert "MATCH A R[1]C[1]:valueA1" to A1-style address', () => {
+    const input = 'MATCH A R[1]C[1]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('B2')
+  })
+})
+
+describe('getFinishingCell', () => {
+  const arrayOfObjects = [
+    { A: 'valueA1', B: 'valueB1' },
+    { A: 'valueA2', B: 'valueB2' },
+    { A: 'valueA3', B: 'valueB3' },
+    { B: 'valueB4' },
+    { A: 'valueA5' },
+    { A: 'valueA6', B: 'valueB6' },
+    {},
+    { A: 'valueA8' }
+  ]
+
+  it('should return the start cell if finish is an empty string', () => {
+    const start = 'A1'
+    const finish = ''
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe(start)
+  })
+
+  it('should convert "ABSOLUTE D8" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'ABSOLUTE D8'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('D8')
+  })
+
+  it('should convert "RELATIVE R[2]C[1]" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'RELATIVE R[2]C[1]'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B3')
+  })
+
+  it('should convert "MATCH A R[0]C[1]:valueA1" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'MATCH A R[0]C[1]:valueA1'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B1')
+  })
+
+  it('should convert "MATCH 1 R[4]C[0]:valueB1" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'MATCH 1 R[4]C[0]:valueB1'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B5')
+  })
+
+  it('should convert "LASTDOWN" to A1-style address of the last non-blank cell in column A', () => {
+    const start = 'A1'
+    const finish = 'LASTDOWN'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('A3')
+  })
+
+  it('should convert "BLANKROW" to A1-style address of the last row with blank cells', () => {
+    const start = 'A1'
+    const finish = 'BLANKROW'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B6')
+  })
+})

client/src/app/xlmap/utils/file.utils.ts

@@ -0,0 +1,31 @@
+export const blobToFile = (blob: Blob, fileName: string): File => {
+  const file = new File([blob], fileName, {
+    lastModified: new Date().getTime()
+  })
+  return file
+}
+
+/**
+ * Convert an array of bytes (Uint8Array) to a binary string.
+ * @param {Uint8Array} res - The array of bytes to convert.
+ * @returns {string} The binary string representation of the array of bytes.
+ */
+export const byteArrayToBinaryString = (res: Uint8Array): string => {
+  // Create a Uint8Array from the input array (if it's not already)
+  const bytes = new Uint8Array(res)
+
+  // Initialize an empty string to store the binary representation
+  let binary = ''
+
+  // Get the length of the byte array
+  const length = bytes.byteLength
+
+  // Iterate through each byte in the array
+  for (let i = 0; i < length; i++) {
+    // Convert each byte to its binary representation and append to the string
+    binary += String.fromCharCode(bytes[i])
+  }
+
+  // Return the binary string
+  return binary
+}

client/src/app/xlmap/utils/xl.utils.ts

@@ -0,0 +1,225 @@
+import * as XLSX from '@sheet/crypto'
+
+/**
+ * Checks if an excel row is blank or not
+ *
+ * @param row object is of shape {[key: string]: any}
+ */
+export const isBlankRow = (row: any) => {
+  for (const key in row) {
+    if (key !== '__rowNum__') {
+      return false
+    }
+  }
+  return true
+}
+
+/**
+ * Extracts row and column number from xlmap rule.
+ *
+ * Input string should be in form of
+ * either "MATCH F R[2]C[0]: CASH BALANCE" or "RELATIVE R[10]C[6]"
+ */
+export const extractRowAndCol = (str: string) => {
+  // Regular expression to match and capture the values inside square brackets
+  const regex = /R\[(\d+)\]C\[(\d+)\]/
+
+  // Match the regular expression against the input string
+  const match = str.match(regex)
+
+  if (!match) {
+    return null
+  }
+
+  // Extract values from the match groups
+  const row = parseInt(match[1], 10)
+  const column = parseInt(match[2], 10)
+
+  return {
+    row,
+    column
+  }
+}
+
+/**
+ * Generate an A1-Style excel cell address from xlmap rule.
+ *
+ * Expect "ABSOLUTE D8" or "RELATIVE R[10]C[6]" or
+ * "MATCH C R[0]C[4]:Common Equity Tier 1 (CET1)" kinds of string as rule input
+ */
+export const getCellAddress = (rule: string, arrayOfObjects: any[]) => {
+  if (rule.startsWith('ABSOLUTE ')) {
+    rule = rule.replace('ABSOLUTE ', '')
+  }
+
+  if (rule.startsWith('RELATIVE ')) {
+    const rowAndCol = extractRowAndCol(rule)
+
+    if (rowAndCol) {
+      const { row, column } = rowAndCol
+
+      // Generate an A1-Style address string from a SheetJS cell address
+      // Spreadsheet applications typically display ordinal row numbers,
+      // where 1 is the first row, 2 is the second row, etc. The numbering starts at 1.
+      // SheetJS follows JavaScript counting conventions,
+      // where 0 is the first row, 1 is the second row, etc. The numbering starts at 0.
+      // Therefore, we have to subtract 1 from row and column to match SheetJS indexing convention
+      rule = XLSX.utils.encode_cell({ r: row - 1, c: column - 1 })
+    }
+  }
+
+  if (rule.startsWith('MATCH ')) {
+    let targetValue = ''
+
+    // using a regular expression to match "C[x]:" and extract the value after it
+    const match = rule.match(/C\[\d+\]:(.+)/)
+
+    // Check if there is a match
+    if (match) {
+      // Extract the value after "C[x]:"
+      targetValue = match[1]
+    }
+
+    // Split the string by spaces to get target row/column
+    const splittedArray = rule.split(' ')
+
+    // Extract the second word
+    const secondWord = splittedArray[1]
+
+    let targetColumn = ''
+    let targetRow = -1
+    let cellAddress = ''
+
+    // Check if the secondWord is a number
+    if (!isNaN(Number(secondWord))) {
+      targetRow = parseInt(secondWord)
+    } else {
+      targetColumn = secondWord
+    }
+
+    if (targetRow !== -1) {
+      // sheetJS index starts from 0,
+      // therefore, decremented 1 to make it correct row address for js array
+      const row = arrayOfObjects[targetRow - 1]
+      for (const col in row) {
+        if (col !== '__rowNum__' && row[col] === targetValue) {
+          cellAddress = col + targetRow
+          break
+        }
+      }
+    } else {
+      for (let i = 0; i < arrayOfObjects.length; i++) {
+        const row = arrayOfObjects[i]
+        if (row[targetColumn] === targetValue) {
+          // sheetJS index starts from 0,
+          // therefore, incremented 1 to make it correct row address
+          const rowIndex = i + 1
+          cellAddress = targetColumn + rowIndex
+          break
+        }
+      }
+    }
+
+    // Converts A1 cell address to 0-indexed form
+    const matchedCellAddress = XLSX.utils.decode_cell(cellAddress)
+
+    // extract number of rows and columns that we have to move
+    // from matched cell to reach target cell
+    const rowAndCol = extractRowAndCol(rule)
+
+    if (rowAndCol) {
+      const { row, column } = rowAndCol
+
+      // Converts 0-indexed cell address to A1 form
+      rule = XLSX.utils.encode_cell({
+        r: matchedCellAddress.r + row,
+        c: matchedCellAddress.c + column
+      })
+    }
+  }
+
+  return rule
+}
+
+/**
+ * Generate an A1-Style excel cell address for last cell
+ *
+ * @param start A1 style excel cell address
+ * @param finish XLMAP_FINISH attribute of xlmap rule
+ * @param arrayOfObjects an array of row objects
+ * @returns
+ */
+export const getFinishingCell = (
+  start: string,
+  finish: string,
+  arrayOfObjects: any[]
+) => {
+  // in this case an individual cell would be extracted
+  if (finish === '') {
+    return start
+  }
+
+  if (finish.startsWith('ABSOLUTE ')) {
+    finish = finish.replace('ABSOLUTE ', '')
+  }
+
+  if (finish.startsWith('RELATIVE ')) {
+    const rowAndCol = extractRowAndCol(finish)
+    if (rowAndCol) {
+      const { row, column } = rowAndCol
+
+      const { r, c } = XLSX.utils.decode_cell(start)
+
+      // finish is relative to starting point
+      // therefore, we need to add extracted row and columns
+      // in starting cell address to get actual finishing cell
+      finish = XLSX.utils.encode_cell({ r: r + row, c: c + column })
+    }
+  }
+
+  if (finish.startsWith('MATCH ')) {
+    finish = getCellAddress(finish, arrayOfObjects)
+  }
+
+  if (finish === 'LASTDOWN') {
+    const { r, c } = XLSX.utils.decode_cell(start)
+    const colName = XLSX.utils.encode_col(c)
+    let lastNonBlank = r
+    for (let i = r + 1; i < arrayOfObjects.length; i++) {
+      const row = arrayOfObjects[i]
+      if (!row[colName]) {
+        break
+      }
+      lastNonBlank = i
+    }
+    finish = colName + (lastNonBlank + 1) // excel numbering starts from 1. So incremented 1 to 0 based index
+  }
+
+  if (finish === 'BLANKROW') {
+    const { r } = XLSX.utils.decode_cell(start)
+    let lastNonBlankRow = r
+    for (let i = r + 1; i < arrayOfObjects.length; i++) {
+      const row = arrayOfObjects[i]
+      if (isBlankRow(row)) {
+        break
+      }
+      lastNonBlankRow = i
+    }
+    const row = arrayOfObjects[lastNonBlankRow]
+
+    // Get the keys of the object (excluding '__rowNum__')
+    const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
+
+    // Finding last column in a row
+    // Find the key with the highest alphanumeric value (assumes keys are letters)
+    const lastColumn = keys.reduce(
+      (maxKey, currentKey) => (currentKey > maxKey ? currentKey : maxKey),
+      ''
+    )
+
+    // make finishing cell address in A1 style
+    finish = lastColumn + (lastNonBlankRow + 1) // excel numbering starts from 1. So incremented 1 to 0 based index
+  }
+
+  return finish
+}

client/src/app/xlmap/xlmap-routing.module.ts

@@ -0,0 +1,22 @@
+import { NgModule } from '@angular/core'
+import { RouterModule, Routes } from '@angular/router'
+
+import { XLMapComponent } from '../xlmap/xlmap.component'
+import { XLMapRouteComponent } from '../routes/xlmap-route/xlmap-route.component'
+
+const routes: Routes = [
+  {
+    path: '',
+    component: XLMapRouteComponent,
+    children: [
+      { path: '', component: XLMapComponent },
+      { path: ':id', component: XLMapComponent }
+    ]
+  }
+]
+
+@NgModule({
+  imports: [RouterModule.forChild(routes)],
+  exports: [RouterModule]
+})
+export class XLMapRoutingModule {}

client/src/app/xlmap/xlmap.component.html

@@ -0,0 +1,252 @@
+<app-sidebar>
+  <div *ngIf="xlmapsLoading" class="my-10-mx-auto text-center">
+    <clr-spinner clrMedium></clr-spinner>
+  </div>
+
+  <clr-tree>
+    <clr-tree-node class="search-node">
+      <div class="tree-search-wrapper">
+        <input
+          clrInput
+          #searchXLMapTreeInput
+          placeholder="Filter by Id"
+          name="input"
+          [(ngModel)]="searchString"
+          (keyup)="xlmapListOnFilter()"
+          autocomplete="off"
+        />
+        <clr-icon
+          *ngIf="searchXLMapTreeInput.value.length < 1"
+          shape="search"
+        ></clr-icon>
+        <clr-icon
+          *ngIf="searchXLMapTreeInput.value.length > 0"
+          (click)="searchString = ''; xlmapListOnFilter()"
+          shape="times"
+        ></clr-icon>
+      </div>
+    </clr-tree-node>
+
+    <ng-container *ngFor="let xlmap of xlmaps">
+      <clr-tree-node>
+        <button
+          (click)="xlmapOnClick(xlmap)"
+          class="clr-treenode-link"
+          [class.table-active]="isActiveXLMap(xlmap.id)"
+        >
+          <clr-icon shape="file"></clr-icon>
+          {{ xlmap.id }}
+        </button>
+      </clr-tree-node>
+    </ng-container>
+  </clr-tree>
+</app-sidebar>
+
+<div class="content-area">
+  <div *ngIf="!selectedXLMap" class="no-table-selected">
+    <clr-icon
+      shape="warning-standard"
+      size="60"
+      class="is-info icon-dc-fill"
+    ></clr-icon>
+    <h3 *ngIf="xlmaps.length > 0" class="text-center color-gray">
+      Please select a map
+    </h3>
+    <h3 *ngIf="xlmaps.length < 1" class="text-center color-gray">
+      No excel map is found
+    </h3>
+  </div>
+
+  <div class="loadingSpinner" *ngIf="isLoading">
+    <span class="spinner"> Loading... </span>
+    <div>
+      <h4>{{ isLoadingDesc }}</h4>
+    </div>
+  </div>
+
+  <div
+    appDragNdrop
+    (fileDraggedOver)="onShowUploadModal()"
+    class="card h-100 d-flex clr-flex-column"
+    *ngIf="!isLoading && selectedXLMap"
+  >
+    <clr-tabs>
+      <clr-tab>
+        <button clrTabLink (click)="selectedTab = TabsEnum.Rules">Rules</button>
+        <clr-tab-content *clrIfActive="selectedTab === TabsEnum.Rules">
+        </clr-tab-content>
+      </clr-tab>
+      <clr-tab>
+        <button clrTabLink (click)="selectedTab = TabsEnum.Data">Data</button>
+        <clr-tab-content *clrIfActive="selectedTab === TabsEnum.Data">
+        </clr-tab-content>
+      </clr-tab>
+    </clr-tabs>
+
+    <ng-container *ngTemplateOutlet="actionButtons"></ng-container>
+
+    <div class="clr-row m-0 mb-10-i viewerTitle">
+      <h3 class="d-flex clr-col-12 clr-justify-content-center mt-5-i">
+        {{ selectedXLMap.id }}
+      </h3>
+      <i class="d-flex clr-col-12 clr-justify-content-center mt-5-i">{{
+        selectedXLMap.description
+      }}</i>
+      <h5 class="d-flex clr-col-12 clr-justify-content-center mt-5-i">
+        Rules Source:
+        <a class="ml-10" [routerLink]="'/view/data/' + rulesSource">
+          {{ rulesSource }}
+        </a>
+      </h5>
+      <h5 class="d-flex clr-col-12 clr-justify-content-center mt-5-i">
+        Target dataset:
+        <a class="ml-10" [routerLink]="'/view/data/' + selectedXLMap.targetDS">
+          {{ selectedXLMap.targetDS }}
+        </a>
+      </h5>
+    </div>
+
+    <div class="clr-flex-1">
+      <hot-table
+        hotId="hotInstance"
+        id="hot-table"
+        [multiColumnSorting]="true"
+        [viewportRowRenderingOffset]="50"
+        [data]="selectedTab === TabsEnum.Rules ? xlmapRules : xlData"
+        [colHeaders]="
+          selectedTab === TabsEnum.Rules ? xlmapRulesHeaders : xlUploadHeader
+        "
+        [columns]="
+          selectedTab === TabsEnum.Rules ? xlmapRulesColumns : xlUploadColumns
+        "
+        [filters]="true"
+        [height]="'100%'"
+        stretchH="all"
+        [modifyColWidth]="maxWidthChecker"
+        [cells]="getCellConfiguration"
+        [maxRows]="hotTableMaxRows"
+        [manualColumnResize]="true"
+        [rowHeaders]="rowHeaders"
+        [rowHeaderWidth]="15"
+        [rowHeights]="20"
+        [licenseKey]="hotTableLicenseKey"
+      >
+      </hot-table>
+    </div>
+  </div>
+
+  <clr-modal
+    appFileDrop
+    (fileOver)="fileOverBase($event)"
+    (fileDrop)="getFileDesc($event, true)"
+    [uploader]="uploader"
+    [clrModalSize]="'xl'"
+    [clrModalStaticBackdrop]="false"
+    [clrModalClosable]="true"
+    [(clrModalOpen)]="showUploadModal"
+    class="relative"
+  >
+    <h3 class="modal-title">Upload File</h3>
+    <div class="modal-body">
+      <div class="drop-area">
+        <span>Drop file anywhere to upload!</span>
+      </div>
+
+      <div class="clr-col-md-12">
+        <div class="clr-row card-block mt-15 d-flex justify-content-between">
+          <div class="clr-col-md-3 filterBtn">
+            <span class="filterBtn w-100">
+              <label
+                for="file-upload"
+                class="btn btn-sm btn-outline profile-buttons w-100"
+              >
+                Browse
+              </label>
+            </span>
+            <input
+              hidden
+              #fileUploadInput
+              id="file-upload"
+              type="file"
+              appFileSelect
+              [uploader]="uploader"
+              (change)="getFileDesc($event)"
+            />
+          </div>
+        </div>
+      </div>
+    </div>
+  </clr-modal>
+
+  <clr-modal [(clrModalOpen)]="submitLimitNotice">
+    <h3 class="modal-title">Notice</h3>
+    <div class="modal-body">
+      <p class="m-0">
+        Due to current licence, only
+        {{ licenceState.value.submit_rows_limit }} rows in a file will be
+        submitted. To remove the restriction, contact
+        support&#64;datacontroller.io
+      </p>
+    </div>
+    <div class="modal-footer">
+      <button
+        type="button"
+        class="btn btn-sm btn-primary"
+        (click)="submitLimitNotice = false"
+      >
+        Cancel
+      </button>
+      <button
+        type="button"
+        class="btn btn-sm btn-primary"
+        (click)="submit(); submitLimitNotice = false"
+      >
+        Submit
+      </button>
+    </div>
+  </clr-modal>
+</div>
+
+<ng-template #actionButtons>
+  <div class="clr-row m-0 clr-justify-content-center">
+    <div
+      *ngIf="status === StatusEnum.ReadyToUpload"
+      class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
+    >
+      <button
+        type="button"
+        class="btn btn-sm btn-success btn-block mr-0"
+        (click)="onShowUploadModal()"
+      >
+        <clr-icon shape="upload"></clr-icon>
+        <span>Upload</span>
+      </button>
+    </div>
+    <div
+      *ngIf="status === StatusEnum.ReadyToSubmit"
+      class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
+    >
+      <button
+        type="button"
+        class="btn btn-sm btn-success btn-block mr-0"
+        (click)="submitExcel()"
+      >
+        <clr-icon shape="upload"></clr-icon>
+        <span>Submit</span>
+      </button>
+    </div>
+    <div
+      *ngIf="status === StatusEnum.ReadyToSubmit"
+      class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
+    >
+      <button
+        type="button"
+        class="btn btn-sm btn-outline-danger btn-block mr-0"
+        (click)="discardExtractedData()"
+      >
+        <clr-icon shape="times"></clr-icon>
+        <span>Discard</span>
+      </button>
+    </div>
+  </div>
+</ng-template>

client/src/app/xlmap/xlmap.component.scss

@@ -0,0 +1,77 @@
+.card {
+  margin-top: 0;
+
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+}
+
+clr-tree-node button {
+  white-space: nowrap;
+}
+
+.no-table-selected {
+  position: relative;
+}
+
+.header-row {
+  .title-col {
+    display: flex;
+    align-items: center;
+  }
+  .options-col {
+    display: flex;
+    justify-content: flex-end;
+  }
+}
+
+.sw {
+  margin: 1rem 0rem 0.5rem 1rem;
+}
+
+.viewerTitle {
+  text-align: center;
+}
+
+.cardFlex {
+  display: flex;
+  justify-content: center;
+}
+
+.content-area {
+  padding: 0.5rem !important;
+
+  display: flex;
+  flex-direction: column;
+}
+
+hot-table {
+  ::ng-deep {
+    .primaryKeyHeaderStyle {
+      background: #306b006e;
+    }
+  }
+}
+
+.drop-area {
+  position: fixed;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+
+  display: flex;
+  justify-content: center;
+
+  margin: 1px;
+
+  border: 2px dashed #fff;
+
+  z-index: -1;
+
+  span {
+    font-size: 20px;
+    margin-top: 20px;
+    color: #fff;
+  }
+}

client/src/app/xlmap/xlmap.component.ts

@@ -0,0 +1,490 @@
+import {
+  AfterContentInit,
+  AfterViewInit,
+  Component,
+  ElementRef,
+  HostBinding,
+  OnInit,
+  QueryList,
+  ViewChildren
+} from '@angular/core'
+import { ActivatedRoute, Router } from '@angular/router'
+import { UploadFile } from '@sasjs/adapter'
+import * as XLSX from '@sheet/crypto'
+import { XLMapListItem, globals } from '../_globals'
+import { FileUploader } from '../models/FileUploader.class'
+import {
+  EventService,
+  LicenceService,
+  LoggerService,
+  SasService,
+  SasStoreService
+} from '../services'
+import { getCellAddress, getFinishingCell } from './utils/xl.utils'
+import { blobToFile, byteArrayToBinaryString } from './utils/file.utils'
+
+interface XLMapRule {
+  XLMAP_ID: string
+  XLMAP_SHEET: string
+  XLMAP_RANGE_ID: string
+  XLMAP_START: string
+  XLMAP_FINISH: string
+}
+
+interface XLUploadEntry {
+  LOAD_REF: string
+  XLMAP_ID: string
+  XLMAP_RANGE_ID: string
+  ROW_NO: number
+  COL_NO: number
+  VALUE_TXT: string
+}
+
+enum Status {
+  NoMapSelected,
+  FetchingRules,
+  ReadyToUpload,
+  ExtractingData,
+  ReadyToSubmit,
+  SubmittingExtractedData,
+  Submitting
+}
+
+enum Tabs {
+  Rules,
+  Data
+}
+
+@Component({
+  selector: 'app-xlmap',
+  templateUrl: './xlmap.component.html',
+  styleUrls: ['./xlmap.component.scss']
+})
+export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
+  @HostBinding('class.content-container') contentContainerClass = true
+  @ViewChildren('fileUploadInput')
+  fileUploadInputCompList: QueryList<ElementRef> = new QueryList()
+
+  StatusEnum = Status
+  TabsEnum = Tabs
+
+  public selectedTab = Tabs.Rules
+  public rulesSource = globals.dcLib + '.MPE_XLMAP_RULES'
+
+  public xlmaps: XLMapListItem[] = []
+  public selectedXLMap: XLMapListItem | undefined = undefined
+  public searchString = ''
+  public xlmapsLoading = true
+  public isLoading = false
+  public isLoadingDesc = ''
+  public status = Status.NoMapSelected
+
+  public xlmapRulesHeaders = [
+    'XLMAP_SHEET',
+    'XLMAP_RANGE_ID',
+    'XLMAP_START',
+    'XLMAP_FINISH'
+  ]
+  public xlmapRulesColumns = [
+    {
+      data: 'XLMAP_SHEET'
+    },
+    {
+      data: 'XLMAP_RANGE_ID'
+    },
+
+    {
+      data: 'XLMAP_START'
+    },
+    {
+      data: 'XLMAP_FINISH'
+    }
+  ]
+  public xlmapRules: XLMapRule[] = []
+
+  public xlUploadHeader = ['XLMAP_RANGE_ID', 'ROW_NO', 'COL_NO', 'VALUE_TXT']
+  public xlUploadColumns = [
+    {
+      data: 'XLMAP_RANGE_ID'
+    },
+    {
+      data: 'ROW_NO'
+    },
+    {
+      data: 'COL_NO'
+    },
+    {
+      data: 'VALUE_TXT'
+    }
+  ]
+  public xlData: XLUploadEntry[] = []
+
+  public showUploadModal = false
+  public hasBaseDropZoneOver = false
+  public filename = ''
+  public submitLimitNotice = false
+
+  public uploader: FileUploader = new FileUploader()
+
+  public licenceState = this.licenceService.licenceState
+
+  public hotTableLicenseKey: string | undefined = undefined
+  public hotTableMaxRows =
+    this.licenceState.value.viewer_rows_allowed || Infinity
+
+  constructor(
+    private eventService: EventService,
+    private licenceService: LicenceService,
+    private loggerService: LoggerService,
+    private route: ActivatedRoute,
+    private router: Router,
+    private sasStoreService: SasStoreService,
+    private sasService: SasService
+  ) {}
+
+  public xlmapOnClick(xlmap: XLMapListItem) {
+    if (xlmap.id !== this.selectedXLMap?.id) {
+      this.selectedXLMap = xlmap
+      this.xlData = []
+      this.filename = ''
+      this.uploader.queue = []
+      if (this.fileUploadInputCompList.first) {
+        this.fileUploadInputCompList.first.nativeElement.value = ''
+      }
+      this.selectedTab = Tabs.Rules
+      this.viewXLMapRules()
+      this.router.navigateByUrl('/home/files/' + xlmap.id)
+    }
+  }
+
+  public xlmapListOnFilter() {
+    if (this.searchString.length > 0) {
+      const array: XLMapListItem[] = globals.xlmaps
+      this.xlmaps = array.filter((item) =>
+        item.id.toLowerCase().includes(this.searchString.toLowerCase())
+      )
+    } else {
+      this.xlmaps = globals.xlmaps
+    }
+  }
+
+  public isActiveXLMap(id: string) {
+    return this.selectedXLMap?.id === id
+  }
+
+  public maxWidthChecker(width: any, col: any) {
+    if (width > 200) return 200
+    else return width
+  }
+
+  public getCellConfiguration() {
+    return { readOnly: true }
+  }
+
+  public rowHeaders() {
+    return ' '
+  }
+
+  public onShowUploadModal() {
+    this.showUploadModal = true
+  }
+
+  /**
+   * Called by FileDropDirective
+   * @param e true if file is dragged over the drop zone
+   */
+  public fileOverBase(e: boolean): void {
+    this.hasBaseDropZoneOver = e
+  }
+
+  public getFileDesc(event: any, dropped = false) {
+    const file = dropped ? event[0] : event.target.files[0]
+
+    if (!file) return
+
+    const filename = file.name
+    this.filename = filename
+
+    const fileType = filename.slice(
+      filename.lastIndexOf('.') + 1,
+      filename.lastIndexOf('.') + 4
+    )
+
+    if (fileType.toLowerCase() === 'xls') {
+      this.showUploadModal = false
+      this.isLoading = true
+      this.isLoadingDesc = 'Extracting Data'
+      this.status = Status.ExtractingData
+
+      const reader = new FileReader()
+      reader.onload = async (theFile: any) => {
+        /* read workbook */
+        const bstr = byteArrayToBinaryString(theFile.target.result)
+        let wb: XLSX.WorkBook | undefined = undefined
+
+        const xlsxOptions: XLSX.ParsingOptions = {
+          type: 'binary',
+          cellDates: false,
+          cellFormula: true,
+          cellStyles: true,
+          cellNF: false,
+          cellText: false
+        }
+
+        try {
+          wb = XLSX.read(bstr, {
+            ...xlsxOptions
+          })
+        } catch (err: any) {
+          this.eventService.showAbortModal(
+            null,
+            err,
+            undefined,
+            'Error reading file'
+          )
+        }
+
+        if (!wb) {
+          this.isLoading = false
+          this.isLoadingDesc = ''
+          this.status = Status.ReadyToUpload
+          this.uploader.queue.pop()
+          return
+        }
+
+        this.extractData(wb)
+        return
+      }
+
+      reader.readAsArrayBuffer(file)
+    } else {
+      this.isLoading = false
+      this.isLoadingDesc = ''
+      this.status = Status.ReadyToUpload
+      this.showUploadModal = true
+      this.uploader.queue.pop()
+
+      const abortMsg =
+        'Invalid file type "<b>' +
+        this.filename +
+        '</b>". Please upload excel file.'
+      this.eventService.showAbortModal(null, abortMsg)
+    }
+  }
+
+  public discardExtractedData() {
+    this.isLoading = false
+    this.isLoadingDesc = ''
+    this.status = Status.ReadyToUpload
+    this.xlData = []
+    this.selectedTab = Tabs.Rules
+    this.filename = ''
+    this.uploader.queue = []
+    if (this.fileUploadInputCompList.first) {
+      this.fileUploadInputCompList.first.nativeElement.value = ''
+    }
+  }
+
+  /**
+   * Submits attached excel file that is in preview mode
+   */
+  public submitExcel() {
+    if (this.licenceState.value.submit_rows_limit !== Infinity) {
+      this.submitLimitNotice = true
+      return
+    }
+
+    this.submit()
+  }
+
+  public submit() {
+    if (!this.selectedXLMap || !this.xlData.length) return
+
+    this.status = Status.Submitting
+    this.isLoading = true
+    this.isLoadingDesc = 'Submitting extracted data'
+
+    const filesToUpload: UploadFile[] = []
+
+    for (const file of this.uploader.queue) {
+      filesToUpload.push({
+        file: file,
+        fileName: file.name
+      })
+    }
+
+    const csvContent =
+      Object.keys(this.xlData[0]).join(',') +
+      '\n' +
+      this.xlData
+        .slice(0, this.licenceState.value.submit_rows_limit)
+        .map((row: any) => Object.values(row).join(','))
+        .join('\n')
+
+    const blob = new Blob([csvContent], { type: 'application/csv' })
+    const file: File = blobToFile(blob, this.filename + '.csv')
+
+    filesToUpload.push({
+      file: file,
+      fileName: file.name
+    })
+
+    const uploadUrl = 'services/editors/loadfile'
+    this.sasService
+      .uploadFile(uploadUrl, filesToUpload, {
+        table: this.selectedXLMap.targetDS
+      })
+      .then((res: any) => {
+        if (res.sasjsAbort) {
+          const abortRes = res
+          const abortMsg = abortRes.sasjsAbort[0].MSG
+          const macMsg = abortRes.sasjsAbort[0].MAC
+
+          this.eventService.showAbortModal('', abortMsg, {
+            SYSWARNINGTEXT: abortRes.SYSWARNINGTEXT,
+            SYSERRORTEXT: abortRes.SYSERRORTEXT,
+            MAC: macMsg
+          })
+        } else if (res.sasparams) {
+          const params = res.sasparams[0]
+          const tableId = params.DSID
+          this.router.navigateByUrl('/stage/' + tableId)
+        }
+      })
+      .catch((err: any) => {
+        this.eventService.catchResponseError('file upload', err)
+      })
+      .finally(() => {
+        this.status = Status.ReadyToSubmit
+        this.isLoading = false
+        this.isLoadingDesc = ''
+      })
+  }
+
+  public extractData(wb: XLSX.WorkBook) {
+    const extractedData: XLUploadEntry[] = []
+
+    this.xlmapRules.forEach((rule) => {
+      let sheetName = rule.XLMAP_SHEET
+      // if sheet name is not an absolute name rather an index string like /1, /2, etc
+      // we extract the index and find absolute sheet name for specified index
+      if (sheetName.startsWith('/')) {
+        const temp = sheetName.split('/')[1]
+        const sheetIndex = parseInt(temp) - 1
+        sheetName = wb.SheetNames[sheetIndex]
+      }
+
+      const sheet = wb.Sheets[sheetName]
+
+      const arrayOfObjects = <any[]>XLSX.utils.sheet_to_json(sheet, {
+        raw: true,
+        header: 'A',
+        blankrows: true
+      })
+
+      const start = getCellAddress(rule.XLMAP_START, arrayOfObjects)
+      const finish = getFinishingCell(start, rule.XLMAP_FINISH, arrayOfObjects)
+
+      const a1Range = `${start}:${finish}`
+
+      const range = XLSX.utils.decode_range(a1Range)
+
+      const rangedData = <any[]>XLSX.utils.sheet_to_json(sheet, {
+        raw: true,
+        range: a1Range,
+        header: 'A',
+        blankrows: true
+      })
+
+      for (let i = 0; i < rangedData.length; i++) {
+        const row = rangedData[i]
+
+        // `range.s.c` is the index of first column in the range
+        // `range.e.c` is the index of last column in the range
+        // we'll iterate from first column to last column and
+        // extract value where defined and push to extracted data array
+        for (let j = range.s.c, x = 0; j <= range.e.c; j++, x++) {
+          const col = XLSX.utils.encode_col(j)
+
+          if (col in row) {
+            // in excel's R1C1 notation indexing starts from 1 but in JS it starts from 0
+            // therefore, we'll have to add 1 to rows and cols
+            extractedData.push({
+              LOAD_REF: '0',
+              XLMAP_ID: rule.XLMAP_ID,
+              XLMAP_RANGE_ID: rule.XLMAP_RANGE_ID,
+              ROW_NO: i + 1,
+              COL_NO: x + 1,
+              VALUE_TXT: row[col]
+            })
+          }
+        }
+      }
+    })
+
+    this.status = Status.ReadyToSubmit
+    this.isLoading = false
+    this.isLoadingDesc = ''
+
+    this.xlData = extractedData
+    this.selectedTab = Tabs.Data
+  }
+
+  async viewXLMapRules() {
+    if (!this.selectedXLMap) return
+
+    this.isLoading = true
+    this.isLoadingDesc = 'Loading excel rules'
+    this.status = Status.FetchingRules
+
+    await this.sasStoreService
+      .getXLMapRules(this.selectedXLMap.id)
+      .then((res) => {
+        this.xlmapRules = res.xlmaprules
+        this.status = Status.ReadyToUpload
+      })
+      .catch((err) => {
+        this.loggerService.error(err)
+      })
+
+    this.isLoading = false
+    this.isLoadingDesc = ''
+  }
+
+  private load() {
+    this.xlmaps = globals.xlmaps
+    this.xlmapsLoading = false
+
+    const id = this.route.snapshot.params['id']
+
+    if (id) {
+      const xlmapListItem = this.xlmaps.find((item) => item.id === id)
+      if (xlmapListItem) {
+        this.selectedXLMap = xlmapListItem
+        this.viewXLMapRules()
+      }
+    }
+  }
+
+  ngOnInit() {
+    this.licenceService.hot_license_key.subscribe(
+      (hot_license_key: string | undefined) => {
+        this.hotTableLicenseKey = hot_license_key
+      }
+    )
+  }
+
+  ngAfterViewInit() {
+    return
+  }
+
+  ngAfterContentInit(): void {
+    if (globals.editor.startupSet) {
+      this.load()
+    } else {
+      this.eventService.onStartupDataLoaded.subscribe(() => {
+        this.load()
+      })
+    }
+  }
+}

client/src/app/xlmap/xlmap.module.ts

@@ -0,0 +1,31 @@
+import { CommonModule } from '@angular/common'
+import { NgModule } from '@angular/core'
+import { FormsModule } from '@angular/forms'
+import { ClarityModule } from '@clr/angular'
+import { HotTableModule } from '@handsontable/angular'
+import { registerAllModules } from 'handsontable/registry'
+import { AppSharedModule } from '../app-shared.module'
+import { DirectivesModule } from '../directives/directives.module'
+import { XLMapRouteComponent } from '../routes/xlmap-route/xlmap-route.component'
+import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
+import { XLMapRoutingModule } from './xlmap-routing.module'
+import { XLMapComponent } from './xlmap.component'
+
+// register Handsontable's modules
+registerAllModules()
+
+@NgModule({
+  declarations: [XLMapRouteComponent, XLMapComponent],
+  imports: [
+    HotTableModule,
+    XLMapRoutingModule,
+    FormsModule,
+    ClarityModule,
+    AppSharedModule,
+    CommonModule,
+    DcTreeModule,
+    DirectivesModule
+  ],
+  exports: [XLMapComponent]
+})
+export class XLMapModule {}

client/src/environments/_eula.ts

@@ -18,4 +18,4 @@ In any case, you must not make any such use of this software as to develop softw
 UNLESS EXPRESSLY AGREED OTHERWISE, 4GL APPS PROVIDES THIS SOFTWARE ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, AND IN NO EVENT AND UNDER NO LEGAL THEORY, SHALL 4GL APPS BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING FROM USE OR INABILITY TO USE THIS SOFTWARE.
 
 
-`
+`

client/src/styles.scss

@@ -1,16 +1,17 @@
 /* You can add global styles to this file, and also import other style files */
 @import '~handsontable/dist/handsontable.full.css';
 
-@import "~@clr/ui/clr-ui.min.css";
+@import '~@clr/ui/clr-ui.min.css';
-@import "~@clr/icons/clr-icons.min.css";
+@import '~@clr/icons/clr-icons.min.css';
 
-@font-face{
+@font-face {
   font-family: text-security-disc;
-  src: url("https://raw.githubusercontent.com/noppa/text-security/master/dist/text-security-disc.woff");
+  src: url('https://raw.githubusercontent.com/noppa/text-security/master/dist/text-security-disc.woff');
 }
 
-body, html {
+body,
-  font-weight: 400!important;
+html {
+  font-weight: 400 !important;
 
   padding: 0;
   margin: 0;
@@ -29,7 +30,7 @@ button {
 }
 
 // Custom loading spinner
-.slider{
+.slider {
   position: absolute;
   width: 320px;
   margin-left: 75px;
@@ -38,33 +39,45 @@ button {
   overflow-x: hidden;
 }
 
-.line{
+.line {
-  position:absolute;
+  position: absolute;
   opacity: 0.4;
-  background:#73D544;
+  background: #73d544;
-  width:150%;
+  width: 150%;
-  height:5px;
+  height: 5px;
 }
 
-.subline{
+.subline {
-  position:absolute;
+  position: absolute;
-  background:#73D544;
+  background: #73d544;
-  height:5px;
+  height: 5px;
 }
-.inc{
+.inc {
   animation: increase 2s infinite;
 }
-.dec{
+.dec {
   animation: decrease 2s 0.5s infinite;
 }
 
 @keyframes increase {
-   from { left: -5%; width: 5%; }
+  from {
-   to { left: 130%; width: 100%;}
+    left: -5%;
+    width: 5%;
+  }
+  to {
+    left: 130%;
+    width: 100%;
+  }
 }
 @keyframes decrease {
-   from { left: -80%; width: 80%; }
+  from {
-   to { left: 110%; width: 10%;}
+    left: -80%;
+    width: 80%;
+  }
+  to {
+    left: 110%;
+    width: 10%;
+  }
 }
 // Custo loading spinner end
 
@@ -276,6 +289,10 @@ button {
   margin-bottom: 10px;
 }
 
+.mb-10-i {
+  margin-bottom: 10px !important;
+}
+
 .mb-20 {
   margin-bottom: 20px;
 }
@@ -321,11 +338,11 @@ button {
 }
 
 .color-dark-gray {
-  color: #495967
+  color: #495967;
 }
 
-.color-darker-gray{
+.color-darker-gray {
-  color: #314351
+  color: #314351;
 }
 
 .color-white {
@@ -333,7 +350,7 @@ button {
 }
 
 .color-white-i {
-  color: white !important
+  color: white !important;
 }
 
 .color-green {
@@ -341,15 +358,15 @@ button {
 }
 
 .color-dc-green {
-  color: #81b440
+  color: #81b440;
 }
 
 .color-red {
-  color: #e45454
+  color: #e45454;
 }
 
 .color-orange {
-  color: #E67E22;
+  color: #e67e22;
 }
 
 .color-blue {
@@ -357,7 +374,7 @@ button {
 }
 
 .color-yellow {
-  color: #f1c40f
+  color: #f1c40f;
 }
 
 .cursor-pointer {
@@ -501,7 +518,7 @@ button {
 }
 
 .z-index-highest {
-  z-index: 10000000
+  z-index: 10000000;
 }
 
 .vertical-align-middle {
@@ -519,35 +536,36 @@ button {
 }
 
 .progresStatic {
-  margin-top:-6px!important;
+  margin-top: -6px !important;
-  position: absolute!important;
+  position: absolute !important;
-  z-index: 10000!important;
+  z-index: 10000 !important;
 }
 
-.progress, .progress-static {
+.progress,
+.progress-static {
   background-color: #f5f6fe;
   border-radius: 0;
   font-size: inherit;
   height: 6px;
   margin: 0;
-  max-height: .583333rem;
+  max-height: 0.583333rem;
-  min-height: .166667rem;
+  min-height: 0.166667rem;
   overflow: hidden;
   display: block;
   width: calc(100% - 63px);
 }
 
 .progress.loop:after {
-    -webkit-animation: clr-progress-looper 1.5s ease-in-out infinite;
+  -webkit-animation: clr-progress-looper 1.5s ease-in-out infinite;
-    animation: clr-progress-looper 1.5s ease-in-out infinite;
+  animation: clr-progress-looper 1.5s ease-in-out infinite;
-    content: " ";
+  content: ' ';
-    top: .166667rem;
+  top: 0.166667rem;
-    bottom: 0;
+  bottom: 0;
-    left: 0;
+  left: 0;
-    position: absolute;
+  position: absolute;
-    display: block;
+  display: block;
-    background-color: #60b515;
+  background-color: #60b515;
-    width: 75%;
+  width: 75%;
 }
 
 // Fix for clarity bug, should be addressed when clarity is updated
@@ -570,9 +588,9 @@ button {
 }
 
 .alert-app-level.alert-danger {
-    background: #D94B2E;
+  background: #d94b2e;
-    color: #fff;
+  color: #fff;
-    border: none;
+  border: none;
 }
 
 .card-header {
@@ -581,7 +599,7 @@ button {
 
 .select select:focus {
   border-bottom: 1px solid #495967;
-  background: linear-gradient(180deg,transparent 95%,#495a67 0) no-repeat;
+  background: linear-gradient(180deg, transparent 95%, #495a67 0) no-repeat;
 }
 
 .clr-treenode-children {
@@ -597,7 +615,9 @@ button {
   background: #d8e3e9;
 }
 
-clr-select-container .clr-control-container, clr-select-container .clr-control-container .clr-select-wrapper, clr-select-container select {
+clr-select-container .clr-control-container,
+clr-select-container .clr-control-container .clr-select-wrapper,
+clr-select-container select {
   width: 100%;
 }
 
@@ -605,42 +625,46 @@ tbody {
   font-weight: 400;
 }
 
-h3, h4 {
+h3,
-    color: #585858;
+h4 {
-    font-weight: 400;
+  color: #585858;
-    letter-spacing: normal;
+  font-weight: 400;
-    line-height: 1rem;
+  letter-spacing: normal;
-    margin-top: 1rem;
+  line-height: 1rem;
-    margin-bottom: 0;
+  margin-top: 1rem;
-    /* text-transform: uppercase; */
+  margin-bottom: 0;
+  /* text-transform: uppercase; */
 }
 
-h1, h2 {
+h1,
-    color: #585858;
+h2 {
-    font-weight: 400;
+  color: #585858;
-    /* font-family: Metropolis,Avenir Next,Helvetica Neue,Arial,sans-serif; */
+  font-weight: 400;
-    letter-spacing: normal;
+  /* font-family: Metropolis,Avenir Next,Helvetica Neue,Arial,sans-serif; */
-    line-height: 2rem;
+  letter-spacing: normal;
-    margin-top: 1rem;
+  line-height: 2rem;
-    margin-bottom: 0;
+  margin-top: 1rem;
-    /* text-transform: uppercase; */
+  margin-bottom: 0;
+  /* text-transform: uppercase; */
 }
 
 clr-icon.is-info {
-    fill: #80b441;
+  fill: #80b441;
 }
 
-.datagrid-host, .datagrid-overlay-wrapper {
+.datagrid-host,
-    display: -webkit-box;
+.datagrid-overlay-wrapper {
-    display: -ms-flexbox;
+  display: -webkit-box;
-    display: -webkit-box!important;
+  display: -ms-flexbox;
-    -webkit-box-direction: normal;
+  display: -webkit-box !important;
+  -webkit-box-direction: normal;
 }
 
-.btn.btn-danger, .btn.btn-warning {
+.btn.btn-danger,
-    border-color: #ef4f2e;
+.btn.btn-warning {
-    background-color: #D94B2E;
+  border-color: #ef4f2e;
-    color: #fff;
+  background-color: #d94b2e;
+  color: #fff;
 }
 
 .d-none {
@@ -685,11 +709,11 @@ clr-icon.is-info {
 }
 
 .handsontable td.htInvalid {
-  background: #e62700ad!important;
+  background: #e62700ad !important;
-    border: 1px solid red !important;
+  border: 1px solid red !important;
-    color: #ffffff!important;
+  color: #ffffff !important;
 }
-.margin-top-20{
+.margin-top-20 {
   margin-top: 20px;
 }
 .hidden {
@@ -823,7 +847,7 @@ clr-icon.is-info {
 }
 
 .datagrid-body {
-  padding-bottom: 2rem!important;
+  padding-bottom: 2rem !important;
 }
 
 .abortMsg {
@@ -831,16 +855,15 @@ clr-icon.is-info {
   font-family: monospace;
 }
 
-
 #graph svg {
-    height: 100%;
+  height: 100%;
-    width: 100%;
+  width: 100%;
 }
 
 .no-table-selected {
-  display:flex;
+  display: flex;
-  justify-content:center;
+  justify-content: center;
-  flex-direction:column;
+  flex-direction: column;
   align-items: center;
   position: absolute;
   background: white;
@@ -851,16 +874,15 @@ clr-icon.is-info {
 }
 
 .copyRight {
-  background:#495967!important;
+  background: #495967 !important;
   color: #fff;
-  display:flex !important;
+  display: flex !important;
-  justify-content:center;
+  justify-content: center;
   align-items: center;
   padding: 5px 0px 4px 0px;
   z-index: 100;
 }
 
-
 .nav-tree > clr-tree-node.clr-expanded {
   display: inline-block !important;
 }
@@ -903,13 +925,13 @@ clr-tree-node {
 }
 
 .tree-search-wrapper {
-    position: relative;
+  position: relative;
-    display: flex;
+  display: flex;
-    align-items: center;
+  align-items: center;
 
-    clr-input-container {
+  clr-input-container {
-      margin: 0;
+    margin: 0;
-    }
+  }
 
   clr-icon {
     position: absolute;
@@ -956,7 +978,8 @@ input::-ms-clear {
     overflow: hidden !important;
   }
 
-  .clr-treenode-content .clr-icon, .clr-treenode-content clr-icon {
+  .clr-treenode-content .clr-icon,
+  .clr-treenode-content clr-icon {
     min-width: 16px;
     min-height: 16px;
   }
@@ -985,12 +1008,12 @@ input::-ms-clear {
 }
 
 .loadingSpinner {
-  height:70vh;
+  height: 70vh;
   flex: 1;
-  display:flex;
+  display: flex;
   justify-content: center;
-  flex-direction:column;
+  flex-direction: column;
-  align-items:center;
+  align-items: center;
 }
 
 .disable-password-manager {
@@ -1025,7 +1048,8 @@ hr.light {
     position: relative;
     min-width: 170px;
 
-    clr-icon, .spinner {
+    clr-icon,
+    .spinner {
       position: absolute;
       right: 19px;
       top: 0px;
@@ -1063,7 +1087,7 @@ hr.light {
   }
 
   /* Firefox */
-  input[type=number] {
+  input[type='number'] {
     -moz-appearance: textfield;
   }
 }
@@ -1076,4 +1100,4 @@ hr.light {
 .link-it {
   cursor: pointer;
   text-decoration: underline;
-}
+}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "dcfrontend",
-  "version": "6.2.7",
+  "version": "6.3.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "dcfrontend",
-      "version": "6.2.7",
+      "version": "6.3.0",
       "hasInstallScript": true,
       "devDependencies": {
         "@saithodev/semantic-release-gitea": "^2.1.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "6.3.1",
+  "version": "6.6.2",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/mocks/sasjs/services/public/startupservice.js

@@ -83,6 +83,12 @@ _webout = `{"SYSDATE" : "26SEP22"
 "DC_RESTRICT_EDITRECORD": "NO"
 }
 ]
+,"xlmaps":
+[
+["BASEL-CR2" ,"" ,"DC695588.MPE_XLMAP_DATA" ]
+,["BASEL-KM1" ,"Basel 3 Key Metrics report" ,"DC695588.MPE_XLMAP_DATA" ]
+,["SAMPLE" ,"" ,"DC695588.MPE_XLMAP_DATA" ]
+]
 ,"_DEBUG" : ""
 ,"_METAUSER": "sasdemo@SAS"
 ,"_METAPERSON": "sasdemo"

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "^4.11.1",
-        "@sasjs/core": "^4.48.4"
+        "@sasjs/core": "^4.49.0"
       }
     },
     "node_modules/@coolaj86/urequest": {
@@ -116,9 +116,9 @@
       "integrity": "sha512-Grwydm5GxBsYk238PZw41XPjXVVQ9vWcvfZ06L2P0bQbvK0sGn7l69JA7H5MGr3QcaLpiD4Kg70cAh7PgE+JOw=="
     },
     "node_modules/@sasjs/core": {
-      "version": "4.48.4",
+      "version": "4.49.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.48.4.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.49.0.tgz",
-      "integrity": "sha512-KTLHRR47I627NKZG0qMW+wGJP4gFGyeEEyBDsVaSnevdvCz01oP/lpLxx4fIESPQ/YzLNEv+RcP8kcxkdSPQow=="
+      "integrity": "sha512-hp3Hb4DkT6FmowyNHTOvSlgmSObW9WeuTJj+TQlwPgnBo59mAB4XFUnUaYSA+7ghvsHqUZf1OP2eSYqmnN5swQ=="
     },
     "node_modules/@sasjs/lint": {
       "version": "2.3.1",
@@ -229,6 +229,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/tough-cookie": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
+      "peer": true
+    },
     "node_modules/abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -1828,9 +1834,9 @@
       }
     },
     "@sasjs/core": {
-      "version": "4.48.4",
+      "version": "4.49.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.48.4.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.49.0.tgz",
-      "integrity": "sha512-KTLHRR47I627NKZG0qMW+wGJP4gFGyeEEyBDsVaSnevdvCz01oP/lpLxx4fIESPQ/YzLNEv+RcP8kcxkdSPQow=="
+      "integrity": "sha512-hp3Hb4DkT6FmowyNHTOvSlgmSObW9WeuTJj+TQlwPgnBo59mAB4XFUnUaYSA+7ghvsHqUZf1OP2eSYqmnN5swQ=="
     },
     "@sasjs/lint": {
       "version": "2.3.1",
@@ -1927,6 +1933,12 @@
         "@types/node": "*"
       }
     },
+    "@types/tough-cookie": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
+      "peer": true
+    },
     "abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -2953,7 +2965,8 @@
     "ws": {
       "version": "8.13.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA=="
+      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
+      "requires": {}
     },
     "xml": {
       "version": "1.0.1",

sas/package.json

@@ -14,7 +14,8 @@
     "sas9e": "sasjs request services/admin/makedata -d deploy/makeDataSas9.json -t sas9  ",
     "sas9f": "sasjs request services/admin/refreshtablelineage -t sas9 ",
     "sas9g": "sasjs request services/admin/refreshcatalog -t sas9",
-    "4gl": "npm run cpfavicon && sasjs cbd -t 4gl && sasjs request services/admin/makedata -d deploy/makeData4GL.json -l sasjsresults/makedata_4gl.log -o sasjsresults/makedata_4gl.json -t 4gl",
+    "4gl": "npm run cpfavicon && sasjs cbd -t 4gl && npm run 4glmakedata",
+    "4glmakedata": "sasjs request services/admin/makedata -d deploy/makeData4GL.json -l sasjsresults/makedata_4gl.log -o sasjsresults/makedata_4gl.json -t 4gl",
     "server": "npm run cpfavicon && sasjs cbd -t server && npm run serverdata",
     "server-mihajlo": "npm run cpfavicon && sasjs cbd -t server-mihajlo && npm run serverdata-mihajlo",
     "serverdata-mihajlo": "sasjs request services/admin/makedata -d deploy/makeDataServer.json -l sasjsresults/makedata_server.log -o sasjsresults/makedata_server.json -t server-mihajlo",
@@ -28,6 +29,6 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "^4.11.1",
-    "@sasjs/core": "^4.48.4"
+    "@sasjs/core": "^4.49.0"
   }
 }

sas/sasjs/db/datactrl/mpe_xlmap_data.ddl

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief DDL for MPE_XLMAP_DATA
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+create table &curlib..MPE_XLMAP_DATA(
+        LOAD_REF char(32) not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        ROW_NO num not null,
+        COL_NO num not null,
+        VALUE_TXT char(4000),
+    constraint pk_MPE_XLMAP_DATA
+      primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO));

sas/sasjs/db/datactrl/mpe_xlmap_info.ddl

@@ -0,0 +1,17 @@
+/**
+  @file
+  @brief DDL for mpe_xlmap_info
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+create table &curlib..mpe_xlmap_info(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_DESCRIPTION char(1000) not null,
+        XLMAP_TARGETLIBDS char(41) not null,
+        tx_to num not null,
+    constraint pk_mpe_xlmap_info
+        primary key(tx_from,XLMAP_ID));

sas/sasjs/db/datactrl/mpe_xlmap_rules.ddl

@@ -0,0 +1,19 @@
+/**
+  @file
+  @brief DDL for mpe_xlmap_rules
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+create table &curlib..mpe_xlmap_rules(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        XLMAP_SHEET char(32) not null,
+        XLMAP_START char(1000) not null,
+        XLMAP_FINISH char(1000),
+        tx_to num not null,
+    constraint pk_mpe_xlmap_rules
+        primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID));

sas/sasjs/db/migrations/20230115_v6.5_release.sas

@@ -0,0 +1,83 @@
+/**
+  @file
+  @brief migration script to move from v5 to v6.5 of data controller
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/your/dc/path";
+
+/**
+  * Change 1
+  * New MPE_SUBMIT table
+  */
+proc sql;
+create table &dclib..mpe_xlmap_rules(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        XLMAP_SHEET char(32) not null,
+        XLMAP_START char(1000) not null,
+        XLMAP_FINISH char(1000),
+        tx_to num not null,
+    constraint pk_mpe_xlmap_rules
+        primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID));
+
+create table &dclib..MPE_XLMAP_DATA(
+        LOAD_REF char(32) not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        ROW_NO num not null,
+        COL_NO num not null,
+        VALUE_TXT char(4000),
+    constraint pk_MPE_XLMAP_DATA
+      primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO));
+
+create table &dclib..mpe_xlmap_info(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_DESCRIPTION char(1000) not null,
+        XLMAP_TARGETLIBDS char(41) not null,
+        tx_to num not null,
+    constraint pk_mpe_xlmap_info
+        primary key(tx_from,XLMAP_ID));
+
+
+/* add mpe_tables entries */
+  insert into &dclib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&dclib"
+      ,dsn='MPE_XLMAP_INFO'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+      ,buskey='XLMAP_ID'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+      ,post_edit_hook='services/hooks/mpe_xlmap_info_postedit'
+    ;
+  insert into &dclib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&dclib"
+      ,dsn='MPE_XLMAP_RULES'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+      ,buskey='XLMAP_ID XLMAP_RANGE_ID'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+      ,post_edit_hook='services/hooks/mpe_xlmap_rules_postedit'
+    ;
+  insert into &dclib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&dclib"
+      ,dsn='MPE_XLMAP_DATA'
+      ,num_of_approvals_required=1
+      ,loadtype='UPDATE'
+      ,buskey='LOAD_REF XLMAP_ID XLMAP_RANGE_ID ROW_NO COL_NO'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+    ;

sas/sasjs/macros/mpe_alerts.sas

@@ -46,7 +46,7 @@
 /* get users TO which the email should be sent */
 
 proc sql noprint;
-create table users as select distinct a.alert_user,
+create table work.users as select distinct a.alert_user,
         b.user_displayname,
         b.user_email
   from &mpelib..mpe_alerts
@@ -58,16 +58,26 @@ create table users as select distinct a.alert_user,
     and a.alert_lib in ("&alert_lib","*ALL*")
     and a.alert_ds in ("&alert_ds","*ALL*");
 
-%local isThere;
+/* ensure the submitter is included on the email */
+%local isThere userdisp user_eml;
+%let isThere=0;
 select count(*) into: isThere from &syslast where alert_user="&from_user";
-%if &isThere>0 %then %do;
+%if &isThere=0 %then %do;
-  insert into &syslast set alert_user="&from_user";
+  select user_displayname, user_email
+    into: userdisp trimmed, :user_eml trimmed
+    from &mpelib..mpe_emails
+    where &dc_dttmtfmt. lt tx_to
+      and user_name="&from_user";
+  insert into work.users
+    set alert_user="&from_user"
+      ,user_displayname="&userdisp"
+      ,user_email="&user_eml";
 %end;
 
 
 /* if no email / displayname is provided, then extract from metadata */
-data emails;
+data work.emails;
-  set users;
+  set work.users;
   length emailuri uri text $256; call missing(emailuri,uri); drop emailuri uri;
 
   /* get displayname */
@@ -92,11 +102,13 @@ data emails;
   end;
   /* only keep valid emails */
   if index(user_email,'@') ;
+  /* dump contents for debugging */
+  if _n_<21 then putlog (_all_)(=);
 run;
 
 %local emails;
 proc sql noprint;
-select user_email into: emails separated by '" "' from emails;
+select quote(trim(user_email)) into: emails separated by ' ' from work.emails;
 
 /* exit if nobody to email */
 %if %mf_getattrn(emails,NLOBS)=0 %then %do;
@@ -110,7 +122,7 @@ data _null_;
   put optname '=' setting;
 run;
 
-filename __out email ("&emails")
+filename __out email (&emails)
   subject="Table &alert_lib..&alert_ds has been &alert_event";
 
 %local SUBMITTED_TXT;
@@ -172,4 +184,4 @@ filename __out email ("&emails")
 
 filename __out clear;
 
-%mend mpe_alerts ;
+%mend mpe_alerts ;

sas/sasjs/macros/mpe_dsmeta.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief Gets table metadata
+  @details Runs mp_dsmeta and adds datadictionary info
+
+  <h4> SAS Macros </h4>
+  @li mp_dsmeta.sas
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_dsmeta(libds, outds=dsmeta);
+  %local ddsd ddld notes lenstmt;
+  %let lenstmt=length ods_table $18 name $100 value $1000;
+  %let libds=%upcase(&libds);
+  %mp_dsmeta(&libds, outds=&outds)
+
+  data _null_;
+    set &mpelib..mpe_datadictionary;
+    where &dc_dttmtfmt < tx_to & dd_source=%upcase("&libds") & dd_type='TABLE';
+    call symputx('ddsd',dd_shortdesc,'l');
+    call symputx('ddld',dd_longdesc,'l');
+  run;
+
+  data &outds;
+    &lenstmt;
+    if last then do;
+      ODS_TABLE='MPE_DATADICTIONARY';
+      NAME='DD_SHORTDESC';
+      VALUE="&ddsd";
+      output;
+      NAME='DD_LONGDESC';
+      VALUE="&ddld";
+      output;
+    end;
+    set &outds end=last;
+    output;
+  run;
+
+  data _data_;
+    set &mpelib..mpe_tables;
+    where libref="%scan(&libds,1,.)"
+        & dsn="%scan(&libds,2,.)"
+        & &dc_dttmtfmt<tx_to;
+    &lenstmt;
+    ODS_TABLE='MPE_TABLES';
+    array c _character_;
+    array n _numeric_;
+    do over c;
+      name=upcase(vname(c));
+      value=c;
+      output;
+    end;
+    do over n;
+      name=upcase(vname(n));
+      value=cats(n);
+      output;
+    end;
+    keep ods_table name value;
+  run;
+
+  proc append base=&outds data=&syslast;
+  run;
+
+
+%mend mpe_dsmeta;

sas/sasjs/macros/mpe_dsmeta.test.sas

@@ -0,0 +1,37 @@
+/**
+  @file
+  @brief Testing mpe_dsmeta macro
+  @details Checking functionality of mpe_dsmeta.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mpe_dsmeta.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* run the macro*/
+%mp_assertscope(SNAPSHOT)
+%mpe_dsmeta(&mpelib..mpe_security, outds=test1)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot
+)
+
+data work.test1;
+  set work.test1;
+  where ods_table in ('MPE_DATADICTIONARY','MPE_TABLES');
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.test1,
+  desc=Test 1 - 27 records returned,
+  test=EQUALS 27,
+  outds=work.test_results
+)
+
+

sas/sasjs/macros/mpe_loader.sas

@@ -24,6 +24,7 @@
   @li mp_lockanytable.sas
   @li mpe_accesscheck.sas
   @li mpe_alerts.sas
+  @li mpe_xlmapvalidate.sas
   @li mpe_loadfail.sas
   @li mpe_runhook.sas
 
@@ -450,7 +451,7 @@ run;
     %do i=1 %to %sysfunc(countw(&pk));
       %let iWord=%scan(&pk,&i);
       call symputx('duplist',symget('duplist')!!
-        " &iWord="!!trim(&iWord));
+        " &iWord="!!cats(&iWord));
     %end;
   run;
   %let msg=This upload contains duplicates on the Primary Key columns %trim(
@@ -472,6 +473,10 @@ run;
   %return;
 %end;
 
+/* If a Complex Excel Upload, needs to have the load ref added to the table */
+%mpe_xlmapvalidate(&mperef,work.staging_ds,&mpelib,&orig_libds)
+
+/* Run the Post Edit Hook prior to creation of staging folder */
 %mpe_runhook(POST_EDIT_HOOK)
 
 /* stop if err */

sas/sasjs/macros/mpe_makedata.sas

@@ -269,6 +269,210 @@ insert into &lib..mpe_datadictionary set
     ,DD_SENSITIVITY="Low"
     ,tx_to='31DEC5999:23:59:59'dt;
 
+/**
+  * mpe_xlmap_info
+  */
+insert into &lib..mpe_xlmap_info set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_description='Basel 3 Key Metrics report'
+  ,XLMAP_TARGETLIBDS="&lib..MPE_XLMAP_DATA";
+
+/**
+  * mpe_xlmap_rules
+  */
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:a';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:b'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:b';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:c'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:c';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:d'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:d';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:e'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:e';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:f'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:f';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[1]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/b'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[2]:Common Equity Tier 1 (CET1)';
+  insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/c'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[3]:Common Equity Tier 1 (CET1)';
+  insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/d'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[4]:Common Equity Tier 1 (CET1)';
+  insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/e'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[5]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/f'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[6]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1a/e'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[1]C[5]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1a/f'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[1]C[6]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='ABSOLUTE D10';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/b'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='ABSOLUTE E10';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/c'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[6]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/d'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[8]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/e'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[9]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/f'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[10]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='ABSOLUTE H11'
+  ,xlmap_finish='RELATIVE R[0]C[1]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:3'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='RELATIVE R[12]C[4]'
+  ,xlmap_finish='ABSOLUTE I13';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-CR2'
+  ,xlmap_range_id='CR2-sec1'
+  ,xlmap_sheet='CR2'
+  ,xlmap_start='ABSOLUTE D8'
+  ,xlmap_finish='BLANKROW';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-CR2'
+  ,xlmap_range_id='CR2-sec2'
+  ,xlmap_sheet='CR2'
+  ,xlmap_start='ABSOLUTE D18'
+  ,xlmap_finish='LASTDOWN';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='SAMPLE'
+  ,xlmap_range_id='header'
+  ,xlmap_sheet='/1'
+  ,xlmap_start='ABSOLUTE B3'
+  ,xlmap_finish='ABSOLUTE B8';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='SAMPLE'
+  ,xlmap_range_id='data'
+  ,xlmap_sheet='/1'
+  ,xlmap_start='ABSOLUTE B13'
+  ,xlmap_finish='ABSOLUTE E16';
+
+
+
 /**
   * MPE_GROUPS
   */
@@ -981,6 +1185,42 @@ insert into &lib..mpe_selectbox set
       ,notes='Docs: https://docs.datacontroller.io/column-level-security'
       ,post_edit_hook='services/hooks/mpe_column_level_security_postedit'
     ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_XLMAP_INFO'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+      ,buskey='XLMAP_ID'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+      ,post_edit_hook='services/hooks/mpe_xlmap_info_postedit'
+    ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_XLMAP_RULES'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+      ,buskey='XLMAP_ID XLMAP_RANGE_ID'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+      ,post_edit_hook='services/hooks/mpe_xlmap_rules_postedit'
+    ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_XLMAP_DATA'
+      ,num_of_approvals_required=1
+      ,loadtype='UPDATE'
+      ,buskey='LOAD_REF XLMAP_ID XLMAP_RANGE_ID ROW_NO COL_NO'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+    ;
   insert into &lib..mpe_tables
     set tx_from=0
       ,tx_to='31DEC5999:23:59:59'dt
@@ -1253,6 +1493,27 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/mpe_alerts.alert_lib"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_XLMAP_INFO"
+    ,base_col="XLMAP_ID"
+    ,rule_type='CASE'
+    ,rule_value='UPCASE'
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_XLMAP_RULES"
+    ,base_col="XLMAP_ID"
+    ,rule_type='CASE'
+    ,rule_value='UPCASE'
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"
@@ -1640,6 +1901,16 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value='1'
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_XLMAP_INFO"
+    ,base_col="XLMAP_ID"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="&lib..MPE_XLMAP_RULES.XLMAP_ID"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
 
 /**
   * MPE_X_TEST

sas/sasjs/macros/mpe_makedatamodel.sas

@@ -268,6 +268,55 @@ proc datasets lib=&lib noprint;
     pk_mpe_excel_config=(tx_to xl_libref xl_table xl_column)
     /nomiss unique;
 quit;
+
+proc sql;
+create table &lib..MPE_XLMAP_DATA(
+        LOAD_REF char(32) &notnull,
+        XLMAP_ID char(32) &notnull,
+        XLMAP_RANGE_ID char(32) &notnull,
+        ROW_NO num &notnull,
+        COL_NO num &notnull,
+        VALUE_TXT char(4000)
+);quit;
+proc datasets lib=&lib noprint;
+  modify MPE_XLMAP_DATA;
+  index create
+    pk_MPE_XLMAP_DATA=(load_ref xlmap_id xlmap_range_id row_no col_no)
+    /nomiss unique;
+quit;
+
+proc sql;
+create table &lib..mpe_xlmap_info(
+        tx_from num &notnull,
+        tx_to num &notnull,
+        XLMAP_ID char(32) &notnull,
+        XLMAP_DESCRIPTION char(1000) &notnull,
+        XLMAP_TARGETLIBDS char(41) &notnull
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_xlmap_info;
+  index create
+    pk_mpe_xlmap_info=(tx_to xlmap_id)
+    /nomiss unique;
+quit;
+
+proc sql;
+create table &lib..mpe_xlmap_rules(
+        tx_from num &notnull,
+        tx_to num &notnull,
+        XLMAP_ID char(32) &notnull,
+        XLMAP_RANGE_ID char(32) &notnull,
+        XLMAP_SHEET char(32) &notnull,
+        XLMAP_START char(1000) &notnull,
+        XLMAP_FINISH char(1000)
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_xlmap_rules;
+  index create
+    pk_mpe_xlmap_rules=(tx_to xlmap_id xlmap_range_id)
+    /nomiss unique;
+quit;
+
 proc sql;
 create table &lib..mpe_filteranytable(
         filter_rk num &notnull,

sas/sasjs/macros/mpe_xlmapvalidate.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Validates excel map structure and adds load ref
+  @details Used in staging, prior to the post edit hook
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_xlmapvalidate(mperef,inds,dclib,tgtds);
+
+  %local ismap;
+  proc sql noprint;
+  select count(*) into: ismap
+    from &dclib..mpe_xlmap_info
+    where XLMAP_TARGETLIBDS="&tgtds" and &dc_dttmtfmt. le TX_TO ;
+
+  %if "&tgtds"="&dclib..MPE_XLMAP_DATA" or &ismap>0 %then %do;
+    data &inds;
+      set &inds;
+      LOAD_REF="&mperef";
+    run;
+  %end;
+
+%mend mpe_xlmapvalidate;

sas/sasjs/macros/mpe_xlmapvalidate.test.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Testing mpe_xlmapvalidate macro
+
+  <h4> SAS Macros </h4>
+  @li mpe_xlmapvalidate.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+  <h4> SAS Includes </h4>
+  @li mpe_xlmap_data.ddl ul
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* create the table */
+%let curlib=work;
+proc sql;
+%inc ul;
+
+data work.test1;
+  if 0 then set work.MPE_XLMAP_DATA;
+  LOAD_REF='0';
+  XLMAP_ID='Sample';
+  XLMAP_RANGE_ID='Range 1';
+  ROW_NO=1;
+  COL_NO=2;
+  VALUE_TXT='something';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mpe_xlmapvalidate(DCTEST1,work.test1,&dclib,NOT.MAP)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot
+)
+
+data _null_;
+  set work.test1;
+  call symputx('test1',load_ref);
+run;
+%mp_assert(
+  iftrue=(&test1=0),
+  desc=Checking load ref was not applied
+)
+
+%mpe_xlmapvalidate(DCTEST2,work.test1,&dclib,&dclib..MPE_XLMAP_DATA)
+
+data _null_;
+  set work.test1;
+  call symputx('test2',load_ref);
+run;
+%mp_assert(
+  iftrue=(&test2=DCTEST2),
+  desc=Checking load ref was applied for default case
+)

sas/sasjs/sasjsconfig.json

@@ -207,7 +207,7 @@
     },
     {
       "name": "4gl",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false

sas/sasjs/services/auditors/postdata.sas

@@ -56,12 +56,12 @@
 data _null_;
   set work.sascontroltable;
   call symputx('ACTION',ACTION);
-  call symputx('TABLE',TABLE);
+  call symputx('LOAD_REF',TABLE);
   /* DIFFTIME is when the DIFF was generated on the frontend */
   call symputx('DIFFTIME',DIFFTIME);
 run;
 
-%global action is_err err_msg;
+%global action is_err err_msg msg;
 %let is_err=0;
 
 %let user=%mf_getuser();
@@ -80,7 +80,7 @@ RUN;
 %let isfmtcat=0;
 data APPROVE1;
   set &mpelib..mpe_submit;
-  where TABLE_ID="&TABLE";
+  where TABLE_ID="&LOAD_REF";
   /* fetch mpe_submit data */
   libds=cats(base_lib,'.',base_ds);
   REVIEWED_ON=put(reviewed_on_dttm,datetime19.);
@@ -115,9 +115,9 @@ run;
 )
 
 %mp_abort(
-  iftrue=(%mf_verifymacvars(difftime orig_libds libds table)=0)
+  iftrue=(%mf_verifymacvars(difftime orig_libds libds load_ref)=0)
   ,mac=&_program
-  ,msg=%str(Missing: difftime orig_libds libds table)
+  ,msg=%str(Missing: difftime orig_libds libds load_ref)
 )
 
 /* security checks */
@@ -186,7 +186,7 @@ run;
   %let prev_upload_check=1;
   proc sql;
   select count(*) into: prev_upload_check from &mpelib..mpe_review
-    where TABLE_ID="&TABLE" and REVIEWED_BY_NM="&user"
+    where TABLE_ID="&LOAD_REF" and REVIEWED_BY_NM="&user"
     and REVIEW_STATUS_ID ne "SUBMITTED";
   %let authcheck=%mf_getattrn(work.authAPP,NLOBS);
   %if &authcheck=0 or &prev_upload_check=1 %then %do;
@@ -233,7 +233,7 @@ run;
   %else %let oldloc=%qsysfunc(getoption(LOG));
   %if %length(&oldloc)>0 %then %do;
     proc printto
-      log="&mpelocapprovals/&TABLE/approval.log";
+      log="&mpelocapprovals/&LOAD_REF/approval.log";
     run;
     data _null_;
       if _n_=1 then do;
@@ -247,7 +247,7 @@ run;
   %end;
   %else %do;
     proc printto
-      log="&mpelocapprovals/&TABLE/approval.log";
+      log="&mpelocapprovals/&LOAD_REF/approval.log";
     run;
   %end;
 
@@ -285,11 +285,11 @@ select PRE_APPROVE_HOOK, POST_APPROVE_HOOK, LOADTYPE, var_txfrom, var_txto
   ,msg=%str(Missing: mpelocapprovals orig_libds)
 )
 
-/* get dataset from approvals location */
+/* get dataset from approvals location (has same name as load_ref) */
 %let tmplib=%mf_getuniquelibref();
-libname &tmplib "&mpelocapprovals/&TABLE";
+libname &tmplib "&mpelocapprovals/&LOAD_REF";
 data STAGING_DS;
-  set &tmplib..&TABLE;
+  set &tmplib..&LOAD_REF;
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
@@ -313,7 +313,7 @@ run;
   %let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
   data work.append_review;
     if 0 then set &mpelib..mpe_review;
-    TABLE_ID="&TABLE";
+    TABLE_ID="&LOAD_REF";
     BASE_TABLE="&orig_libds";
     REVIEW_STATUS_ID="APPROVED";
     REVIEWED_BY_NM="&user";
@@ -323,7 +323,7 @@ run;
     stop;
   run;
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
+    lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc append base=&mpelib..mpe_review data=work.append_review;
@@ -335,7 +335,7 @@ run;
 
   /* update mpe_submit table */
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval),
+    lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc sql;
@@ -343,7 +343,7 @@ run;
     set num_of_approvals_remaining=&num_of_approvals_remaining-1,
       reviewed_by_nm="&user",
       reviewed_on_dttm=&sastime
-    where table_id="&table";
+    where table_id="&LOAD_REF";
   %mp_lockanytable(UNLOCK,
     lib=&mpelib,ds=mpe_submit,
     ctl_ds=&mpelib..mpe_lockanytable
@@ -369,7 +369,7 @@ run;
 )
 %mpe_targetloader(libds=&orig_libds
   ,now= &sastime
-  ,etlsource=&TABLE
+  ,etlsource=&LOAD_REF
   ,STAGING_DS=STAGING_DS
   ,dclib=&mpelib
 %if &action=APPROVE_TABLE %then %do;
@@ -405,7 +405,7 @@ run;
   proc sql noprint;
   select max(processed_dttm)-1 format=datetime19. into: tstamp
     from &mpelib..mpe_dataloads
-    where libref="&libref" and dsn="&ds" and ETLSOURCE="&TABLE";
+    where libref="&libref" and dsn="&ds" and ETLSOURCE="&LOAD_REF";
   quit;
   %if &tstamp=. %then %let tstamp=%sysfunc(datetime(),datetime19.);
 
@@ -498,7 +498,7 @@ run;
     else if _____orig then _____status='ORIGINAL';
   run;
   proc export data=TEMPDIFFS dbms=csv replace
-    outfile="&mpelocapprovals/&TABLE/&tempDIFFS_CSV" ;
+    outfile="&mpelocapprovals/&LOAD_REF/&tempDIFFS_CSV" ;
   run;
   proc sql noprint;
   select filesize format=sizekmg10.1, filesize as filesize_raw
@@ -545,7 +545,7 @@ run;
   proc sort data=&mpelib..mpe_submit(where=(
       submit_status_cd='SUBMITTED'
       and cats(base_lib,'.',base_ds)="&orig_libds"
-      and table_id ne "&TABLE"
+      and table_id ne "&LOAD_REF"
     )) out=submits;
     by descending submitted_on_dttm;
   run;
@@ -599,7 +599,7 @@ run;
     data work.outds_mod; run;
     data work.outds_del; run;
   %end;
-  libname approve "&mpelocapprovals/&TABLE";
+  libname approve "&mpelocapprovals/&LOAD_REF";
   data; set &libds;stop;run;
   %let emptybasetable=&syslast;
   data approve.ActualDiffs;
@@ -621,7 +621,7 @@ run;
   run;
 
   proc export data=approve.ActualDiffs
-    outfile="&mpelocapprovals/&TABLE/ActualDiffs.csv"
+    outfile="&mpelocapprovals/&LOAD_REF/ActualDiffs.csv"
     dbms=csv
     replace;
   run;
@@ -631,7 +631,7 @@ run;
   %let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
   data work.append_review;
     if 0 then set &mpelib..mpe_review;
-    TABLE_ID="&TABLE";
+    TABLE_ID="&LOAD_REF";
     BASE_TABLE="&orig_libds";
     REVIEW_STATUS_ID="APPROVED";
     REVIEWED_BY_NM="&user";
@@ -641,7 +641,7 @@ run;
     stop;
   run;
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
+    lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc append base=&mpelib..mpe_review data=work.append_review;
@@ -653,7 +653,7 @@ run;
 
   /* update mpe_submit table */
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval in auditors/postdata),
+    lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval in auditors/postdata),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc sql;
@@ -662,7 +662,7 @@ run;
       num_of_approvals_remaining=&num_of_approvals_remaining-1,
       reviewed_by_nm="&user",
       reviewed_on_dttm=&sastime
-    where table_id="&table";
+    where table_id="&LOAD_REF";
   %mp_lockanytable(UNLOCK,
     lib=&mpelib,ds=mpe_submit,
     ctl_ds=&mpelib..mpe_lockanytable
@@ -688,7 +688,7 @@ run;
   %mpe_alerts(alert_event=APPROVED
     , alert_lib=&libref
     , alert_ds=&ds
-    , dsid=&TABLE
+    , dsid=&LOAD_REF
   )
 
   %removecolsfromwork(___TMP___MD5)

sas/sasjs/services/auditors/postdata.test.2.sas

@@ -34,7 +34,8 @@ run;
 %mp_testservice(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
-  outlib=web1
+  outlib=web1,
+  mdebug=&sasjs_mdebug
 )
 
 %let status=0;

sas/sasjs/services/editors/getdata.sas

@@ -14,8 +14,9 @@
   <h5> sasdata </h5>
   <h5> sasparams </h5>
   Contains info on the request.  One row is returned.
-  * CLS_FLG - set to 0 if there are no CLS rules (everything should be editable)
+  @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
     else set to 1 (CLS rules exist)
+  @li ISMAP - set to 1 if the target DS is an excel map target, else 0
 
   <h5> approvers </h5>
   <h5> dqrules </h5>
@@ -55,12 +56,12 @@
   @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
   @li mp_cntlout.sas
-  @li mp_dsmeta.sas
   @li mp_getcols.sas
   @li mp_getmaxvarlengths.sas
   @li mp_validatecol.sas
   @li mpe_accesscheck.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_dsmeta.sas
   @li mpe_getlabels.sas
   @li mpe_filtermaster.sas
   @li mpe_runhook.sas
@@ -534,6 +535,11 @@ data _null_;
 run;
 
 %put params;
+%let ismap=0;
+proc sql noprint;
+select count(*) into: ismap from &mpelib..mpe_xlmap_info
+  where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
+
 data sasparams;
   length colHeaders $20000 filter_text $32767;
   colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
@@ -551,8 +557,11 @@ data sasparams;
   if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
   else cls_flag=1;
   put (_all_)(=);
+  if "&orig_libds"="&mpelib..MPE_XLMAP_DATA" or &ismap ne 0 then ismap=1;
+  else ismap=0;
 run;
 
+
 /* Extract validation DQ Rules */
 proc sort data=&mpelib..mpe_validations
   (where=(&dc_dttmtfmt. le TX_TO
@@ -636,11 +645,10 @@ create table dqdata as
 %dq_selects()
 
 proc sort data=dqdata;
-  by base_col selectbox_order;
+  /* order by selectbox_order then the value */
+  by base_col selectbox_order rule_data;
 run;
 
-
-
 %mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)
 
 data maxvarlengths;
@@ -657,7 +665,7 @@ data xl_rules;
   keep xl_column xl_rule;
 run;
 
-%mp_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&libds, outds=dsmeta)
 
 /* send to the client */
 %webout(OPEN)

sas/sasjs/services/editors/getdata.test.sas

@@ -141,13 +141,15 @@ run;
 data work.fmts;
   length fmtname $32;
   fmtname="&fmtname";
+  type='N';
   do start=1 to 10;
     label= cats("&fmtname",start);
+    end=start;
     output;
   end;
 run;
 proc sort data=work.fmts nodupkey;
-  by fmtname;
+  by fmtname type start;
 run;
 proc format cntlin=work.fmts library=dctest.dcfmts;
 run;
@@ -157,8 +159,9 @@ data work.inquery3;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
+  RAW_VALUE="'&fmtname'";
 datalines4;
-AND,AND,1,FMTNAME,CONTAINS,"'&fmtname'"
+AND,AND,1,FMTNAME,CONTAINS,placeholder (see line above)
 ;;;;
 run;
 %mp_filterstore(

sas/sasjs/services/editors/getxlmaps.sas

@@ -0,0 +1,82 @@
+/**
+  @file getxlmaps.sas
+  @brief Returns a list of rules and other info for a specific xlmap_id
+
+  <h4> Service Inputs </h4>
+
+  <h5> getxlmaps_in </h5>
+
+  |XLMAP_ID|
+  |---|
+  |Sample|
+
+  <h4> Service Outputs </h4>
+
+  <h5> xlmaprules </h5>
+
+  Filtered output of the dc.MPE_XLMAP_RULES table
+
+  |XLMAP_ID|XLMAP_RANGE_ID|XLMAP_SHEET|XLMAP_START|XLMAP_FINISH|
+  |---|---|---|---|---|
+  |Sample|Range1|Sheet1|ABSOLUTE A1| |
+  |Sample|Range2|Sheet1|RELATIVE R[2]C[2]|ABSOLUTE H11|
+
+  <h5> xlmapinfo </h5>
+  Extra info for a map id
+
+  |TARGET_DS|
+  |---|
+  |DCXXX.MPE_XLMAP_DATA|
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mpeinit.sas
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%mpeinit()
+
+data _null_;
+  set work.getxlmaps_in;
+  putlog (_all_)(=);
+  call symputx('xlmap_id',xlmap_id);
+run;
+
+proc sql noprint;
+create table work.xlmaprules as
+  select xlmap_id
+    ,XLMAP_RANGE_ID
+    ,XLMAP_SHEET
+    ,XLMAP_START
+    ,XLMAP_FINISH
+  from &mpelib..MPE_XLMAP_RULES
+  where &dc_dttmtfmt. lt tx_to and xlmap_id="&xlmap_id"
+  order by xlmap_sheet, xlmap_range_id;
+
+%global target_ds;
+select XLMAP_TARGETLIBDS into: target_ds
+  from &mpelib..MPE_XLMAP_INFO
+  where &dc_dttmtfmt. lt tx_to and xlmap_id="&xlmap_id";
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program..sas
+  ,msg=%str(syscc=&syscc)
+)
+
+data work.xlmapinfo;
+  target_ds=coalescec("&target_ds","&mpelib..MPE_XLMAP_DATA");
+  output;
+  stop;
+run;
+
+%webout(OPEN)
+%webout(OBJ,xlmaprules)
+%webout(OBJ,xlmapinfo)
+%webout(CLOSE)
+

sas/sasjs/services/editors/getxlmaps.test.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief testing getxlmaps service
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mx_testservice.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+
+
+**/
+
+%let _program=&appLoc/services/editors/getxlmaps;
+
+/**
+  * Test 1 - basic send
+  */
+
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'XLMAP_ID:$char12.';
+  put "BASEL-KM1";
+run;
+
+%mx_testservice(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:getxlmaps_in,
+  outlib=web1,
+  mdebug=&sasjs_mdebug
+)
+
+data work.xlmaprules;
+  set web1.xlmaprules;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.xlmaprules,
+  test=ATLEAST 2,
+  desc=Checking successful return of at least 2 rules for the BASEL-KM1 map,
+  outds=work.test_results
+)
+
+
+/**
+  * Test 2 - info returned
+  */
+data work.xlmapinfo;
+  set web1.xlmapinfo;
+  putlog (_all_)(=);
+  call symputx('tgtds',target_ds);
+run;
+%mp_assert(
+  iftrue=(&tgtds=&dclib..MPE_XLMAP_DATA),
+  desc=Checking correct target table is returned,
+  outds=work.test_results
+)

sas/sasjs/services/editors/stagedata.test.2.sas

@@ -24,6 +24,7 @@ proc format lib=DCTEST.DCFMTS cntlout=work.fmtextract;
 run;
 data work.jsdata;
   set work.fmtextract;
+  fmtrow=_n_;
   if _n_<5 then _____DELETE__THIS__RECORD_____='Yes';
   else _____DELETE__THIS__RECORD_____='No';
   if _n_>20 then stop;

sas/sasjs/services/hooks/mpe_xlmap_info_postedit.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_XLMAP_INFO table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li dc_assignlib.sas
+  @li mp_validatecol.sas
+
+**/
+
+data work.staging_ds;
+  set work.staging_ds;
+
+  /* apply the first excel map to all cells */
+  length tgtds $41;
+  retain tgtds;
+  drop tgtds is_libds;
+  if _n_=1 then do;
+    if missing(XLMAP_TARGETLIBDS) then tgtds="&dc_libref..MPE_XLMAP_DATA";
+    else tgtds=upcase(XLMAP_TARGETLIBDS);
+    %mp_validatecol(XLMAP_TARGETLIBDS,LIBDS,is_libds)
+    call symputx('tgtds',tgtds);
+    call symputx('is_libds',is_libds);
+  end;
+  XLMAP_TARGETLIBDS=tgtds;
+
+run;
+
+%mp_abort(iftrue=(&is_libds ne 1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=Invalid target dataset (&tgtds)
+)
+
+/**
+  * make sure that the supplied target dataset exists and
+  * has the necessary columns
+  */
+%dc_assignlib(READ,%scan(&tgtds,1,.))
+
+%mp_abort(iftrue=(%mf_existds(libds=&tgtds) ne 1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=Target dataset (&tgtds) could not be opened
+)
+
+%let tgtvars=%upcase(%mf_getvarlist(&tgtds));
+%let srcvars=%upcase(%mf_getvarlist(&dc_libref..MPE_XLMAP_DATA));
+%let badvars1=%mf_wordsInStr1ButNotStr2(Str1=&srcvars,Str2=&tgtvars);
+%let badvars2=%mf_wordsInStr1ButNotStr2(Str1=&tgtvars,Str2=&srcvars);
+
+%mp_abort(iftrue=(%length(&badvars1.X)>1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=%str(Target dataset (&tgtds) has missing vars: &badvars1)
+)
+
+%mp_abort(iftrue=(%length(&badvars2.X)>1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=%str(Target dataset (&tgtds) has unrecognised vars: &badvars2)
+)

sas/sasjs/services/hooks/mpe_xlmap_rules_postedit.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_XLMAP_RULES table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+
+**/
+
+data work.staging_ds;
+  set work.staging_ds;
+
+  /* ensure uppercasing */
+  XLMAP_ID=upcase(XLMAP_ID);
+
+run;
+

sas/sasjs/services/hooks/sample_xlmap_data_postapprove.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Sample XLMAP Data hook program (sample_xlmap_data_postapprove)
+  @details This hook script should NOT be modified in place, as the changes
+    would be lost in your next Data Controller deployment.
+    Instead, create a copy of this hook script and place it OUTSIDE the
+    Data Controller metadata folder.
+
+  Available macro variables:
+    @li LOAD_REF - The Load Reference (unique upload id)
+    @li ORIG_LIBDS - The target library.dataset that was just loaded
+
+**/
+
+
+data _null_;
+  set work.staging_ds;
+  putlog 'load ref is in the staged data: ' load_ref;
+  stop;
+run;
+
+%put the unique identifier (LOAD_REF) is also a macro variable:  &LOAD_REF;

sas/sasjs/services/hooks/sample_xlmap_data_postedit.sas

@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Sample XLMAP Data hook program
+  @details This hook script should NOT be modified in place, as the changes
+    would be lost in your next Data Controller deployment.
+    Instead, create a copy of this hook script and place it OUTSIDE the
+    Data Controller metadata folder.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The target dataset being loaded
+
+**/
+
+%let abort=0;
+%let errmsg=;
+
+data work.staging_ds;
+  set work.staging_ds;
+  length errmsg $1000;
+  drop err:;
+  /* KM1 validations */
+  if XLMAP_ID='BASEL-KM1' then do;
+    if XLMAP_RANGE_ID='KM1:a' & input(value_txt,8.)<100 then do;
+      errmsg='Should be greater than 100';
+      err=1;
+    end;
+  end;
+  /* CR2 Validations */
+  if XLMAP_ID='BASEL-CR2' then do;
+    if XLMAP_RANGE_ID='CR2-sec1' & row_no=3 & input(value_txt,8.)>0 then do;
+      errmsg='Should be negative';
+      err=1;
+    end;
+  end;
+
+  /* publish error message */
+  if err=1 then do;
+    errmsg=catx(' ',xlmap_range_id,':',value_txt,'->',errmsg);
+    call symputx('errmsg',errmsg);
+    call symputx('abort',1);
+  end;
+run;
+
+%mp_abort(iftrue=(&abort ne 0)
+  ,mac=xlmap_data_postedit
+  ,msg=%superq(errmsg)
+)

sas/sasjs/services/public/startupservice.sas

@@ -3,10 +3,8 @@
   @brief List the libraries and tables the mp-editor user can access
   @details If user is in a control group (&mpeadmins, configured in mpeinit.sas)
     then they have access to all libraries / tables.  Otherwise a join is made
-    to the &mpelib..mp_editor_access table.
+    to the &mpelib..mpe_security table.
 
-    This service is also callable from EUCs - just add EUCDLM= parameter.
-    EUCDLM values:  TAB or CSV
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -129,10 +127,26 @@ create table saslibs as
   ,msg=%str(issue with security validation)
 )
 
+proc sql;
+create table work.xlmaps as
+  select distinct a.XLMAP_ID
+    ,b.XLMAP_DESCRIPTION
+    ,coalescec(b.XLMAP_TARGETLIBDS,"&mpelib..MPE_XLMAP_DATA")
+      as XLMAP_TARGETLIBDS
+  from &mpelib..MPE_XLMAP_RULES a
+  left join &mpelib..MPE_XLMAP_INFO(where=(&dc_dttmtfmt. lt tx_to)) b
+  on a.XLMAP_ID=b.XLMAP_ID
+  where &dc_dttmtfmt. lt a.tx_to;
+
+/* we don't want the XLMAP target datasets to be directly editable */
+delete from sasdatasets
+  where cats(libref,'.',dsn) in (select XLMAP_TARGETLIBDS from xlmaps);
+
 %webout(OPEN)
 %webout(OBJ,sasDatasets)
 %webout(OBJ,saslibs)
 %webout(OBJ,globvars)
+%webout(ARR,xlmaps)
 %webout(CLOSE)
 
 %mpeterm()

sas/sasjs/services/public/startupservice.test.sas

@@ -16,13 +16,24 @@
 )
 
 
-data globvars;
+data work.globvars;
   set webout.globvars;
   putlog (_all_)(=);
 run;
 
+data work.xlmaps;
+  set webout.xlmaps;
+  putlog (_all_)(=);
+run;
+
 %mp_assertdsobs(work.globvars,
   desc=Fromsas table returned,
   test=HASOBS,
   outds=work.test_results
 )
+
+%mp_assertdsobs(work.xlmaps,
+  desc=xlmaps table returned,
+  test=HASOBS,
+  outds=work.test_results
+)

sas/sasjs/services/public/viewdata.sas

@@ -44,13 +44,13 @@
   @li mf_verifymacvars.sas
   @li mp_abort.sas
   @li mp_cntlout.sas
-  @li mp_dsmeta.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
   @li mp_jsonout.sas
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_dsmeta.sas
   @li mpe_filtermaster.sas
 
 
@@ -351,7 +351,7 @@ run;
 
 %mp_getcols(&libds, outds=cols)
 
-%mp_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&libds, outds=dsmeta)
 
 %webout(OPEN)
 %webout(OBJ,cls_rules)