feat: Complex Excel Uploads

Reviewed-on: #71

Closes #69

.vscode/settings.json

@@ -1,18 +1,19 @@
 {
-    "cSpell.words": [
-        "SYSERRORTEXT",
-        "SYSWARNINGTEXT"
-    ],
-    "editor.rulers": [
-        80
-    ],
-    "files.trimTrailingWhitespace": true,
-    "[markdown]": {
-        "files.trimTrailingWhitespace": false
-    },
-    "workbench.colorCustomizations": {
-        "titleBar.activeForeground": "#ebe8e8",
-        "titleBar.activeBackground": "#95ff0053",
-    },
-    "terminal.integrated.wordSeparators": " ()[]{}',\"`─‘’"
-}
+  "cSpell.words": [
+    "Licence",
+    "SYSERRORTEXT",
+    "SYSWARNINGTEXT",
+    "xlmaprules",
+    "xlmaps"
+  ],
+  "editor.rulers": [80],
+  "files.trimTrailingWhitespace": true,
+  "[markdown]": {
+    "files.trimTrailingWhitespace": false
+  },
+  "workbench.colorCustomizations": {
+    "titleBar.activeForeground": "#ebe8e8",
+    "titleBar.activeBackground": "#95ff0053"
+  },
+  "terminal.integrated.wordSeparators": " ()[]{}',\"`─‘’"
+}

client/src/app/_globals.ts

@@ -37,6 +37,12 @@ export const initFilter: { filter: FilterCache } = {
   }
 }
 
+export interface XLMapListItem {
+  id: string
+  description: string
+  targetDS: string
+}
+
 /**
  * Cached filtering values across whole app (editor, viewer, viewboxes)
  * Cached lineage libraries, tables
@@ -46,6 +52,8 @@ export const initFilter: { filter: FilterCache } = {
  */
 export const globals: {
   rootParam: string
+  dcLib: string
+  xlmaps: XLMapListItem[]
   editor: any
   viewer: any
   viewboxes: ViewboxCache
@@ -57,11 +65,13 @@ export const globals: {
   [key: string]: any
 } = {
   rootParam: <string>'',
+  dcLib: '',
+  xlmaps: [],
   editor: {
     startupSet: <boolean>false,
     treeNodeLibraries: <any[] | null>[],
     libsAndTables: <any[]>[],
-    libraries: <String[] | undefined>[],
+    libraries: <string[] | undefined>[],
     library: <string>'',
     table: <string>'',
     filter: <FilterCache>{

client/src/app/app.component.html

@@ -168,7 +168,7 @@
           </button>
           <clr-dropdown-menu *clrIfOpen clrPosition="bottom-left">
             <a [routerLink]="['/view']" clrDropdownItem>VIEW</a>
-            <a [routerLink]="['/home']" clrDropdownItem>EDIT</a>
+            <a [routerLink]="['/home']" clrDropdownItem>LOAD</a>
             <a [routerLink]="['/review/submitted']" clrDropdownItem>REVIEW</a>
           </clr-dropdown-menu>
         </clr-dropdown>
@@ -189,7 +189,7 @@
             router.url.includes('edit-record') ||
             router.url.includes('home')
           "
-          >EDIT</a
+          >LOAD</a
         >
         <a
           [routerLink]="['/review/submitted']"

client/src/app/app.routing.ts

@@ -4,19 +4,19 @@
  * The full license information can be found in LICENSE in the root directory of this project.
  */
 import { ModuleWithProviders } from '@angular/core'
-import { Routes, RouterModule } from '@angular/router'
+import { RouterModule, Routes } from '@angular/router'
 
-import { HomeComponent } from './home/home.component'
 import { NotFoundComponent } from './not-found/not-found.component'
 
+import { DeployModule } from './deploy/deploy.module'
+import { EditorModule } from './editor/editor.module'
+import { HomeModule } from './home/home.module'
+import { LicensingModule } from './licensing/licensing.module'
+import { ReviewModule } from './review/review.module'
 import { ReviewRouteComponent } from './routes/review-route/review-route.component'
 import { StageModule } from './stage/stage.module'
-import { EditorModule } from './editor/editor.module'
-import { ViewerModule } from './viewer/viewer.module'
-import { ReviewModule } from './review/review.module'
-import { DeployModule } from './deploy/deploy.module'
-import { LicensingModule } from './licensing/licensing.module'
 import { SystemModule } from './system/system.module'
+import { ViewerModule } from './viewer/viewer.module'
 
 /**
  * Defining routes
@@ -45,7 +45,7 @@ export const ROUTES: Routes = [
     path: 'licensing',
     loadChildren: () => LicensingModule
   },
-  { path: 'home', component: HomeComponent },
+  { path: 'home', loadChildren: () => HomeModule },
   {
     /**
      * Load editor module with subroutes

client/src/app/editor/editor.component.ts

@@ -38,7 +38,7 @@ import { HotTableInterface } from '../models/HotTable.interface'
 import {
   $DataFormats,
   DSMeta,
-  EditorsGetdataServiceResponse
+  EditorsGetDataServiceResponse
 } from '../models/sas/editors-getdata.model'
 import { DataFormat } from '../models/sas/common/DateFormat'
 import SheetInfo from '../models/SheetInfo'
@@ -2964,7 +2964,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
       await this.sasStoreService
         .callService(myParams, 'SASControlTable', 'editors/getdata', this.libds)
-        .then((res: EditorsGetdataServiceResponse) => {
+        .then((res: EditorsGetDataServiceResponse) => {
           this.initSetup(res)
         })
         .catch((err: any) => {
@@ -2976,7 +2976,7 @@ export class EditorComponent implements OnInit, AfterViewInit {
 
   ngAfterViewInit() {}
 
-  initSetup(response: EditorsGetdataServiceResponse) {
+  initSetup(response: EditorsGetDataServiceResponse) {
     this.hotInstance = this.hotRegisterer.getInstance('hotInstance')
 
     if (this.getdataError) return

client/src/app/home/home-routing.module.ts

@@ -0,0 +1,23 @@
+import { NgModule } from '@angular/core'
+import { RouterModule, Routes } from '@angular/router'
+import { HomeRouteComponent } from '../routes/home-route/home-route.component'
+import { HomeComponent } from './home.component'
+import { XLMapModule } from '../xlmap/xlmap.module'
+
+const routes: Routes = [
+  {
+    path: '',
+    component: HomeRouteComponent,
+    children: [
+      { path: '', pathMatch: 'full', redirectTo: 'tables' },
+      { path: 'tables', component: HomeComponent },
+      { path: 'files', loadChildren: () => XLMapModule }
+    ]
+  }
+]
+
+@NgModule({
+  imports: [RouterModule.forChild(routes)],
+  exports: [RouterModule]
+})
+export class HomeRoutingModule {}

client/src/app/home/home.module.ts

@@ -1,15 +1,18 @@
-import { NgModule } from '@angular/core'
 import { CommonModule } from '@angular/common'
-import { HomeComponent } from './home.component'
-import { ClarityModule } from '@clr/angular'
+import { NgModule } from '@angular/core'
 import { FormsModule } from '@angular/forms'
+import { ClarityModule } from '@clr/angular'
 import { AppSharedModule } from '../app-shared.module'
-import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
 import { DirectivesModule } from '../directives/directives.module'
+import { HomeRouteComponent } from '../routes/home-route/home-route.component'
+import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
+import { HomeRoutingModule } from './home-routing.module'
+import { HomeComponent } from './home.component'
 
 @NgModule({
-  declarations: [HomeComponent],
+  declarations: [HomeComponent, HomeRouteComponent],
   imports: [
+    HomeRoutingModule,
     FormsModule,
     ClarityModule,
     AppSharedModule,

client/src/app/models/sas/editors-getdata.model.ts

@@ -4,12 +4,12 @@ import { DQData, SASParam } from '../TableData'
 import { BaseSASResponse } from './common/BaseSASResponse'
 import { DataFormat } from './common/DateFormat'
 
-export interface EditorsGetdataServiceResponse {
-  data: EditorsGetdataSASResponse
+export interface EditorsGetDataServiceResponse {
+  data: EditorsGetDataSASResponse
   libds: string
 }
 
-export interface EditorsGetdataSASResponse extends BaseSASResponse {
+export interface EditorsGetDataSASResponse extends BaseSASResponse {
   $sasdata: $DataFormats
   sasdata: Sasdata[]
   sasparams: SASParam[]

client/src/app/routes/home-route/home-route.component.html

@@ -0,0 +1 @@
+<router-outlet></router-outlet>

client/src/app/routes/home-route/home-route.component.ts

@@ -0,0 +1,17 @@
+import { Component, OnInit, OnDestroy } from '@angular/core'
+
+@Component({
+  selector: 'app-home-route',
+  templateUrl: './home-route.component.html',
+  styleUrls: ['./home-route.component.scss'],
+  host: {
+    class: 'content-container'
+  }
+})
+export class HomeRouteComponent implements OnInit, OnDestroy {
+  constructor() {}
+
+  ngOnInit() {}
+
+  ngOnDestroy() {}
+}

client/src/app/routes/xlmap-route/xlmap-route.component.html

@@ -0,0 +1 @@
+<router-outlet></router-outlet>

client/src/app/routes/xlmap-route/xlmap-route.component.ts

@@ -0,0 +1,17 @@
+import { Component, OnInit, OnDestroy } from '@angular/core'
+
+@Component({
+  selector: 'app-xlmap-route',
+  templateUrl: './xlmap-route.component.html',
+  styleUrls: ['./xlmap-route.component.scss'],
+  host: {
+    class: 'content-container'
+  }
+})
+export class XLMapRouteComponent implements OnInit, OnDestroy {
+  constructor() {}
+
+  ngOnInit() {}
+
+  ngOnDestroy() {}
+}

client/src/app/services/app.service.ts

@@ -74,6 +74,7 @@ export class AppService {
           missingProps.push('Globvars')
         if (!res.sasdatasets) missingProps.push('Sasdatasets')
         if (!res.saslibs) missingProps.push('Saslibs')
+        if (!res.xlmaps) missingProps.push('XLMaps')
 
         if (missingProps.length > 0) {
           startupServiceError = true
@@ -135,10 +136,17 @@ export class AppService {
           globals.editor.libsAndTables = libsAndTables
         }
 
+        globals.xlmaps = res.xlmaps.map((xlmap: any) => ({
+          id: xlmap[0],
+          description: xlmap[1],
+          targetDS: xlmap[2]
+        }))
         globals.editor.treeNodeLibraries = treeNodeLibraries
         globals.editor.libraries = libraries
         globals.editor.startupSet = true
 
+        globals.dcLib = res.globvars[0].DCLIB
+
         await this.licenceService.activation(res)
       })
       .catch((err: any) => {

client/src/app/services/sas-store.service.ts

@@ -10,8 +10,8 @@ import { globals } from '../_globals'
 import { FilterClause, FilterGroup, FilterQuery } from '../models/FilterQuery'
 import {
   $DataFormats,
-  EditorsGetdataSASResponse,
-  EditorsGetdataServiceResponse
+  EditorsGetDataSASResponse,
+  EditorsGetDataServiceResponse
 } from '../models/sas/editors-getdata.model'
 import { LoggerService } from './logger.service'
 import { isSpecialMissing } from '@sasjs/utils/input/validators'
@@ -57,13 +57,13 @@ export class SasStoreService {
     libds: string
   ) {
     this.libds = libds
-    let tables: any = {}
+    const tables: any = {}
     tables[tableName] = [tableData]
-    let res: EditorsGetdataSASResponse = await this.sasService.request(
+    const res: EditorsGetDataSASResponse = await this.sasService.request(
       program,
       tables
     )
-    let response: EditorsGetdataServiceResponse = {
+    const response: EditorsGetDataServiceResponse = {
       data: res,
       libds: this.libds
     }
@@ -209,6 +209,14 @@ export class SasStoreService {
     return res
   }
 
+  public async getXLMapRules(id: string) {
+    const tables = {
+      getxlmaps_in: [{ XLMAP_ID: id }]
+    }
+    const res: any = await this.sasService.request('editors/getxlmaps', tables)
+    return res
+  }
+
   public async getDetails(tableData: any, tableName: string, program: string) {
     let tables: any = {}
     tables[tableName] = [tableData]

client/src/app/shared/sidebar/sidebar.component.html

@@ -107,7 +107,29 @@
       </clr-tab-content>
     </clr-tab>
   </clr-tabs>
-  <p *ngIf="isMainRoute('home')" class="page-title">Edit</p>
+
+  <div
+    *ngIf="isMainRoute('home')"
+    class="d-flex justify-content-center sub-dropdown"
+  >
+    <clr-dropdown>
+      <button class="dropdown-toggle btn btn-link" clrDropdownTrigger>
+        {{ getSubPage() }}
+        <clr-icon shape="caret down"></clr-icon>
+      </button>
+      <clr-dropdown-menu *clrIfOpen>
+        <a
+          clrVerticalNavLink
+          routerLink="/home/tables"
+          routerLinkActive="active"
+          >Tables</a
+        >
+        <a clrVerticalNavLink routerLink="/home/files" routerLinkActive="active"
+          >Files</a
+        >
+      </clr-dropdown-menu>
+    </clr-dropdown>
+  </div>
 
   <div class="nav-divider"></div>
 

client/src/app/stage/stage.component.ts

@@ -7,6 +7,7 @@ import { EventService } from '../services/event.service'
 import { AppService } from '../services/app.service'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { LicenceService } from '../services/licence.service'
+import { globals } from '../_globals'
 
 @Component({
   selector: 'app-stage',
@@ -55,7 +56,15 @@ export class StageComponent implements OnInit {
   }
 
   public goBack() {
-    this.route.navigateByUrl('/editor/' + this.tableDetails.BASE_TABLE)
+    const xlmap = globals.xlmaps.find(
+      (xlmap) => xlmap.targetDS === this.tableDetails.BASE_TABLE
+    )
+    if (xlmap) {
+      const id = this.hotTable.data[0].XLMAP_ID
+      this.route.navigateByUrl('/home/files/' + id)
+    } else {
+      this.route.navigateByUrl('/editor/' + this.tableDetails.BASE_TABLE)
+    }
   }
 
   public download(id: any) {

client/src/app/xlmap/tests/xl.utils.spec.ts

@@ -0,0 +1,159 @@
+import {
+  extractRowAndCol,
+  getCellAddress,
+  getFinishingCell,
+  isBlankRow
+} from '../utils/xl.utils'
+
+describe('isBlankRow', () => {
+  it('should return true for a blank row', () => {
+    const blankRow = { __rowNum__: 1 }
+    expect(isBlankRow(blankRow)).toBeTrue()
+  })
+
+  it('should return false for a non-blank row', () => {
+    const nonBlankRow = {
+      B: 3,
+      C: 'some value',
+      D: -203
+    }
+    expect(isBlankRow(nonBlankRow)).toBeFalse()
+  })
+})
+
+describe('extractRowAndCol', () => {
+  it('should extract row and column from "MATCH F R[2]C[0]: CASH BALANCE"', () => {
+    const input = 'MATCH F R[2]C[0]: CASH BALANCE'
+    const result = extractRowAndCol(input)
+    expect(result).toEqual({ row: 2, column: 0 })
+  })
+
+  it('should extract row and column from "RELATIVE R[10]C[6]"', () => {
+    const input = 'RELATIVE R[10]C[6]'
+    const result = extractRowAndCol(input)
+    expect(result).toEqual({ row: 10, column: 6 })
+  })
+
+  it('should return null for invalid input', () => {
+    const invalidInput = 'INVALID INPUT'
+    const result = extractRowAndCol(invalidInput)
+    expect(result).toBeNull()
+  })
+})
+
+describe('getCellAddress', () => {
+  const arrayOfObjects = [
+    { A: 'valueA1', B: 'valueB1' },
+    { A: 'valueA2', B: 'valueB2' }
+  ]
+
+  it('should convert "ABSOLUTE D8" to A1-style address', () => {
+    const input = 'ABSOLUTE D8'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('D8')
+  })
+
+  it('should convert "RELATIVE R[10]C[6]" to A1-style address', () => {
+    const input = 'RELATIVE R[10]C[6]'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('F10')
+  })
+
+  it('should convert "MATCH 1 R[0]C[0]:valueA1" to A1-style address', () => {
+    const input = 'MATCH 1 R[0]C[0]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('A1')
+  })
+
+  it('should convert "MATCH A R[0]C[0]:valueA1" to A1-style address', () => {
+    const input = 'MATCH A R[0]C[0]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('A1')
+  })
+
+  it('should convert "MATCH 1 R[1]C[0]:valueA1" to A1-style address', () => {
+    const input = 'MATCH 1 R[1]C[0]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('A2')
+  })
+
+  it('should convert "MATCH A R[0]C[1]:valueA1" to A1-style address', () => {
+    const input = 'MATCH A R[0]C[1]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('B1')
+  })
+
+  it('should convert "MATCH 1 R[1]C[1]:valueA1" to A1-style address', () => {
+    const input = 'MATCH 1 R[1]C[1]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('B2')
+  })
+
+  it('should convert "MATCH A R[1]C[1]:valueA1" to A1-style address', () => {
+    const input = 'MATCH A R[1]C[1]:valueA1'
+    const result = getCellAddress(input, arrayOfObjects)
+    expect(result).toBe('B2')
+  })
+})
+
+describe('getFinishingCell', () => {
+  const arrayOfObjects = [
+    { A: 'valueA1', B: 'valueB1' },
+    { A: 'valueA2', B: 'valueB2' },
+    { A: 'valueA3', B: 'valueB3' },
+    { B: 'valueB4' },
+    { A: 'valueA5' },
+    { A: 'valueA6', B: 'valueB6' },
+    {},
+    { A: 'valueA8' }
+  ]
+
+  it('should return the start cell if finish is an empty string', () => {
+    const start = 'A1'
+    const finish = ''
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe(start)
+  })
+
+  it('should convert "ABSOLUTE D8" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'ABSOLUTE D8'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('D8')
+  })
+
+  it('should convert "RELATIVE R[2]C[1]" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'RELATIVE R[2]C[1]'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B3')
+  })
+
+  it('should convert "MATCH A R[0]C[1]:valueA1" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'MATCH A R[0]C[1]:valueA1'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B1')
+  })
+
+  it('should convert "MATCH 1 R[4]C[0]:valueB1" to A1-style address', () => {
+    const start = 'A1'
+    const finish = 'MATCH 1 R[4]C[0]:valueB1'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B5')
+  })
+
+  it('should convert "LASTDOWN" to A1-style address of the last non-blank cell in column A', () => {
+    const start = 'A1'
+    const finish = 'LASTDOWN'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('A3')
+  })
+
+  it('should convert "BLANKROW" to A1-style address of the last row with blank cells', () => {
+    const start = 'A1'
+    const finish = 'BLANKROW'
+    const result = getFinishingCell(start, finish, arrayOfObjects)
+    expect(result).toBe('B6')
+  })
+})

client/src/app/xlmap/utils/file.utils.ts

@@ -0,0 +1,31 @@
+export const blobToFile = (blob: Blob, fileName: string): File => {
+  const file = new File([blob], fileName, {
+    lastModified: new Date().getTime()
+  })
+  return file
+}
+
+/**
+ * Convert an array of bytes (Uint8Array) to a binary string.
+ * @param {Uint8Array} res - The array of bytes to convert.
+ * @returns {string} The binary string representation of the array of bytes.
+ */
+export const byteArrayToBinaryString = (res: Uint8Array): string => {
+  // Create a Uint8Array from the input array (if it's not already)
+  const bytes = new Uint8Array(res)
+
+  // Initialize an empty string to store the binary representation
+  let binary = ''
+
+  // Get the length of the byte array
+  const length = bytes.byteLength
+
+  // Iterate through each byte in the array
+  for (let i = 0; i < length; i++) {
+    // Convert each byte to its binary representation and append to the string
+    binary += String.fromCharCode(bytes[i])
+  }
+
+  // Return the binary string
+  return binary
+}

client/src/app/xlmap/utils/xl.utils.ts

@@ -0,0 +1,225 @@
+import * as XLSX from '@sheet/crypto'
+
+/**
+ * Checks if an excel row is blank or not
+ *
+ * @param row object is of shape {[key: string]: any}
+ */
+export const isBlankRow = (row: any) => {
+  for (const key in row) {
+    if (key !== '__rowNum__') {
+      return false
+    }
+  }
+  return true
+}
+
+/**
+ * Extracts row and column number from xlmap rule.
+ *
+ * Input string should be in form of
+ * either "MATCH F R[2]C[0]: CASH BALANCE" or "RELATIVE R[10]C[6]"
+ */
+export const extractRowAndCol = (str: string) => {
+  // Regular expression to match and capture the values inside square brackets
+  const regex = /R\[(\d+)\]C\[(\d+)\]/
+
+  // Match the regular expression against the input string
+  const match = str.match(regex)
+
+  if (!match) {
+    return null
+  }
+
+  // Extract values from the match groups
+  const row = parseInt(match[1], 10)
+  const column = parseInt(match[2], 10)
+
+  return {
+    row,
+    column
+  }
+}
+
+/**
+ * Generate an A1-Style excel cell address from xlmap rule.
+ *
+ * Expect "ABSOLUTE D8" or "RELATIVE R[10]C[6]" or
+ * "MATCH C R[0]C[4]:Common Equity Tier 1 (CET1)" kinds of string as rule input
+ */
+export const getCellAddress = (rule: string, arrayOfObjects: any[]) => {
+  if (rule.startsWith('ABSOLUTE ')) {
+    rule = rule.replace('ABSOLUTE ', '')
+  }
+
+  if (rule.startsWith('RELATIVE ')) {
+    const rowAndCol = extractRowAndCol(rule)
+
+    if (rowAndCol) {
+      const { row, column } = rowAndCol
+
+      // Generate an A1-Style address string from a SheetJS cell address
+      // Spreadsheet applications typically display ordinal row numbers,
+      // where 1 is the first row, 2 is the second row, etc. The numbering starts at 1.
+      // SheetJS follows JavaScript counting conventions,
+      // where 0 is the first row, 1 is the second row, etc. The numbering starts at 0.
+      // Therefore, we have to subtract 1 from row and column to match SheetJS indexing convention
+      rule = XLSX.utils.encode_cell({ r: row - 1, c: column - 1 })
+    }
+  }
+
+  if (rule.startsWith('MATCH ')) {
+    let targetValue = ''
+
+    // using a regular expression to match "C[x]:" and extract the value after it
+    const match = rule.match(/C\[\d+\]:(.+)/)
+
+    // Check if there is a match
+    if (match) {
+      // Extract the value after "C[x]:"
+      targetValue = match[1]
+    }
+
+    // Split the string by spaces to get target row/column
+    const splittedArray = rule.split(' ')
+
+    // Extract the second word
+    const secondWord = splittedArray[1]
+
+    let targetColumn = ''
+    let targetRow = -1
+    let cellAddress = ''
+
+    // Check if the secondWord is a number
+    if (!isNaN(Number(secondWord))) {
+      targetRow = parseInt(secondWord)
+    } else {
+      targetColumn = secondWord
+    }
+
+    if (targetRow !== -1) {
+      // sheetJS index starts from 0,
+      // therefore, decremented 1 to make it correct row address for js array
+      const row = arrayOfObjects[targetRow - 1]
+      for (const col in row) {
+        if (col !== '__rowNum__' && row[col] === targetValue) {
+          cellAddress = col + targetRow
+          break
+        }
+      }
+    } else {
+      for (let i = 0; i < arrayOfObjects.length; i++) {
+        const row = arrayOfObjects[i]
+        if (row[targetColumn] === targetValue) {
+          // sheetJS index starts from 0,
+          // therefore, incremented 1 to make it correct row address
+          const rowIndex = i + 1
+          cellAddress = targetColumn + rowIndex
+          break
+        }
+      }
+    }
+
+    // Converts A1 cell address to 0-indexed form
+    const matchedCellAddress = XLSX.utils.decode_cell(cellAddress)
+
+    // extract number of rows and columns that we have to move
+    // from matched cell to reach target cell
+    const rowAndCol = extractRowAndCol(rule)
+
+    if (rowAndCol) {
+      const { row, column } = rowAndCol
+
+      // Converts 0-indexed cell address to A1 form
+      rule = XLSX.utils.encode_cell({
+        r: matchedCellAddress.r + row,
+        c: matchedCellAddress.c + column
+      })
+    }
+  }
+
+  return rule
+}
+
+/**
+ * Generate an A1-Style excel cell address for last cell
+ *
+ * @param start A1 style excel cell address
+ * @param finish XLMAP_FINISH attribute of xlmap rule
+ * @param arrayOfObjects an array of row objects
+ * @returns
+ */
+export const getFinishingCell = (
+  start: string,
+  finish: string,
+  arrayOfObjects: any[]
+) => {
+  // in this case an individual cell would be extracted
+  if (finish === '') {
+    return start
+  }
+
+  if (finish.startsWith('ABSOLUTE ')) {
+    finish = finish.replace('ABSOLUTE ', '')
+  }
+
+  if (finish.startsWith('RELATIVE ')) {
+    const rowAndCol = extractRowAndCol(finish)
+    if (rowAndCol) {
+      const { row, column } = rowAndCol
+
+      const { r, c } = XLSX.utils.decode_cell(start)
+
+      // finish is relative to starting point
+      // therefore, we need to add extracted row and columns
+      // in starting cell address to get actual finishing cell
+      finish = XLSX.utils.encode_cell({ r: r + row, c: c + column })
+    }
+  }
+
+  if (finish.startsWith('MATCH ')) {
+    finish = getCellAddress(finish, arrayOfObjects)
+  }
+
+  if (finish === 'LASTDOWN') {
+    const { r, c } = XLSX.utils.decode_cell(start)
+    const colName = XLSX.utils.encode_col(c)
+    let lastNonBlank = r
+    for (let i = r + 1; i < arrayOfObjects.length; i++) {
+      const row = arrayOfObjects[i]
+      if (!row[colName]) {
+        break
+      }
+      lastNonBlank = i
+    }
+    finish = colName + (lastNonBlank + 1) // excel numbering starts from 1. So incremented 1 to 0 based index
+  }
+
+  if (finish === 'BLANKROW') {
+    const { r } = XLSX.utils.decode_cell(start)
+    let lastNonBlankRow = r
+    for (let i = r + 1; i < arrayOfObjects.length; i++) {
+      const row = arrayOfObjects[i]
+      if (isBlankRow(row)) {
+        break
+      }
+      lastNonBlankRow = i
+    }
+    const row = arrayOfObjects[lastNonBlankRow]
+
+    // Get the keys of the object (excluding '__rowNum__')
+    const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
+
+    // Finding last column in a row
+    // Find the key with the highest alphanumeric value (assumes keys are letters)
+    const lastColumn = keys.reduce(
+      (maxKey, currentKey) => (currentKey > maxKey ? currentKey : maxKey),
+      ''
+    )
+
+    // make finishing cell address in A1 style
+    finish = lastColumn + (lastNonBlankRow + 1) // excel numbering starts from 1. So incremented 1 to 0 based index
+  }
+
+  return finish
+}

client/src/app/xlmap/xlmap-routing.module.ts

@@ -0,0 +1,22 @@
+import { NgModule } from '@angular/core'
+import { RouterModule, Routes } from '@angular/router'
+
+import { XLMapComponent } from '../xlmap/xlmap.component'
+import { XLMapRouteComponent } from '../routes/xlmap-route/xlmap-route.component'
+
+const routes: Routes = [
+  {
+    path: '',
+    component: XLMapRouteComponent,
+    children: [
+      { path: '', component: XLMapComponent },
+      { path: ':id', component: XLMapComponent }
+    ]
+  }
+]
+
+@NgModule({
+  imports: [RouterModule.forChild(routes)],
+  exports: [RouterModule]
+})
+export class XLMapRoutingModule {}

client/src/app/xlmap/xlmap.component.html

@@ -0,0 +1,252 @@
+<app-sidebar>
+  <div *ngIf="xlmapsLoading" class="my-10-mx-auto text-center">
+    <clr-spinner clrMedium></clr-spinner>
+  </div>
+
+  <clr-tree>
+    <clr-tree-node class="search-node">
+      <div class="tree-search-wrapper">
+        <input
+          clrInput
+          #searchXLMapTreeInput
+          placeholder="Filter by Id"
+          name="input"
+          [(ngModel)]="searchString"
+          (keyup)="xlmapListOnFilter()"
+          autocomplete="off"
+        />
+        <clr-icon
+          *ngIf="searchXLMapTreeInput.value.length < 1"
+          shape="search"
+        ></clr-icon>
+        <clr-icon
+          *ngIf="searchXLMapTreeInput.value.length > 0"
+          (click)="searchString = ''; xlmapListOnFilter()"
+          shape="times"
+        ></clr-icon>
+      </div>
+    </clr-tree-node>
+
+    <ng-container *ngFor="let xlmap of xlmaps">
+      <clr-tree-node>
+        <button
+          (click)="xlmapOnClick(xlmap)"
+          class="clr-treenode-link"
+          [class.table-active]="isActiveXLMap(xlmap.id)"
+        >
+          <clr-icon shape="file"></clr-icon>
+          {{ xlmap.id }}
+        </button>
+      </clr-tree-node>
+    </ng-container>
+  </clr-tree>
+</app-sidebar>
+
+<div class="content-area">
+  <div *ngIf="!selectedXLMap" class="no-table-selected">
+    <clr-icon
+      shape="warning-standard"
+      size="60"
+      class="is-info icon-dc-fill"
+    ></clr-icon>
+    <h3 *ngIf="xlmaps.length > 0" class="text-center color-gray">
+      Please select a map
+    </h3>
+    <h3 *ngIf="xlmaps.length < 1" class="text-center color-gray">
+      No excel map is found
+    </h3>
+  </div>
+
+  <div class="loadingSpinner" *ngIf="isLoading">
+    <span class="spinner"> Loading... </span>
+    <div>
+      <h4>{{ isLoadingDesc }}</h4>
+    </div>
+  </div>
+
+  <div
+    appDragNdrop
+    (fileDraggedOver)="onShowUploadModal()"
+    class="card h-100 d-flex clr-flex-column"
+    *ngIf="!isLoading && selectedXLMap"
+  >
+    <clr-tabs>
+      <clr-tab>
+        <button clrTabLink (click)="selectedTab = TabsEnum.Rules">Rules</button>
+        <clr-tab-content *clrIfActive="selectedTab === TabsEnum.Rules">
+        </clr-tab-content>
+      </clr-tab>
+      <clr-tab>
+        <button clrTabLink (click)="selectedTab = TabsEnum.Data">Data</button>
+        <clr-tab-content *clrIfActive="selectedTab === TabsEnum.Data">
+        </clr-tab-content>
+      </clr-tab>
+    </clr-tabs>
+
+    <ng-container *ngTemplateOutlet="actionButtons"></ng-container>
+
+    <div class="clr-row m-0 mb-10-i viewerTitle">
+      <h3 class="d-flex clr-col-12 clr-justify-content-center mt-5-i">
+        {{ selectedXLMap.id }}
+      </h3>
+      <i class="d-flex clr-col-12 clr-justify-content-center mt-5-i">{{
+        selectedXLMap.description
+      }}</i>
+      <h5 class="d-flex clr-col-12 clr-justify-content-center mt-5-i">
+        Rules Source:
+        <a class="ml-10" [routerLink]="'/view/data/' + rulesSource">
+          {{ rulesSource }}
+        </a>
+      </h5>
+      <h5 class="d-flex clr-col-12 clr-justify-content-center mt-5-i">
+        Target dataset:
+        <a class="ml-10" [routerLink]="'/view/data/' + selectedXLMap.targetDS">
+          {{ selectedXLMap.targetDS }}
+        </a>
+      </h5>
+    </div>
+
+    <div class="clr-flex-1">
+      <hot-table
+        hotId="hotInstance"
+        id="hot-table"
+        [multiColumnSorting]="true"
+        [viewportRowRenderingOffset]="50"
+        [data]="selectedTab === TabsEnum.Rules ? xlmapRules : xlData"
+        [colHeaders]="
+          selectedTab === TabsEnum.Rules ? xlmapRulesHeaders : xlUploadHeader
+        "
+        [columns]="
+          selectedTab === TabsEnum.Rules ? xlmapRulesColumns : xlUploadColumns
+        "
+        [filters]="true"
+        [height]="'100%'"
+        stretchH="all"
+        [modifyColWidth]="maxWidthChecker"
+        [cells]="getCellConfiguration"
+        [maxRows]="hotTableMaxRows"
+        [manualColumnResize]="true"
+        [rowHeaders]="rowHeaders"
+        [rowHeaderWidth]="15"
+        [rowHeights]="20"
+        [licenseKey]="hotTableLicenseKey"
+      >
+      </hot-table>
+    </div>
+  </div>
+
+  <clr-modal
+    appFileDrop
+    (fileOver)="fileOverBase($event)"
+    (fileDrop)="getFileDesc($event, true)"
+    [uploader]="uploader"
+    [clrModalSize]="'xl'"
+    [clrModalStaticBackdrop]="false"
+    [clrModalClosable]="true"
+    [(clrModalOpen)]="showUploadModal"
+    class="relative"
+  >
+    <h3 class="modal-title">Upload File</h3>
+    <div class="modal-body">
+      <div class="drop-area">
+        <span>Drop file anywhere to upload!</span>
+      </div>
+
+      <div class="clr-col-md-12">
+        <div class="clr-row card-block mt-15 d-flex justify-content-between">
+          <div class="clr-col-md-3 filterBtn">
+            <span class="filterBtn w-100">
+              <label
+                for="file-upload"
+                class="btn btn-sm btn-outline profile-buttons w-100"
+              >
+                Browse
+              </label>
+            </span>
+            <input
+              hidden
+              #fileUploadInput
+              id="file-upload"
+              type="file"
+              appFileSelect
+              [uploader]="uploader"
+              (change)="getFileDesc($event)"
+            />
+          </div>
+        </div>
+      </div>
+    </div>
+  </clr-modal>
+
+  <clr-modal [(clrModalOpen)]="submitLimitNotice">
+    <h3 class="modal-title">Notice</h3>
+    <div class="modal-body">
+      <p class="m-0">
+        Due to current licence, only
+        {{ licenceState.value.submit_rows_limit }} rows in a file will be
+        submitted. To remove the restriction, contact
+        support&#64;datacontroller.io
+      </p>
+    </div>
+    <div class="modal-footer">
+      <button
+        type="button"
+        class="btn btn-sm btn-primary"
+        (click)="submitLimitNotice = false"
+      >
+        Cancel
+      </button>
+      <button
+        type="button"
+        class="btn btn-sm btn-primary"
+        (click)="submit(); submitLimitNotice = false"
+      >
+        Submit
+      </button>
+    </div>
+  </clr-modal>
+</div>
+
+<ng-template #actionButtons>
+  <div class="clr-row m-0 clr-justify-content-center">
+    <div
+      *ngIf="status === StatusEnum.ReadyToUpload"
+      class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
+    >
+      <button
+        type="button"
+        class="btn btn-sm btn-success btn-block mr-0"
+        (click)="onShowUploadModal()"
+      >
+        <clr-icon shape="upload"></clr-icon>
+        <span>Upload</span>
+      </button>
+    </div>
+    <div
+      *ngIf="status === StatusEnum.ReadyToSubmit"
+      class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
+    >
+      <button
+        type="button"
+        class="btn btn-sm btn-success btn-block mr-0"
+        (click)="submitExcel()"
+      >
+        <clr-icon shape="upload"></clr-icon>
+        <span>Submit</span>
+      </button>
+    </div>
+    <div
+      *ngIf="status === StatusEnum.ReadyToSubmit"
+      class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
+    >
+      <button
+        type="button"
+        class="btn btn-sm btn-outline-danger btn-block mr-0"
+        (click)="discardExtractedData()"
+      >
+        <clr-icon shape="times"></clr-icon>
+        <span>Discard</span>
+      </button>
+    </div>
+  </div>
+</ng-template>

client/src/app/xlmap/xlmap.component.scss

@@ -0,0 +1,77 @@
+.card {
+  margin-top: 0;
+
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+}
+
+clr-tree-node button {
+  white-space: nowrap;
+}
+
+.no-table-selected {
+  position: relative;
+}
+
+.header-row {
+  .title-col {
+    display: flex;
+    align-items: center;
+  }
+  .options-col {
+    display: flex;
+    justify-content: flex-end;
+  }
+}
+
+.sw {
+  margin: 1rem 0rem 0.5rem 1rem;
+}
+
+.viewerTitle {
+  text-align: center;
+}
+
+.cardFlex {
+  display: flex;
+  justify-content: center;
+}
+
+.content-area {
+  padding: 0.5rem !important;
+
+  display: flex;
+  flex-direction: column;
+}
+
+hot-table {
+  ::ng-deep {
+    .primaryKeyHeaderStyle {
+      background: #306b006e;
+    }
+  }
+}
+
+.drop-area {
+  position: fixed;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+
+  display: flex;
+  justify-content: center;
+
+  margin: 1px;
+
+  border: 2px dashed #fff;
+
+  z-index: -1;
+
+  span {
+    font-size: 20px;
+    margin-top: 20px;
+    color: #fff;
+  }
+}

client/src/app/xlmap/xlmap.component.ts

@@ -0,0 +1,477 @@
+import {
+  AfterContentInit,
+  AfterViewInit,
+  Component,
+  ElementRef,
+  HostBinding,
+  OnInit,
+  QueryList,
+  ViewChildren
+} from '@angular/core'
+import { ActivatedRoute, Router } from '@angular/router'
+import { UploadFile } from '@sasjs/adapter'
+import * as XLSX from '@sheet/crypto'
+import { XLMapListItem, globals } from '../_globals'
+import { FileUploader } from '../models/FileUploader.class'
+import {
+  EventService,
+  LicenceService,
+  LoggerService,
+  SasService,
+  SasStoreService
+} from '../services'
+import { getCellAddress, getFinishingCell } from './utils/xl.utils'
+import { blobToFile, byteArrayToBinaryString } from './utils/file.utils'
+
+interface XLMapRule {
+  XLMAP_ID: string
+  XLMAP_SHEET: string
+  XLMAP_RANGE_ID: string
+  XLMAP_START: string
+  XLMAP_FINISH: string
+}
+
+interface XLUploadEntry {
+  LOAD_REF: string
+  XLMAP_ID: string
+  XLMAP_RANGE_ID: string
+  ROW_NO: number
+  COL_NO: number
+  VALUE_TXT: string
+}
+
+enum Status {
+  NoMapSelected,
+  FetchingRules,
+  ReadyToUpload,
+  ExtractingData,
+  ReadyToSubmit,
+  SubmittingExtractedData,
+  Submitting
+}
+
+enum Tabs {
+  Rules,
+  Data
+}
+
+@Component({
+  selector: 'app-xlmap',
+  templateUrl: './xlmap.component.html',
+  styleUrls: ['./xlmap.component.scss']
+})
+export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
+  @HostBinding('class.content-container') contentContainerClass = true
+  @ViewChildren('fileUploadInput')
+  fileUploadInputCompList: QueryList<ElementRef> = new QueryList()
+
+  StatusEnum = Status
+  TabsEnum = Tabs
+
+  public selectedTab = Tabs.Rules
+  public rulesSource = globals.dcLib + '.MPE_XLMAP_RULES'
+
+  public xlmaps: XLMapListItem[] = []
+  public selectedXLMap: XLMapListItem | undefined = undefined
+  public searchString = ''
+  public xlmapsLoading = true
+  public isLoading = false
+  public isLoadingDesc = ''
+  public status = Status.NoMapSelected
+
+  public xlmapRulesHeaders = [
+    'XLMAP_SHEET',
+    'XLMAP_RANGE_ID',
+    'XLMAP_START',
+    'XLMAP_FINISH'
+  ]
+  public xlmapRulesColumns = [
+    {
+      data: 'XLMAP_SHEET'
+    },
+    {
+      data: 'XLMAP_RANGE_ID'
+    },
+
+    {
+      data: 'XLMAP_START'
+    },
+    {
+      data: 'XLMAP_FINISH'
+    }
+  ]
+  public xlmapRules: XLMapRule[] = []
+
+  public xlUploadHeader = ['XLMAP_RANGE_ID', 'ROW_NO', 'COL_NO', 'VALUE_TXT']
+  public xlUploadColumns = [
+    {
+      data: 'XLMAP_RANGE_ID'
+    },
+    {
+      data: 'ROW_NO'
+    },
+    {
+      data: 'COL_NO'
+    },
+    {
+      data: 'VALUE_TXT'
+    }
+  ]
+  public xlData: XLUploadEntry[] = []
+
+  public showUploadModal = false
+  public hasBaseDropZoneOver = false
+  public filename = ''
+  public submitLimitNotice = false
+
+  public uploader: FileUploader = new FileUploader()
+
+  public licenceState = this.licenceService.licenceState
+
+  public hotTableLicenseKey: string | undefined = undefined
+  public hotTableMaxRows =
+    this.licenceState.value.viewer_rows_allowed || Infinity
+
+  constructor(
+    private eventService: EventService,
+    private licenceService: LicenceService,
+    private loggerService: LoggerService,
+    private route: ActivatedRoute,
+    private router: Router,
+    private sasStoreService: SasStoreService,
+    private sasService: SasService
+  ) {}
+
+  public xlmapOnClick(xlmap: XLMapListItem) {
+    if (xlmap.id !== this.selectedXLMap?.id) {
+      this.selectedXLMap = xlmap
+      this.viewXLMapRules()
+      this.router.navigateByUrl('/home/files/' + xlmap.id)
+    }
+  }
+
+  public xlmapListOnFilter() {
+    if (this.searchString.length > 0) {
+      const array: XLMapListItem[] = globals.xlmaps
+      this.xlmaps = array.filter((item) =>
+        item.id.toLowerCase().includes(this.searchString.toLowerCase())
+      )
+    } else {
+      this.xlmaps = globals.xlmaps
+    }
+  }
+
+  public isActiveXLMap(id: string) {
+    return this.selectedXLMap?.id === id
+  }
+
+  public maxWidthChecker(width: any, col: any) {
+    if (width > 200) return 200
+    else return width
+  }
+
+  public getCellConfiguration() {
+    return { readOnly: true }
+  }
+
+  public rowHeaders() {
+    return ' '
+  }
+
+  public onShowUploadModal() {
+    this.showUploadModal = true
+  }
+
+  /**
+   * Called by FileDropDirective
+   * @param e true if file is dragged over the drop zone
+   */
+  public fileOverBase(e: boolean): void {
+    this.hasBaseDropZoneOver = e
+  }
+
+  public getFileDesc(event: any, dropped = false) {
+    const file = dropped ? event[0] : event.target.files[0]
+
+    if (!file) return
+
+    const filename = file.name
+    this.filename = filename
+
+    const fileType = filename.slice(
+      filename.lastIndexOf('.') + 1,
+      filename.lastIndexOf('.') + 4
+    )
+
+    if (fileType.toLowerCase() === 'xls') {
+      this.showUploadModal = false
+      this.isLoading = true
+      this.isLoadingDesc = 'Extracting Data'
+      this.status = Status.ExtractingData
+
+      const reader = new FileReader()
+      reader.onload = async (theFile: any) => {
+        /* read workbook */
+        const bstr = byteArrayToBinaryString(theFile.target.result)
+        let wb: XLSX.WorkBook | undefined = undefined
+
+        const xlsxOptions: XLSX.ParsingOptions = {
+          type: 'binary',
+          cellDates: false,
+          cellFormula: true,
+          cellStyles: true,
+          cellNF: false,
+          cellText: false
+        }
+
+        try {
+          wb = XLSX.read(bstr, {
+            ...xlsxOptions
+          })
+        } catch (err: any) {
+          this.eventService.showAbortModal(
+            null,
+            err,
+            undefined,
+            'Error reading file'
+          )
+        }
+
+        if (!wb) {
+          this.isLoading = false
+          this.isLoadingDesc = ''
+          this.status = Status.ReadyToUpload
+          this.uploader.queue.pop()
+          return
+        }
+
+        this.extractData(wb)
+        return
+      }
+
+      reader.readAsArrayBuffer(file)
+    } else {
+      this.isLoading = false
+      this.isLoadingDesc = ''
+      this.status = Status.ReadyToUpload
+      this.showUploadModal = true
+      this.uploader.queue.pop()
+
+      const abortMsg =
+        'Invalid file type "<b>' +
+        this.filename +
+        '</b>". Please upload excel file.'
+      this.eventService.showAbortModal(null, abortMsg)
+    }
+  }
+
+  public discardExtractedData() {
+    this.isLoading = false
+    this.isLoadingDesc = ''
+    this.status = Status.ReadyToUpload
+    this.xlData = []
+    this.filename = ''
+    this.uploader.queue = []
+    if (this.fileUploadInputCompList.first) {
+      this.fileUploadInputCompList.first.nativeElement.value = ''
+    }
+  }
+
+  /**
+   * Submits attached excel file that is in preview mode
+   */
+  public submitExcel() {
+    if (this.licenceState.value.submit_rows_limit !== Infinity) {
+      this.submitLimitNotice = true
+      return
+    }
+
+    this.submit()
+  }
+
+  public submit() {
+    if (!this.selectedXLMap || !this.xlData.length) return
+
+    this.status = Status.Submitting
+    this.isLoading = true
+    this.isLoadingDesc = 'Submitting extracted data'
+
+    const filesToUpload: UploadFile[] = []
+
+    for (const file of this.uploader.queue) {
+      filesToUpload.push({
+        file: file,
+        fileName: file.name
+      })
+    }
+
+    const csvContent =
+      Object.keys(this.xlData[0]).join(',') +
+      '\n' +
+      this.xlData
+        .slice(0, this.licenceState.value.submit_rows_limit)
+        .map((row: any) => Object.values(row).join(','))
+        .join('\n')
+
+    const blob = new Blob([csvContent], { type: 'application/csv' })
+    const file: File = blobToFile(blob, this.filename + '.csv')
+
+    filesToUpload.push({
+      file: file,
+      fileName: file.name
+    })
+
+    const uploadUrl = 'services/editors/loadfile'
+    this.sasService
+      .uploadFile(uploadUrl, filesToUpload, {
+        table: this.selectedXLMap.targetDS
+      })
+      .then((res: any) => {
+        if (res.sasjsAbort) {
+          const abortRes = res
+          const abortMsg = abortRes.sasjsAbort[0].MSG
+          const macMsg = abortRes.sasjsAbort[0].MAC
+
+          this.eventService.showAbortModal('', abortMsg, {
+            SYSWARNINGTEXT: abortRes.SYSWARNINGTEXT,
+            SYSERRORTEXT: abortRes.SYSERRORTEXT,
+            MAC: macMsg
+          })
+        } else if (res.sasparams) {
+          const params = res.sasparams[0]
+          const tableId = params.DSID
+          this.router.navigateByUrl('/stage/' + tableId)
+        }
+      })
+      .catch((err: any) => {
+        this.eventService.catchResponseError('file upload', err)
+      })
+      .finally(() => {
+        this.status = Status.ReadyToSubmit
+        this.isLoading = false
+        this.isLoadingDesc = ''
+      })
+  }
+
+  public extractData(wb: XLSX.WorkBook) {
+    const extractedData: XLUploadEntry[] = []
+
+    this.xlmapRules.forEach((rule) => {
+      let sheetName = rule.XLMAP_SHEET
+      // if sheet name is not an absolute name rather an index string like /1, /2, etc
+      // we extract the index and find absolute sheet name for specified index
+      if (sheetName.startsWith('/')) {
+        const temp = sheetName.split('/')[1]
+        const sheetIndex = parseInt(temp) - 1
+        sheetName = wb.SheetNames[sheetIndex]
+      }
+
+      const sheet = wb.Sheets[sheetName]
+
+      const arrayOfObjects = <any[]>XLSX.utils.sheet_to_json(sheet, {
+        raw: true,
+        header: 'A',
+        blankrows: true
+      })
+
+      const start = getCellAddress(rule.XLMAP_START, arrayOfObjects)
+      const finish = getFinishingCell(start, rule.XLMAP_FINISH, arrayOfObjects)
+
+      const range = `${start}:${finish}`
+
+      const rangedData = <any[]>XLSX.utils.sheet_to_json(sheet, {
+        raw: true,
+        range: range,
+        header: 'A',
+        blankrows: true
+      })
+
+      for (let i = 0; i < rangedData.length; i++) {
+        const row = rangedData[i]
+        // Get the keys of the object (excluding '__rowNum__')
+        const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
+
+        for (let j = 0; j < keys.length; j++) {
+          const key = keys[j]
+          const val = row[key]
+
+          // in excel's R1C1 notation indexing starts from 1 but in JS it starts from 0
+          // therefore, we'll have to add 1 to rows and cols
+          extractedData.push({
+            LOAD_REF: '0',
+            XLMAP_ID: rule.XLMAP_ID,
+            XLMAP_RANGE_ID: rule.XLMAP_RANGE_ID,
+            ROW_NO: i + 1,
+            COL_NO: j + 1,
+            VALUE_TXT: val
+          })
+        }
+      }
+    })
+
+    this.status = Status.ReadyToSubmit
+    this.isLoading = false
+    this.isLoadingDesc = ''
+
+    this.xlData = extractedData
+    this.selectedTab = Tabs.Data
+  }
+
+  async viewXLMapRules() {
+    if (!this.selectedXLMap) return
+
+    this.isLoading = true
+    this.isLoadingDesc = 'Loading excel rules'
+    this.status = Status.FetchingRules
+
+    await this.sasStoreService
+      .getXLMapRules(this.selectedXLMap.id)
+      .then((res) => {
+        this.xlmapRules = res.xlmaprules
+        this.status = Status.ReadyToUpload
+      })
+      .catch((err) => {
+        this.loggerService.error(err)
+      })
+
+    this.isLoading = false
+    this.isLoadingDesc = ''
+  }
+
+  private load() {
+    this.xlmaps = globals.xlmaps
+    this.xlmapsLoading = false
+
+    const id = this.route.snapshot.params['id']
+
+    if (id) {
+      const xlmapListItem = this.xlmaps.find((item) => item.id === id)
+      if (xlmapListItem) {
+        this.selectedXLMap = xlmapListItem
+        this.viewXLMapRules()
+      }
+    }
+  }
+
+  ngOnInit() {
+    this.licenceService.hot_license_key.subscribe(
+      (hot_license_key: string | undefined) => {
+        this.hotTableLicenseKey = hot_license_key
+      }
+    )
+  }
+
+  ngAfterViewInit() {
+    return
+  }
+
+  ngAfterContentInit(): void {
+    if (globals.editor.startupSet) {
+      this.load()
+    } else {
+      this.eventService.onStartupDataLoaded.subscribe(() => {
+        this.load()
+      })
+    }
+  }
+}

client/src/app/xlmap/xlmap.module.ts

@@ -0,0 +1,31 @@
+import { CommonModule } from '@angular/common'
+import { NgModule } from '@angular/core'
+import { FormsModule } from '@angular/forms'
+import { ClarityModule } from '@clr/angular'
+import { HotTableModule } from '@handsontable/angular'
+import { registerAllModules } from 'handsontable/registry'
+import { AppSharedModule } from '../app-shared.module'
+import { DirectivesModule } from '../directives/directives.module'
+import { XLMapRouteComponent } from '../routes/xlmap-route/xlmap-route.component'
+import { DcTreeModule } from '../shared/dc-tree/dc-tree.module'
+import { XLMapRoutingModule } from './xlmap-routing.module'
+import { XLMapComponent } from './xlmap.component'
+
+// register Handsontable's modules
+registerAllModules()
+
+@NgModule({
+  declarations: [XLMapRouteComponent, XLMapComponent],
+  imports: [
+    HotTableModule,
+    XLMapRoutingModule,
+    FormsModule,
+    ClarityModule,
+    AppSharedModule,
+    CommonModule,
+    DcTreeModule,
+    DirectivesModule
+  ],
+  exports: [XLMapComponent]
+})
+export class XLMapModule {}

client/src/environments/_eula.ts

@@ -18,4 +18,4 @@ In any case, you must not make any such use of this software as to develop softw
 UNLESS EXPRESSLY AGREED OTHERWISE, 4GL APPS PROVIDES THIS SOFTWARE ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, AND IN NO EVENT AND UNDER NO LEGAL THEORY, SHALL 4GL APPS BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER ARISING FROM USE OR INABILITY TO USE THIS SOFTWARE.
 
 
-`
+`

client/src/styles.scss

@@ -1,16 +1,17 @@
 /* You can add global styles to this file, and also import other style files */
 @import '~handsontable/dist/handsontable.full.css';
 
-@import "~@clr/ui/clr-ui.min.css";
-@import "~@clr/icons/clr-icons.min.css";
+@import '~@clr/ui/clr-ui.min.css';
+@import '~@clr/icons/clr-icons.min.css';
 
-@font-face{
+@font-face {
   font-family: text-security-disc;
-  src: url("https://raw.githubusercontent.com/noppa/text-security/master/dist/text-security-disc.woff");
+  src: url('https://raw.githubusercontent.com/noppa/text-security/master/dist/text-security-disc.woff');
 }
 
-body, html {
-  font-weight: 400!important;
+body,
+html {
+  font-weight: 400 !important;
 
   padding: 0;
   margin: 0;
@@ -29,7 +30,7 @@ button {
 }
 
 // Custom loading spinner
-.slider{
+.slider {
   position: absolute;
   width: 320px;
   margin-left: 75px;
@@ -38,33 +39,45 @@ button {
   overflow-x: hidden;
 }
 
-.line{
-  position:absolute;
+.line {
+  position: absolute;
   opacity: 0.4;
-  background:#73D544;
-  width:150%;
-  height:5px;
+  background: #73d544;
+  width: 150%;
+  height: 5px;
 }
 
-.subline{
-  position:absolute;
-  background:#73D544;
-  height:5px;
+.subline {
+  position: absolute;
+  background: #73d544;
+  height: 5px;
 }
-.inc{
+.inc {
   animation: increase 2s infinite;
 }
-.dec{
+.dec {
   animation: decrease 2s 0.5s infinite;
 }
 
 @keyframes increase {
-   from { left: -5%; width: 5%; }
-   to { left: 130%; width: 100%;}
+  from {
+    left: -5%;
+    width: 5%;
+  }
+  to {
+    left: 130%;
+    width: 100%;
+  }
 }
 @keyframes decrease {
-   from { left: -80%; width: 80%; }
-   to { left: 110%; width: 10%;}
+  from {
+    left: -80%;
+    width: 80%;
+  }
+  to {
+    left: 110%;
+    width: 10%;
+  }
 }
 // Custo loading spinner end
 
@@ -276,6 +289,10 @@ button {
   margin-bottom: 10px;
 }
 
+.mb-10-i {
+  margin-bottom: 10px !important;
+}
+
 .mb-20 {
   margin-bottom: 20px;
 }
@@ -321,11 +338,11 @@ button {
 }
 
 .color-dark-gray {
-  color: #495967
+  color: #495967;
 }
 
-.color-darker-gray{
-  color: #314351
+.color-darker-gray {
+  color: #314351;
 }
 
 .color-white {
@@ -333,7 +350,7 @@ button {
 }
 
 .color-white-i {
-  color: white !important
+  color: white !important;
 }
 
 .color-green {
@@ -341,15 +358,15 @@ button {
 }
 
 .color-dc-green {
-  color: #81b440
+  color: #81b440;
 }
 
 .color-red {
-  color: #e45454
+  color: #e45454;
 }
 
 .color-orange {
-  color: #E67E22;
+  color: #e67e22;
 }
 
 .color-blue {
@@ -357,7 +374,7 @@ button {
 }
 
 .color-yellow {
-  color: #f1c40f
+  color: #f1c40f;
 }
 
 .cursor-pointer {
@@ -501,7 +518,7 @@ button {
 }
 
 .z-index-highest {
-  z-index: 10000000
+  z-index: 10000000;
 }
 
 .vertical-align-middle {
@@ -519,35 +536,36 @@ button {
 }
 
 .progresStatic {
-  margin-top:-6px!important;
-  position: absolute!important;
-  z-index: 10000!important;
+  margin-top: -6px !important;
+  position: absolute !important;
+  z-index: 10000 !important;
 }
 
-.progress, .progress-static {
+.progress,
+.progress-static {
   background-color: #f5f6fe;
   border-radius: 0;
   font-size: inherit;
   height: 6px;
   margin: 0;
-  max-height: .583333rem;
-  min-height: .166667rem;
+  max-height: 0.583333rem;
+  min-height: 0.166667rem;
   overflow: hidden;
   display: block;
   width: calc(100% - 63px);
 }
 
 .progress.loop:after {
-    -webkit-animation: clr-progress-looper 1.5s ease-in-out infinite;
-    animation: clr-progress-looper 1.5s ease-in-out infinite;
-    content: " ";
-    top: .166667rem;
-    bottom: 0;
-    left: 0;
-    position: absolute;
-    display: block;
-    background-color: #60b515;
-    width: 75%;
+  -webkit-animation: clr-progress-looper 1.5s ease-in-out infinite;
+  animation: clr-progress-looper 1.5s ease-in-out infinite;
+  content: ' ';
+  top: 0.166667rem;
+  bottom: 0;
+  left: 0;
+  position: absolute;
+  display: block;
+  background-color: #60b515;
+  width: 75%;
 }
 
 // Fix for clarity bug, should be addressed when clarity is updated
@@ -570,9 +588,9 @@ button {
 }
 
 .alert-app-level.alert-danger {
-    background: #D94B2E;
-    color: #fff;
-    border: none;
+  background: #d94b2e;
+  color: #fff;
+  border: none;
 }
 
 .card-header {
@@ -581,7 +599,7 @@ button {
 
 .select select:focus {
   border-bottom: 1px solid #495967;
-  background: linear-gradient(180deg,transparent 95%,#495a67 0) no-repeat;
+  background: linear-gradient(180deg, transparent 95%, #495a67 0) no-repeat;
 }
 
 .clr-treenode-children {
@@ -597,7 +615,9 @@ button {
   background: #d8e3e9;
 }
 
-clr-select-container .clr-control-container, clr-select-container .clr-control-container .clr-select-wrapper, clr-select-container select {
+clr-select-container .clr-control-container,
+clr-select-container .clr-control-container .clr-select-wrapper,
+clr-select-container select {
   width: 100%;
 }
 
@@ -605,42 +625,46 @@ tbody {
   font-weight: 400;
 }
 
-h3, h4 {
-    color: #585858;
-    font-weight: 400;
-    letter-spacing: normal;
-    line-height: 1rem;
-    margin-top: 1rem;
-    margin-bottom: 0;
-    /* text-transform: uppercase; */
+h3,
+h4 {
+  color: #585858;
+  font-weight: 400;
+  letter-spacing: normal;
+  line-height: 1rem;
+  margin-top: 1rem;
+  margin-bottom: 0;
+  /* text-transform: uppercase; */
 }
 
-h1, h2 {
-    color: #585858;
-    font-weight: 400;
-    /* font-family: Metropolis,Avenir Next,Helvetica Neue,Arial,sans-serif; */
-    letter-spacing: normal;
-    line-height: 2rem;
-    margin-top: 1rem;
-    margin-bottom: 0;
-    /* text-transform: uppercase; */
+h1,
+h2 {
+  color: #585858;
+  font-weight: 400;
+  /* font-family: Metropolis,Avenir Next,Helvetica Neue,Arial,sans-serif; */
+  letter-spacing: normal;
+  line-height: 2rem;
+  margin-top: 1rem;
+  margin-bottom: 0;
+  /* text-transform: uppercase; */
 }
 
 clr-icon.is-info {
-    fill: #80b441;
+  fill: #80b441;
 }
 
-.datagrid-host, .datagrid-overlay-wrapper {
-    display: -webkit-box;
-    display: -ms-flexbox;
-    display: -webkit-box!important;
-    -webkit-box-direction: normal;
+.datagrid-host,
+.datagrid-overlay-wrapper {
+  display: -webkit-box;
+  display: -ms-flexbox;
+  display: -webkit-box !important;
+  -webkit-box-direction: normal;
 }
 
-.btn.btn-danger, .btn.btn-warning {
-    border-color: #ef4f2e;
-    background-color: #D94B2E;
-    color: #fff;
+.btn.btn-danger,
+.btn.btn-warning {
+  border-color: #ef4f2e;
+  background-color: #d94b2e;
+  color: #fff;
 }
 
 .d-none {
@@ -685,11 +709,11 @@ clr-icon.is-info {
 }
 
 .handsontable td.htInvalid {
-  background: #e62700ad!important;
-    border: 1px solid red !important;
-    color: #ffffff!important;
+  background: #e62700ad !important;
+  border: 1px solid red !important;
+  color: #ffffff !important;
 }
-.margin-top-20{
+.margin-top-20 {
   margin-top: 20px;
 }
 .hidden {
@@ -823,7 +847,7 @@ clr-icon.is-info {
 }
 
 .datagrid-body {
-  padding-bottom: 2rem!important;
+  padding-bottom: 2rem !important;
 }
 
 .abortMsg {
@@ -831,16 +855,15 @@ clr-icon.is-info {
   font-family: monospace;
 }
 
-
 #graph svg {
-    height: 100%;
-    width: 100%;
+  height: 100%;
+  width: 100%;
 }
 
 .no-table-selected {
-  display:flex;
-  justify-content:center;
-  flex-direction:column;
+  display: flex;
+  justify-content: center;
+  flex-direction: column;
   align-items: center;
   position: absolute;
   background: white;
@@ -851,16 +874,15 @@ clr-icon.is-info {
 }
 
 .copyRight {
-  background:#495967!important;
+  background: #495967 !important;
   color: #fff;
-  display:flex !important;
-  justify-content:center;
+  display: flex !important;
+  justify-content: center;
   align-items: center;
   padding: 5px 0px 4px 0px;
   z-index: 100;
 }
 
-
 .nav-tree > clr-tree-node.clr-expanded {
   display: inline-block !important;
 }
@@ -903,13 +925,13 @@ clr-tree-node {
 }
 
 .tree-search-wrapper {
-    position: relative;
-    display: flex;
-    align-items: center;
+  position: relative;
+  display: flex;
+  align-items: center;
 
-    clr-input-container {
-      margin: 0;
-    }
+  clr-input-container {
+    margin: 0;
+  }
 
   clr-icon {
     position: absolute;
@@ -956,7 +978,8 @@ input::-ms-clear {
     overflow: hidden !important;
   }
 
-  .clr-treenode-content .clr-icon, .clr-treenode-content clr-icon {
+  .clr-treenode-content .clr-icon,
+  .clr-treenode-content clr-icon {
     min-width: 16px;
     min-height: 16px;
   }
@@ -985,12 +1008,12 @@ input::-ms-clear {
 }
 
 .loadingSpinner {
-  height:70vh;
+  height: 70vh;
   flex: 1;
-  display:flex;
+  display: flex;
   justify-content: center;
-  flex-direction:column;
-  align-items:center;
+  flex-direction: column;
+  align-items: center;
 }
 
 .disable-password-manager {
@@ -1025,7 +1048,8 @@ hr.light {
     position: relative;
     min-width: 170px;
 
-    clr-icon, .spinner {
+    clr-icon,
+    .spinner {
       position: absolute;
       right: 19px;
       top: 0px;
@@ -1063,7 +1087,7 @@ hr.light {
   }
 
   /* Firefox */
-  input[type=number] {
+  input[type='number'] {
     -moz-appearance: textfield;
   }
 }
@@ -1076,4 +1100,4 @@ hr.light {
 .link-it {
   cursor: pointer;
   text-decoration: underline;
-}
+}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "dcfrontend",
-  "version": "6.2.7",
+  "version": "6.3.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "dcfrontend",
-      "version": "6.2.7",
+      "version": "6.3.0",
       "hasInstallScript": true,
       "devDependencies": {
         "@saithodev/semantic-release-gitea": "^2.1.0",

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "^4.11.1",
-        "@sasjs/core": "^4.48.4"
+        "@sasjs/core": "^4.49.0"
       }
     },
     "node_modules/@coolaj86/urequest": {
@@ -116,9 +116,9 @@
       "integrity": "sha512-Grwydm5GxBsYk238PZw41XPjXVVQ9vWcvfZ06L2P0bQbvK0sGn7l69JA7H5MGr3QcaLpiD4Kg70cAh7PgE+JOw=="
     },
     "node_modules/@sasjs/core": {
-      "version": "4.48.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.48.4.tgz",
-      "integrity": "sha512-KTLHRR47I627NKZG0qMW+wGJP4gFGyeEEyBDsVaSnevdvCz01oP/lpLxx4fIESPQ/YzLNEv+RcP8kcxkdSPQow=="
+      "version": "4.49.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.49.0.tgz",
+      "integrity": "sha512-hp3Hb4DkT6FmowyNHTOvSlgmSObW9WeuTJj+TQlwPgnBo59mAB4XFUnUaYSA+7ghvsHqUZf1OP2eSYqmnN5swQ=="
     },
     "node_modules/@sasjs/lint": {
       "version": "2.3.1",
@@ -229,6 +229,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/tough-cookie": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
+      "peer": true
+    },
     "node_modules/abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -1828,9 +1834,9 @@
       }
     },
     "@sasjs/core": {
-      "version": "4.48.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.48.4.tgz",
-      "integrity": "sha512-KTLHRR47I627NKZG0qMW+wGJP4gFGyeEEyBDsVaSnevdvCz01oP/lpLxx4fIESPQ/YzLNEv+RcP8kcxkdSPQow=="
+      "version": "4.49.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.49.0.tgz",
+      "integrity": "sha512-hp3Hb4DkT6FmowyNHTOvSlgmSObW9WeuTJj+TQlwPgnBo59mAB4XFUnUaYSA+7ghvsHqUZf1OP2eSYqmnN5swQ=="
     },
     "@sasjs/lint": {
       "version": "2.3.1",
@@ -1927,6 +1933,12 @@
         "@types/node": "*"
       }
     },
+    "@types/tough-cookie": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz",
+      "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==",
+      "peer": true
+    },
     "abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -2953,7 +2965,8 @@
     "ws": {
       "version": "8.13.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA=="
+      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
+      "requires": {}
     },
     "xml": {
       "version": "1.0.1",

sas/package.json

@@ -14,7 +14,8 @@
     "sas9e": "sasjs request services/admin/makedata -d deploy/makeDataSas9.json -t sas9  ",
     "sas9f": "sasjs request services/admin/refreshtablelineage -t sas9 ",
     "sas9g": "sasjs request services/admin/refreshcatalog -t sas9",
-    "4gl": "npm run cpfavicon && sasjs cbd -t 4gl && sasjs request services/admin/makedata -d deploy/makeData4GL.json -l sasjsresults/makedata_4gl.log -o sasjsresults/makedata_4gl.json -t 4gl",
+    "4gl": "npm run cpfavicon && sasjs cbd -t 4gl && npm run 4glmakedata",
+    "4glmakedata": "sasjs request services/admin/makedata -d deploy/makeData4GL.json -l sasjsresults/makedata_4gl.log -o sasjsresults/makedata_4gl.json -t 4gl",
     "server": "npm run cpfavicon && sasjs cbd -t server && npm run serverdata",
     "server-mihajlo": "npm run cpfavicon && sasjs cbd -t server-mihajlo && npm run serverdata-mihajlo",
     "serverdata-mihajlo": "sasjs request services/admin/makedata -d deploy/makeDataServer.json -l sasjsresults/makedata_server.log -o sasjsresults/makedata_server.json -t server-mihajlo",
@@ -28,6 +29,6 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "^4.11.1",
-    "@sasjs/core": "^4.48.4"
+    "@sasjs/core": "^4.49.0"
   }
 }

sas/sasjs/db/datactrl/mpe_xlmap_data.ddl

@@ -0,0 +1,18 @@
+/**
+  @file
+  @brief DDL for MPE_XLMAP_DATA
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+create table &curlib..MPE_XLMAP_DATA(
+        LOAD_REF char(32) not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        ROW_NO num not null,
+        COL_NO num not null,
+        VALUE_TXT char(4000),
+    constraint pk_MPE_XLMAP_DATA
+      primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO));

sas/sasjs/db/datactrl/mpe_xlmap_info.ddl

@@ -0,0 +1,17 @@
+/**
+  @file
+  @brief DDL for mpe_xlmap_info
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+create table &curlib..mpe_xlmap_info(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_DESCRIPTION char(1000) not null,
+        XLMAP_TARGETLIBDS char(41) not null,
+        tx_to num not null,
+    constraint pk_mpe_xlmap_info
+        primary key(tx_from,XLMAP_ID));

sas/sasjs/db/datactrl/mpe_xlmap_rules.ddl

@@ -0,0 +1,19 @@
+/**
+  @file
+  @brief DDL for mpe_xlmap_rules
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+create table &curlib..mpe_xlmap_rules(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        XLMAP_SHEET char(32) not null,
+        XLMAP_START char(1000) not null,
+        XLMAP_FINISH char(1000),
+        tx_to num not null,
+    constraint pk_mpe_xlmap_rules
+        primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID));

sas/sasjs/db/migrations/20230115_v6.5_release.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief migration script to move from v5 to v6.5 of data controller
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/your/dc/path";
+
+/**
+  * Change 1
+  * New MPE_SUBMIT table
+  */
+proc sql;
+create table &dclib..mpe_xlmap_rules(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        XLMAP_SHEET char(32) not null,
+        XLMAP_START char(1000) not null,
+        XLMAP_FINISH char(1000),
+        tx_to num not null,
+    constraint pk_mpe_xlmap_rules
+        primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID));
+
+create table &dclib..MPE_XLMAP_DATA(
+        LOAD_REF char(32) not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_RANGE_ID char(32) not null,
+        ROW_NO num not null,
+        COL_NO num not null,
+        VALUE_TXT char(4000),
+    constraint pk_MPE_XLMAP_DATA
+      primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO));
+
+create table &dclib..mpe_xlmap_info(
+        tx_from num not null,
+        XLMAP_ID char(32) not null,
+        XLMAP_DESCRIPTION char(1000) not null,
+        XLMAP_TARGETLIBDS char(41) not null,
+        tx_to num not null,
+    constraint pk_mpe_xlmap_info
+        primary key(tx_from,XLMAP_ID));

sas/sasjs/macros/mpe_loader.sas

@@ -24,6 +24,7 @@
   @li mp_lockanytable.sas
   @li mpe_accesscheck.sas
   @li mpe_alerts.sas
+  @li mpe_xlmapvalidate.sas
   @li mpe_loadfail.sas
   @li mpe_runhook.sas
 
@@ -450,7 +451,7 @@ run;
     %do i=1 %to %sysfunc(countw(&pk));
       %let iWord=%scan(&pk,&i);
       call symputx('duplist',symget('duplist')!!
-        " &iWord="!!trim(&iWord));
+        " &iWord="!!cats(&iWord));
     %end;
   run;
   %let msg=This upload contains duplicates on the Primary Key columns %trim(
@@ -472,6 +473,10 @@ run;
   %return;
 %end;
 
+/* If a Complex Excel Upload, needs to have the load ref added to the table */
+%mpe_xlmapvalidate(&mperef,work.staging_ds,&mpelib,&orig_libds)
+
+/* Run the Post Edit Hook prior to creation of staging folder */
 %mpe_runhook(POST_EDIT_HOOK)
 
 /* stop if err */

sas/sasjs/macros/mpe_makedata.sas

@@ -269,6 +269,210 @@ insert into &lib..mpe_datadictionary set
     ,DD_SENSITIVITY="Low"
     ,tx_to='31DEC5999:23:59:59'dt;
 
+/**
+  * mpe_xlmap_info
+  */
+insert into &lib..mpe_xlmap_info set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_description='Basel 3 Key Metrics report'
+  ,XLMAP_TARGETLIBDS="&lib..MPE_XLMAP_DATA";
+
+/**
+  * mpe_xlmap_rules
+  */
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:a';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:b'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:b';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:c'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:c';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:d'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:d';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:e'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:e';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:f'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH 4 R[2]C[0]:f';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[1]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/b'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[2]:Common Equity Tier 1 (CET1)';
+  insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/c'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[3]:Common Equity Tier 1 (CET1)';
+  insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/d'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[4]:Common Equity Tier 1 (CET1)';
+  insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/e'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[5]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1/f'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[0]C[6]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1a/e'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[1]C[5]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:1a/f'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='MATCH C R[1]C[6]:Common Equity Tier 1 (CET1)';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='ABSOLUTE D10';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/b'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='ABSOLUTE E10';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/c'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[6]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/d'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[8]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/e'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[9]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2/f'
+  ,xlmap_sheet='/3'
+  ,xlmap_start='RELATIVE R[10]C[10]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:2a'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='ABSOLUTE H11'
+  ,xlmap_finish='RELATIVE R[0]C[1]';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-KM1'
+  ,xlmap_range_id='KM1:3'
+  ,xlmap_sheet='KM1'
+  ,xlmap_start='RELATIVE R[12]C[4]'
+  ,xlmap_finish='ABSOLUTE I13';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-CR2'
+  ,xlmap_range_id='CR2-sec1'
+  ,xlmap_sheet='CR2'
+  ,xlmap_start='ABSOLUTE D8'
+  ,xlmap_finish='BLANKROW';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='BASEL-CR2'
+  ,xlmap_range_id='CR2-sec2'
+  ,xlmap_sheet='CR2'
+  ,xlmap_start='ABSOLUTE D18'
+  ,xlmap_finish='LASTDOWN';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='SAMPLE'
+  ,xlmap_range_id='header'
+  ,xlmap_sheet='/1'
+  ,xlmap_start='ABSOLUTE B3'
+  ,xlmap_finish='ABSOLUTE B8';
+insert into &lib..mpe_xlmap_rules set
+  tx_from=0
+  ,tx_to='31DEC5999:23:59:59'dt
+  ,xlmap_id='SAMPLE'
+  ,xlmap_range_id='data'
+  ,xlmap_sheet='/1'
+  ,xlmap_start='ABSOLUTE B13'
+  ,xlmap_finish='ABSOLUTE E16';
+
+
+
 /**
   * MPE_GROUPS
   */
@@ -981,6 +1185,42 @@ insert into &lib..mpe_selectbox set
       ,notes='Docs: https://docs.datacontroller.io/column-level-security'
       ,post_edit_hook='services/hooks/mpe_column_level_security_postedit'
     ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_XLMAP_INFO'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+      ,buskey='XLMAP_ID'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+      ,post_edit_hook='services/hooks/mpe_xlmap_info_postedit'
+    ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_XLMAP_RULES'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+      ,buskey='XLMAP_ID XLMAP_RANGE_ID'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+      ,post_edit_hook='services/hooks/mpe_xlmap_rules_postedit'
+    ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_XLMAP_DATA'
+      ,num_of_approvals_required=1
+      ,loadtype='UPDATE'
+      ,buskey='LOAD_REF XLMAP_ID XLMAP_RANGE_ID ROW_NO COL_NO'
+      ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
+    ;
   insert into &lib..mpe_tables
     set tx_from=0
       ,tx_to='31DEC5999:23:59:59'dt
@@ -1253,6 +1493,27 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/mpe_alerts.alert_lib"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_XLMAP_INFO"
+    ,base_col="XLMAP_ID"
+    ,rule_type='CASE'
+    ,rule_value='UPCASE'
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_XLMAP_RULES"
+    ,base_col="XLMAP_ID"
+    ,rule_type='CASE'
+    ,rule_value='UPCASE'
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"
@@ -1640,6 +1901,16 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value='1'
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_XLMAP_INFO"
+    ,base_col="XLMAP_ID"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="&lib..MPE_XLMAP_RULES.XLMAP_ID"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
 
 /**
   * MPE_X_TEST

sas/sasjs/macros/mpe_makedatamodel.sas

@@ -268,6 +268,55 @@ proc datasets lib=&lib noprint;
     pk_mpe_excel_config=(tx_to xl_libref xl_table xl_column)
     /nomiss unique;
 quit;
+
+proc sql;
+create table &lib..MPE_XLMAP_DATA(
+        LOAD_REF char(32) &notnull,
+        XLMAP_ID char(32) &notnull,
+        XLMAP_RANGE_ID char(32) &notnull,
+        ROW_NO num &notnull,
+        COL_NO num &notnull,
+        VALUE_TXT char(4000)
+);quit;
+proc datasets lib=&lib noprint;
+  modify MPE_XLMAP_DATA;
+  index create
+    pk_MPE_XLMAP_DATA=(load_ref xlmap_id xlmap_range_id row_no col_no)
+    /nomiss unique;
+quit;
+
+proc sql;
+create table &lib..mpe_xlmap_info(
+        tx_from num &notnull,
+        tx_to num &notnull,
+        XLMAP_ID char(32) &notnull,
+        XLMAP_DESCRIPTION char(1000) &notnull,
+        XLMAP_TARGETLIBDS char(41) &notnull
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_xlmap_info;
+  index create
+    pk_mpe_xlmap_info=(tx_to xlmap_id)
+    /nomiss unique;
+quit;
+
+proc sql;
+create table &lib..mpe_xlmap_rules(
+        tx_from num &notnull,
+        tx_to num &notnull,
+        XLMAP_ID char(32) &notnull,
+        XLMAP_RANGE_ID char(32) &notnull,
+        XLMAP_SHEET char(32) &notnull,
+        XLMAP_START char(1000) &notnull,
+        XLMAP_FINISH char(1000)
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_xlmap_rules;
+  index create
+    pk_mpe_xlmap_rules=(tx_to xlmap_id xlmap_range_id)
+    /nomiss unique;
+quit;
+
 proc sql;
 create table &lib..mpe_filteranytable(
         filter_rk num &notnull,

sas/sasjs/macros/mpe_xlmapvalidate.sas

@@ -0,0 +1,28 @@
+/**
+  @file
+  @brief Validates excel map structure and adds load ref
+  @details Used in staging, prior to the post edit hook
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_xlmapvalidate(mperef,inds,dclib,tgtds);
+
+  %local ismap;
+  proc sql noprint;
+  select count(*) into: ismap
+    from &dclib..mpe_xlmap_info
+    where XLMAP_TARGETLIBDS="&tgtds" and &dc_dttmtfmt. le TX_TO ;
+
+  %if "&tgtds"="&dclib..MPE_XLMAP_DATA" or &ismap>0 %then %do;
+    data &inds;
+      set &inds;
+      LOAD_REF="&mperef";
+    run;
+  %end;
+
+%mend mpe_xlmapvalidate;

sas/sasjs/macros/mpe_xlmapvalidate.test.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Testing mpe_xlmapvalidate macro
+
+  <h4> SAS Macros </h4>
+  @li mpe_xlmapvalidate.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+  <h4> SAS Includes </h4>
+  @li mpe_xlmap_data.ddl ul
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* create the table */
+%let curlib=work;
+proc sql;
+%inc ul;
+
+data work.test1;
+  if 0 then set work.MPE_XLMAP_DATA;
+  LOAD_REF='0';
+  XLMAP_ID='Sample';
+  XLMAP_RANGE_ID='Range 1';
+  ROW_NO=1;
+  COL_NO=2;
+  VALUE_TXT='something';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mpe_xlmapvalidate(DCTEST1,work.test1,&dclib,NOT.MAP)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot
+)
+
+data _null_;
+  set work.test1;
+  call symputx('test1',load_ref);
+run;
+%mp_assert(
+  iftrue=(&test1=0),
+  desc=Checking load ref was not applied
+)
+
+%mpe_xlmapvalidate(DCTEST2,work.test1,&dclib,&dclib..MPE_XLMAP_DATA)
+
+data _null_;
+  set work.test1;
+  call symputx('test2',load_ref);
+run;
+%mp_assert(
+  iftrue=(&test2=DCTEST2),
+  desc=Checking load ref was applied for default case
+)

sas/sasjs/sasjsconfig.json

@@ -207,7 +207,7 @@
     },
     {
       "name": "4gl",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false

sas/sasjs/services/auditors/postdata.sas

@@ -56,12 +56,12 @@
 data _null_;
   set work.sascontroltable;
   call symputx('ACTION',ACTION);
-  call symputx('TABLE',TABLE);
+  call symputx('LOAD_REF',TABLE);
   /* DIFFTIME is when the DIFF was generated on the frontend */
   call symputx('DIFFTIME',DIFFTIME);
 run;
 
-%global action is_err err_msg;
+%global action is_err err_msg msg;
 %let is_err=0;
 
 %let user=%mf_getuser();
@@ -80,7 +80,7 @@ RUN;
 %let isfmtcat=0;
 data APPROVE1;
   set &mpelib..mpe_submit;
-  where TABLE_ID="&TABLE";
+  where TABLE_ID="&LOAD_REF";
   /* fetch mpe_submit data */
   libds=cats(base_lib,'.',base_ds);
   REVIEWED_ON=put(reviewed_on_dttm,datetime19.);
@@ -115,9 +115,9 @@ run;
 )
 
 %mp_abort(
-  iftrue=(%mf_verifymacvars(difftime orig_libds libds table)=0)
+  iftrue=(%mf_verifymacvars(difftime orig_libds libds load_ref)=0)
   ,mac=&_program
-  ,msg=%str(Missing: difftime orig_libds libds table)
+  ,msg=%str(Missing: difftime orig_libds libds load_ref)
 )
 
 /* security checks */
@@ -186,7 +186,7 @@ run;
   %let prev_upload_check=1;
   proc sql;
   select count(*) into: prev_upload_check from &mpelib..mpe_review
-    where TABLE_ID="&TABLE" and REVIEWED_BY_NM="&user"
+    where TABLE_ID="&LOAD_REF" and REVIEWED_BY_NM="&user"
     and REVIEW_STATUS_ID ne "SUBMITTED";
   %let authcheck=%mf_getattrn(work.authAPP,NLOBS);
   %if &authcheck=0 or &prev_upload_check=1 %then %do;
@@ -233,7 +233,7 @@ run;
   %else %let oldloc=%qsysfunc(getoption(LOG));
   %if %length(&oldloc)>0 %then %do;
     proc printto
-      log="&mpelocapprovals/&TABLE/approval.log";
+      log="&mpelocapprovals/&LOAD_REF/approval.log";
     run;
     data _null_;
       if _n_=1 then do;
@@ -247,7 +247,7 @@ run;
   %end;
   %else %do;
     proc printto
-      log="&mpelocapprovals/&TABLE/approval.log";
+      log="&mpelocapprovals/&LOAD_REF/approval.log";
     run;
   %end;
 
@@ -285,11 +285,11 @@ select PRE_APPROVE_HOOK, POST_APPROVE_HOOK, LOADTYPE, var_txfrom, var_txto
   ,msg=%str(Missing: mpelocapprovals orig_libds)
 )
 
-/* get dataset from approvals location */
+/* get dataset from approvals location (has same name as load_ref) */
 %let tmplib=%mf_getuniquelibref();
-libname &tmplib "&mpelocapprovals/&TABLE";
+libname &tmplib "&mpelocapprovals/&LOAD_REF";
 data STAGING_DS;
-  set &tmplib..&TABLE;
+  set &tmplib..&LOAD_REF;
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
@@ -313,7 +313,7 @@ run;
   %let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
   data work.append_review;
     if 0 then set &mpelib..mpe_review;
-    TABLE_ID="&TABLE";
+    TABLE_ID="&LOAD_REF";
     BASE_TABLE="&orig_libds";
     REVIEW_STATUS_ID="APPROVED";
     REVIEWED_BY_NM="&user";
@@ -323,7 +323,7 @@ run;
     stop;
   run;
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
+    lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc append base=&mpelib..mpe_review data=work.append_review;
@@ -335,7 +335,7 @@ run;
 
   /* update mpe_submit table */
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval),
+    lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc sql;
@@ -343,7 +343,7 @@ run;
     set num_of_approvals_remaining=&num_of_approvals_remaining-1,
       reviewed_by_nm="&user",
       reviewed_on_dttm=&sastime
-    where table_id="&table";
+    where table_id="&LOAD_REF";
   %mp_lockanytable(UNLOCK,
     lib=&mpelib,ds=mpe_submit,
     ctl_ds=&mpelib..mpe_lockanytable
@@ -369,7 +369,7 @@ run;
 )
 %mpe_targetloader(libds=&orig_libds
   ,now= &sastime
-  ,etlsource=&TABLE
+  ,etlsource=&LOAD_REF
   ,STAGING_DS=STAGING_DS
   ,dclib=&mpelib
 %if &action=APPROVE_TABLE %then %do;
@@ -405,7 +405,7 @@ run;
   proc sql noprint;
   select max(processed_dttm)-1 format=datetime19. into: tstamp
     from &mpelib..mpe_dataloads
-    where libref="&libref" and dsn="&ds" and ETLSOURCE="&TABLE";
+    where libref="&libref" and dsn="&ds" and ETLSOURCE="&LOAD_REF";
   quit;
   %if &tstamp=. %then %let tstamp=%sysfunc(datetime(),datetime19.);
 
@@ -498,7 +498,7 @@ run;
     else if _____orig then _____status='ORIGINAL';
   run;
   proc export data=TEMPDIFFS dbms=csv replace
-    outfile="&mpelocapprovals/&TABLE/&tempDIFFS_CSV" ;
+    outfile="&mpelocapprovals/&LOAD_REF/&tempDIFFS_CSV" ;
   run;
   proc sql noprint;
   select filesize format=sizekmg10.1, filesize as filesize_raw
@@ -545,7 +545,7 @@ run;
   proc sort data=&mpelib..mpe_submit(where=(
       submit_status_cd='SUBMITTED'
       and cats(base_lib,'.',base_ds)="&orig_libds"
-      and table_id ne "&TABLE"
+      and table_id ne "&LOAD_REF"
     )) out=submits;
     by descending submitted_on_dttm;
   run;
@@ -599,7 +599,7 @@ run;
     data work.outds_mod; run;
     data work.outds_del; run;
   %end;
-  libname approve "&mpelocapprovals/&TABLE";
+  libname approve "&mpelocapprovals/&LOAD_REF";
   data; set &libds;stop;run;
   %let emptybasetable=&syslast;
   data approve.ActualDiffs;
@@ -621,7 +621,7 @@ run;
   run;
 
   proc export data=approve.ActualDiffs
-    outfile="&mpelocapprovals/&TABLE/ActualDiffs.csv"
+    outfile="&mpelocapprovals/&LOAD_REF/ActualDiffs.csv"
     dbms=csv
     replace;
   run;
@@ -631,7 +631,7 @@ run;
   %let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
   data work.append_review;
     if 0 then set &mpelib..mpe_review;
-    TABLE_ID="&TABLE";
+    TABLE_ID="&LOAD_REF";
     BASE_TABLE="&orig_libds";
     REVIEW_STATUS_ID="APPROVED";
     REVIEWED_BY_NM="&user";
@@ -641,7 +641,7 @@ run;
     stop;
   run;
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
+    lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc append base=&mpelib..mpe_review data=work.append_review;
@@ -653,7 +653,7 @@ run;
 
   /* update mpe_submit table */
   %mp_lockanytable(LOCK,
-    lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval in auditors/postdata),
+    lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval in auditors/postdata),
     ctl_ds=&mpelib..mpe_lockanytable
   )
   proc sql;
@@ -662,7 +662,7 @@ run;
       num_of_approvals_remaining=&num_of_approvals_remaining-1,
       reviewed_by_nm="&user",
       reviewed_on_dttm=&sastime
-    where table_id="&table";
+    where table_id="&LOAD_REF";
   %mp_lockanytable(UNLOCK,
     lib=&mpelib,ds=mpe_submit,
     ctl_ds=&mpelib..mpe_lockanytable
@@ -688,7 +688,7 @@ run;
   %mpe_alerts(alert_event=APPROVED
     , alert_lib=&libref
     , alert_ds=&ds
-    , dsid=&TABLE
+    , dsid=&LOAD_REF
   )
 
   %removecolsfromwork(___TMP___MD5)

sas/sasjs/services/auditors/postdata.test.2.sas

@@ -34,7 +34,8 @@ run;
 %mp_testservice(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
-  outlib=web1
+  outlib=web1,
+  mdebug=&sasjs_mdebug
 )
 
 %let status=0;

sas/sasjs/services/editors/getdata.sas

@@ -14,8 +14,9 @@
   <h5> sasdata </h5>
   <h5> sasparams </h5>
   Contains info on the request.  One row is returned.
-  * CLS_FLG - set to 0 if there are no CLS rules (everything should be editable)
+  @li CLS_FLG - set to 0 if there are no CLS rules (everything should be editable)
     else set to 1 (CLS rules exist)
+  @li ISMAP - set to 1 if the target DS is an excel map target, else 0
 
   <h5> approvers </h5>
   <h5> dqrules </h5>
@@ -534,6 +535,11 @@ data _null_;
 run;
 
 %put params;
+%let ismap=0;
+proc sql noprint;
+select count(*) into: ismap from &mpelib..mpe_xlmap_info
+  where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
+
 data sasparams;
   length colHeaders $20000 filter_text $32767;
   colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
@@ -551,8 +557,11 @@ data sasparams;
   if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
   else cls_flag=1;
   put (_all_)(=);
+  if "&orig_libds"="&mpelib..MPE_XLMAP_DATA" or &ismap ne 0 then ismap=1;
+  else ismap=0;
 run;
 
+
 /* Extract validation DQ Rules */
 proc sort data=&mpelib..mpe_validations
   (where=(&dc_dttmtfmt. le TX_TO
@@ -639,8 +648,6 @@ proc sort data=dqdata;
   by base_col selectbox_order;
 run;
 
-
-
 %mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)
 
 data maxvarlengths;

sas/sasjs/services/editors/getdata.test.sas

@@ -141,13 +141,15 @@ run;
 data work.fmts;
   length fmtname $32;
   fmtname="&fmtname";
+  type='N';
   do start=1 to 10;
     label= cats("&fmtname",start);
+    end=start;
     output;
   end;
 run;
 proc sort data=work.fmts nodupkey;
-  by fmtname;
+  by fmtname type start;
 run;
 proc format cntlin=work.fmts library=dctest.dcfmts;
 run;
@@ -157,8 +159,9 @@ data work.inquery3;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
+  RAW_VALUE="'&fmtname'";
 datalines4;
-AND,AND,1,FMTNAME,CONTAINS,"'&fmtname'"
+AND,AND,1,FMTNAME,CONTAINS,placeholder (see line above)
 ;;;;
 run;
 %mp_filterstore(

sas/sasjs/services/editors/getxlmaps.sas

@@ -0,0 +1,82 @@
+/**
+  @file getxlmaps.sas
+  @brief Returns a list of rules and other info for a specific xlmap_id
+
+  <h4> Service Inputs </h4>
+
+  <h5> getxlmaps_in </h5>
+
+  |XLMAP_ID|
+  |---|
+  |Sample|
+
+  <h4> Service Outputs </h4>
+
+  <h5> xlmaprules </h5>
+
+  Filtered output of the dc.MPE_XLMAP_RULES table
+
+  |XLMAP_ID|XLMAP_RANGE_ID|XLMAP_SHEET|XLMAP_START|XLMAP_FINISH|
+  |---|---|---|---|---|
+  |Sample|Range1|Sheet1|ABSOLUTE A1| |
+  |Sample|Range2|Sheet1|RELATIVE R[2]C[2]|ABSOLUTE H11|
+
+  <h5> xlmapinfo </h5>
+  Extra info for a map id
+
+  |TARGET_DS|
+  |---|
+  |DCXXX.MPE_XLMAP_DATA|
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mpeinit.sas
+
+  @version 9.3
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%mpeinit()
+
+data _null_;
+  set work.getxlmaps_in;
+  putlog (_all_)(=);
+  call symputx('xlmap_id',xlmap_id);
+run;
+
+proc sql noprint;
+create table work.xlmaprules as
+  select xlmap_id
+    ,XLMAP_RANGE_ID
+    ,XLMAP_SHEET
+    ,XLMAP_START
+    ,XLMAP_FINISH
+  from &mpelib..MPE_XLMAP_RULES
+  where &dc_dttmtfmt. lt tx_to and xlmap_id="&xlmap_id"
+  order by xlmap_sheet, xlmap_range_id;
+
+%global target_ds;
+select XLMAP_TARGETLIBDS into: target_ds
+  from &mpelib..MPE_XLMAP_INFO
+  where &dc_dttmtfmt. lt tx_to and xlmap_id="&xlmap_id";
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program..sas
+  ,msg=%str(syscc=&syscc)
+)
+
+data work.xlmapinfo;
+  target_ds=coalescec("&target_ds","&mpelib..MPE_XLMAP_DATA");
+  output;
+  stop;
+run;
+
+%webout(OPEN)
+%webout(OBJ,xlmaprules)
+%webout(OBJ,xlmapinfo)
+%webout(CLOSE)
+

sas/sasjs/services/editors/getxlmaps.test.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief testing getxlmaps service
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mx_testservice.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+
+
+**/
+
+%let _program=&appLoc/services/editors/getxlmaps;
+
+/**
+  * Test 1 - basic send
+  */
+
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'XLMAP_ID:$char12.';
+  put "Sample";
+run;
+
+%mx_testservice(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:getxlmaps_in,
+  outlib=web1,
+  mdebug=&sasjs_mdebug
+)
+
+data work.xlmaprules;
+  set web1.xlmaprules;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.xlmaprules,
+  test=ATLEAST 2,
+  desc=Checking successful return of at least 2 rules for the Sample map,
+  outds=work.test_results
+)
+
+
+/**
+  * Test 2 - info returned
+  */
+data work.xlmapinfo;
+  set web1.xlmapinfo;
+  putlog (_all_)(=);
+  call symputx('tgtds',target_ds);
+run;
+%mp_assert(
+  iftrue=(&tgtds=&dclib..MPE_XLMAP_DATA),
+  desc=Checking correct target table is returned,
+  outds=work.test_results
+)

sas/sasjs/services/editors/stagedata.test.2.sas

@@ -24,6 +24,7 @@ proc format lib=DCTEST.DCFMTS cntlout=work.fmtextract;
 run;
 data work.jsdata;
   set work.fmtextract;
+  fmtrow=_n_;
   if _n_<5 then _____DELETE__THIS__RECORD_____='Yes';
   else _____DELETE__THIS__RECORD_____='No';
   if _n_>20 then stop;

sas/sasjs/services/hooks/mpe_xlmap_info_postedit.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_XLMAP_INFO table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li dc_assignlib.sas
+  @li mp_validatecol.sas
+
+**/
+
+data work.staging_ds;
+  set work.staging_ds;
+
+  /* apply the first excel map to all cells */
+  length tgtds $41;
+  retain tgtds;
+  drop tgtds is_libds;
+  if _n_=1 then do;
+    if missing(XLMAP_TARGETLIBDS) then tgtds="&dc_libref..MPE_XLMAP_DATA";
+    else tgtds=upcase(XLMAP_TARGETLIBDS);
+    %mp_validatecol(XLMAP_TARGETLIBDS,LIBDS,is_libds)
+    call symputx('tgtds',tgtds);
+    call symputx('is_libds',is_libds);
+  end;
+  XLMAP_TARGETLIBDS=tgtds;
+
+run;
+
+%mp_abort(iftrue=(&is_libds ne 1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=Invalid target dataset (&tgtds)
+)
+
+/**
+  * make sure that the supplied target dataset exists and
+  * has the necessary columns
+  */
+%dc_assignlib(READ,%scan(&tgtds,1,.))
+
+%mp_abort(iftrue=(%mf_existds(libds=&tgtds) ne 1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=Target dataset (&tgtds) could not be opened
+)
+
+%let tgtvars=%upcase(%mf_getvarlist(&tgtds));
+%let srcvars=%upcase(%mf_getvarlist(&dc_libref..MPE_XLMAP_DATA));
+%let badvars1=%mf_wordsInStr1ButNotStr2(Str1=&srcvars,Str2=&tgtvars);
+%let badvars2=%mf_wordsInStr1ButNotStr2(Str1=&tgtvars,Str2=&srcvars);
+
+%mp_abort(iftrue=(%length(&badvars1.X)>1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=%str(Target dataset (&tgtds) has missing vars: &badvars1)
+)
+
+%mp_abort(iftrue=(%length(&badvars2.X)>1)
+  ,mac=mpe_xlmap_info_postedit
+  ,msg=%str(Target dataset (&tgtds) has unrecognised vars: &badvars2)
+)

sas/sasjs/services/hooks/mpe_xlmap_rules_postedit.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_XLMAP_RULES table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+
+**/
+
+data work.staging_ds;
+  set work.staging_ds;
+
+  /* ensure uppercasing */
+  XLMAP_ID=upcase(XLMAP_ID);
+
+run;
+

sas/sasjs/services/hooks/sample_xlmap_data_postapprove.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Sample XLMAP Data hook program (sample_xlmap_data_postapprove)
+  @details This hook script should NOT be modified in place, as the changes
+    would be lost in your next Data Controller deployment.
+    Instead, create a copy of this hook script and place it OUTSIDE the
+    Data Controller metadata folder.
+
+  Available macro variables:
+    @li LOAD_REF - The Load Reference (unique upload id)
+    @li ORIG_LIBDS - The target library.dataset that was just loaded
+
+**/
+
+
+data _null_;
+  set work.staging_ds;
+  putlog 'load ref is in the staged data: ' load_ref;
+  stop;
+run;
+
+%put the unique identifier (LOAD_REF) is also a macro variable:  &LOAD_REF;

sas/sasjs/services/hooks/sample_xlmap_data_postedit.sas

@@ -0,0 +1,49 @@
+/**
+  @file
+  @brief Sample XLMAP Data hook program
+  @details This hook script should NOT be modified in place, as the changes
+    would be lost in your next Data Controller deployment.
+    Instead, create a copy of this hook script and place it OUTSIDE the
+    Data Controller metadata folder.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The target dataset being loaded
+
+**/
+
+%let abort=0;
+%let errmsg=;
+
+data work.staging_ds;
+  set work.staging_ds;
+  length errmsg $1000;
+  drop err:;
+  /* KM1 validations */
+  if XLMAP_ID='BASEL-KM1' then do;
+    if XLMAP_RANGE_ID='KM1:a' & input(value_txt,8.)<100 then do;
+      errmsg='Should be greater than 100';
+      err=1;
+    end;
+  end;
+  /* CR2 Validations */
+  if XLMAP_ID='BASEL-CR2' then do;
+    if XLMAP_RANGE_ID='CR2-sec1' & row_no=3 & input(value_txt,8.)>0 then do;
+      errmsg='Should be negative';
+      err=1;
+    end;
+  end;
+
+  /* publish error message */
+  if err=1 then do;
+    errmsg=catx(' ',xlmap_range_id,':',value_txt,'->',errmsg);
+    call symputx('errmsg',errmsg);
+    call symputx('abort',1);
+  end;
+run;
+
+%mp_abort(iftrue=(&abort ne 0)
+  ,mac=xlmap_data_postedit
+  ,msg=%superq(errmsg)
+)

sas/sasjs/services/public/startupservice.sas

@@ -3,10 +3,8 @@
   @brief List the libraries and tables the mp-editor user can access
   @details If user is in a control group (&mpeadmins, configured in mpeinit.sas)
     then they have access to all libraries / tables.  Otherwise a join is made
-    to the &mpelib..mp_editor_access table.
+    to the &mpelib..mpe_security table.
 
-    This service is also callable from EUCs - just add EUCDLM= parameter.
-    EUCDLM values:  TAB or CSV
 
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
@@ -129,10 +127,26 @@ create table saslibs as
   ,msg=%str(issue with security validation)
 )
 
+proc sql;
+create table work.xlmaps as
+  select distinct a.XLMAP_ID
+    ,b.XLMAP_DESCRIPTION
+    ,coalescec(b.XLMAP_TARGETLIBDS,"&mpelib..MPE_XLMAP_DATA")
+      as XLMAP_TARGETLIBDS
+  from &mpelib..MPE_XLMAP_RULES a
+  left join &mpelib..MPE_XLMAP_INFO(where=(&dc_dttmtfmt. lt tx_to)) b
+  on a.XLMAP_ID=b.XLMAP_ID
+  where &dc_dttmtfmt. lt a.tx_to;
+
+/* we don't want the XLMAP target datasets to be directly editable */
+delete from sasdatasets
+  where cats(libref,'.',dsn) in (select XLMAP_TARGETLIBDS from xlmaps);
+
 %webout(OPEN)
 %webout(OBJ,sasDatasets)
 %webout(OBJ,saslibs)
 %webout(OBJ,globvars)
+%webout(ARR,xlmaps)
 %webout(CLOSE)
 
 %mpeterm()

sas/sasjs/services/public/startupservice.test.sas

@@ -16,13 +16,24 @@
 )
 
 
-data globvars;
+data work.globvars;
   set webout.globvars;
   putlog (_all_)(=);
 run;
 
+data work.xlmaps;
+  set webout.xlmaps;
+  putlog (_all_)(=);
+run;
+
 %mp_assertdsobs(work.globvars,
   desc=Fromsas table returned,
   test=HASOBS,
   outds=work.test_results
 )
+
+%mp_assertdsobs(work.xlmaps,
+  desc=xlmaps table returned,
+  test=HASOBS,
+  outds=work.test_results
+)