Merge pull request 'demodata' (#203) from demodata into main

Reviewed-on: #203
2026-02-10 19:01:17 +00:00
parent 6ac3f660e9 6ceb681463
commit f97ac70678
38 changed files with 1948 additions and 956 deletions
--- a/.gitea/workflows/release.yaml
+++ b/.gitea/workflows/release.yaml
@@ -228,6 +228,8 @@ jobs:
          cp sasjs/utils/favicon.ico ../client/dist/favicon.ico
          sasjs c -t server
          rm -rf sasjsbuild/tests
+          server_apploc="/Public/app/dc"
+          sed -i "s|apploc=\"[^\"]*\"|apploc=\"${server_apploc}\"|g" sasjsbuild/services/web/index.html
          sasjs b -t server
          cp sasjsbuild/server.json.zip ./sasjs_server.json.zip

--- a/client/angular.json
+++ b/client/angular.json
@@ -41,6 +41,8 @@
              "zone.js",
              "text-encoding",
              "crypto-js/md5",
+              "crypto-js/sha1",
+              "crypto-js/sha512",
              "buffer",
              "numbro",
              "@clr/icons",
--- a/client/cypress/e2e/excel.cy.ts
+++ b/client/cypress/e2e/excel.cy.ts
@@ -309,6 +309,83 @@ context('excel tests: ', function () {
    })
  })

+  it('22 | Uploads password protected Excel and unlocks with correct password', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    cy.get('.buttonBar button:last-child')
+      .should('exist')
+      .click()
+      .then(() => {
+        cy.get('input[type="file"]#file-upload')
+          .attachFile(`/${fixturePath}/regular_excel_password.xlsx`)
+          .then(() => {
+            // Wait for password modal to appear
+            cy.get('#filePasswordInput', { timeout: 10000 })
+              .should('be.visible')
+              .type('123123')
+
+            // Click Unlock button
+            cy.get('.btn.btn-success-outline').should('not.be.disabled').click()
+
+            // Click away the overlay
+            cy.get('.modal-footer .btn.btn-primary', { timeout: 5000 }).click()
+
+            // Verify file loads successfully
+            cy.get('.btn-upload-preview', { timeout: 60000 })
+              .should('be.visible')
+              .then(() => {
+                submitExcel()
+                rejectExcel(done)
+              })
+          })
+      })
+  })
+
+  it('23 | Uploads password protected Excel and handles wrong password', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    cy.get('.buttonBar button:last-child')
+      .should('exist')
+      .click()
+      .then(() => {
+        cy.get('input[type="file"]#file-upload')
+          .attachFile(`/${fixturePath}/regular_excel_password.xlsx`)
+          .then(() => {
+            // First attempt: Enter wrong password
+            cy.get('#filePasswordInput', { timeout: 10000 })
+              .should('be.visible')
+              .type('wrongpassword')
+
+            cy.get('.btn.btn-success-outline').should('not.be.disabled').click()
+
+            // Verify error message appears
+            cy.get('.modal-footer .color-red', { timeout: 10000 })
+              .should('be.visible')
+              .should('contain', "Sorry that didn't work, try again.")
+
+            // Modal should still be open for retry
+            cy.get('#filePasswordInput')
+              .should('be.visible')
+              .clear()
+              .type('123123')
+
+            // Second attempt: Enter correct password
+            cy.get('.btn.btn-success-outline').should('not.be.disabled').click()
+
+            // Click away the overlay
+            cy.get('.modal-footer .btn.btn-primary', { timeout: 5000 }).click()
+
+            // Verify file loads successfully
+            cy.get('.btn-upload-preview', { timeout: 60000 })
+              .should('be.visible')
+              .then(() => {
+                submitExcel()
+                rejectExcel(done)
+              })
+          })
+      })
+  })
+
  // Large files break Cypress

  // it ('? | Uploads Excel with size of 5MB', (done) => {
--- a/client/cypress/fixtures/excels/regular_excel_password.xlsx
+++ b/client/cypress/fixtures/excels/regular_excel_password.xlsx
--- a/client/libraries/xlsx-0.20.3.tgz
+++ b/client/libraries/xlsx-0.20.3.tgz
--- a/client/package-lock.json
+++ b/client/package-lock.json
--- a/client/package.json
+++ b/client/package.json
@@ -37,21 +37,21 @@
  },
  "private": true,
  "dependencies": {
-    "@angular/animations": "^19.2.17",
+    "@angular/animations": "^19.2.18",
    "@angular/cdk": "^19.2.19",
-    "@angular/common": "^19.2.17",
-    "@angular/compiler": "^19.2.17",
-    "@angular/core": "^19.2.17",
-    "@angular/forms": "^19.2.17",
-    "@angular/platform-browser": "^19.2.17",
-    "@angular/platform-browser-dynamic": "^19.2.17",
-    "@angular/router": "^19.2.17",
+    "@angular/common": "^19.2.18",
+    "@angular/compiler": "^19.2.18",
+    "@angular/core": "^19.2.18",
+    "@angular/forms": "^19.2.18",
+    "@angular/platform-browser": "^19.2.18",
+    "@angular/platform-browser-dynamic": "^19.2.18",
+    "@angular/router": "^19.2.18",
    "@cds/core": "^6.15.1",
    "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
    "@clr/icons": "^13.0.2",
    "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
    "@handsontable/angular-wrapper": "16.0.1",
-    "@sasjs/adapter": "^4.16.2",
+    "@sasjs/adapter": "^4.16.3",
    "@sasjs/utils": "^3.5.3",
    "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
    "@types/d3-graphviz": "^2.6.7",
@@ -82,7 +82,7 @@
    "tslib": "^2.3.0",
    "vm": "^0.1.0",
    "webpack": "^5.91.0",
-    "xlsx": "^0.18.5",
+    "xlsx": "file:libraries/xlsx-0.20.3.tgz",
    "zone.js": "~0.15.1"
  },
  "devDependencies": {
@@ -93,7 +93,7 @@
    "@angular-eslint/schematics": "19.8.1",
    "@angular-eslint/template-parser": "19.8.1",
    "@angular/cli": "^19.2.19",
-    "@angular/compiler-cli": "^19.2.17",
+    "@angular/compiler-cli": "^19.2.18",
    "@babel/plugin-proposal-private-methods": "^7.18.6",
    "@compodoc/compodoc": "^1.1.21",
    "@cypress/webpack-preprocessor": "^5.17.1",
--- a/client/src/app/editor/components/edit-record/edit-record.component.ts
+++ b/client/src/app/editor/components/edit-record/edit-record.component.ts
@@ -14,6 +14,7 @@ import { HelperService } from 'src/app/services/helper.service'
 import { SasStoreService } from 'src/app/services/sas-store.service'
 import { DcValidator } from 'src/app/shared/dc-validator/dc-validator'
 import { DcValidation } from 'src/app/shared/dc-validator/models/dc-validation.model'
+import { isEmpty } from 'src/app/shared/dc-validator/utils/isEmpty'
 import {
  EditRecordDropdownChangeEvent,
  EditRecordInputFocusedEvent
@@ -146,23 +147,63 @@ export class EditRecordComponent implements OnInit {
    }, 0)
  }

-  async recordInputChange(event: any, colName: string) {
+  async recordInputChange(event: any, colName: string): Promise<void> {
    const colRules = this.currentRecordValidator?.getRule(colName)
    const value = event.target.value

    this.helperService.debounceCall(300, () => {
      this.validateRecordCol(colRules, value).then((valid: boolean) => {
-        const index = this.currentRecordInvalidCols.indexOf(colName)
+        this.updateValidationState(colName, valid)

-        if (valid) {
-          if (index > -1) this.currentRecordInvalidCols.splice(index, 1)
-        } else {
-          if (index < 0) this.currentRecordInvalidCols.push(colName)
+        if (!valid) {
+          this.tryAutoPopulateNotNull(event, colName, colRules, value)
        }
      })
    })
  }

+  /**
+   * Updates the invalid columns list based on validation result
+   */
+  private updateValidationState(colName: string, valid: boolean): void {
+    const index = this.currentRecordInvalidCols.indexOf(colName)
+
+    if (valid && index > -1) {
+      this.currentRecordInvalidCols.splice(index, 1)
+    } else if (!valid && index < 0) {
+      this.currentRecordInvalidCols.push(colName)
+    }
+  }
+
+  /**
+   * Auto-populates NOTNULL default value when the field is empty and has a default
+   */
+  private tryAutoPopulateNotNull(
+    event: any,
+    colName: string,
+    colRules: DcValidation | undefined,
+    value: any
+  ): void {
+    if (
+      !isEmpty(value) ||
+      !this.currentRecordValidator ||
+      !this.currentRecord
+    ) {
+      return
+    }
+
+    const defaultValue =
+      this.currentRecordValidator.getNotNullDefaultValue(colName)
+    if (defaultValue === undefined) return
+
+    this.currentRecord[colName] = defaultValue
+    event.target.value = defaultValue
+
+    this.validateRecordCol(colRules, defaultValue).then((isValid: boolean) => {
+      this.updateValidationState(colName, isValid)
+    })
+  }
+
  onNextRecordClick() {
    this.onNextRecord.emit()
  }
--- a/client/src/app/editor/editor.component.ts
+++ b/client/src/app/editor/editor.component.ts
@@ -43,6 +43,7 @@ import { Col } from '../shared/dc-validator/models/col.model'
 import { DcValidation } from '../shared/dc-validator/models/dc-validation.model'
 import { DQRule } from '../shared/dc-validator/models/dq-rules.model'
 import { getHotDataSchema } from '../shared/dc-validator/utils/getHotDataSchema'
+import { isEmpty } from '../shared/dc-validator/utils/isEmpty'
 import { globals } from '../_globals'
 import { UploadStaterComponent } from './components/upload-stater/upload-stater.component'
 import { DynamicExtendedCellValidation } from './models/dynamicExtendedCellValidation'
@@ -1045,12 +1046,16 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
  }

  /**
-   * Creates a new empty row object with proper structure
+   * Creates a new empty row object with proper structure.
+   * Columns with NOTNULL DQ rules are pre-populated with their RULE_VALUE.
   */
  private createEmptyRow(): any {
    const newRow: any = {}
-    this.headerColumns.forEach((col: string) => {
-      newRow[col] = ''
+    this.cellValidation.forEach((rule: any) => {
+      const dataKey = rule.data
+      newRow[dataKey] = this.hotDataSchema.hasOwnProperty(dataKey)
+        ? this.hotDataSchema[dataKey]
+        : ''
    })
    newRow['noLinkOption'] = true
    return newRow
@@ -2676,13 +2681,14 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
    // Note: this.headerColumns and this.columnHeader contains same data
    // need to resolve redundancy

-    // default schema
+    // default schema - includes NOTNULL defaults from DQ rules
    for (let i = 0; i < this.headerColumns.length; i++) {
      const colType = this.cellValidation[i].type

      this.hotDataSchema[this.cellValidation[i].data] = getHotDataSchema(
        colType,
-        this.cellValidation[i]
+        this.cellValidation[i],
+        this.dcValidator?.getDqDetails()
      )
    }

@@ -2987,6 +2993,29 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
      }
    )

+    // Auto-populate NOTNULL default when validation fails due to empty value
+    hot.addHook(
+      'afterValidate',
+      (isValid: boolean, value: any, row: number, prop: string | number) => {
+        if (isValid || !isEmpty(value)) return
+
+        const colName =
+          typeof prop === 'string'
+            ? prop
+            : (hot.colToProp(prop as number) as string)
+
+        const defaultValue = this.dcValidator?.getNotNullDefaultValue(colName)
+        if (defaultValue === undefined) return
+
+        // Auto-populate using setTimeout to avoid modifying during validation
+        setTimeout(() => {
+          if (isEmpty(hot.getDataAtRowProp(row, colName))) {
+            hot.setDataAtRowProp(row, colName, defaultValue, 'autoPopulate')
+          }
+        }, 0)
+      }
+    )
+
    hot.addHook('beforePaste', (data: any, cords: any) => {
      const startCol = cords[0].startCol

--- a/client/src/app/lineage/lineage.component.html
+++ b/client/src/app/lineage/lineage.component.html
@@ -239,13 +239,7 @@

              <clr-dropdown-menu clrPosition="bottom-left" *clrIfOpen>
                <div (click)="downloadSVG()" clrDropdownItem>SVG</div>
-                <div
-                  *ngIf="!helperService.isMicrosoft"
-                  (click)="downloadPNG()"
-                  clrDropdownItem
-                >
-                  PNG
-                </div>
+                <div (click)="downloadPNG()" clrDropdownItem>PNG</div>
                <div (click)="downloadDot()" clrDropdownItem>Dot</div>
                <div *ngIf="flatdata" (click)="downloadCSV()" clrDropdownItem>
                  CSV
@@ -366,13 +360,7 @@

        <clr-dropdown-menu clrPosition="bottom-left" *clrIfOpen>
          <div (click)="renderToDownload('SVG')" clrDropdownItem>SVG</div>
-          <div
-            *ngIf="!helperService.isMicrosoft"
-            (click)="renderToDownload('PNG')"
-            clrDropdownItem
-          >
-            PNG
-          </div>
+          <div (click)="renderToDownload('PNG')" clrDropdownItem>PNG</div>
          <div (click)="downloadDot(); cancelRenderingGraph()" clrDropdownItem>
            Dot
          </div>
--- a/client/src/app/services/helper.service.ts
+++ b/client/src/app/services/helper.service.ts
@@ -11,12 +11,8 @@ const librariesToShow = 50
 export class HelperService {
  public shownLibraries: number = librariesToShow
  public loadMoreCount: number = librariesToShow
-  public isMicrosoft: boolean = false

-  constructor(private sasService: SasService) {
-    this.isMicrosoft = this.isIEorEDGE()
-    console.log('Is IE or Edge?', this.isMicrosoft)
-  }
+  constructor(private sasService: SasService) {}

  /**
   * Converts a JavaScript date object to a SAS Date or Datetime, given the logic below:
@@ -215,32 +211,6 @@ export class HelperService {
    })
  }

-  public isIEorEDGE() {
-    var ua = window.navigator.userAgent
-
-    var msie = ua.indexOf('MSIE ')
-    if (msie > 0) {
-      // IE 10 or older => return version number
-      return true
-    }
-
-    var trident = ua.indexOf('Trident/')
-    if (trident > 0) {
-      // IE 11 => return version number
-      var rv = ua.indexOf('rv:')
-      return true
-    }
-
-    var edge = ua.indexOf('Edge/')
-    if (edge > 0) {
-      // Edge (IE 12+) => return version number
-      return true
-    }
-
-    // other browser
-    return false
-  }
-
  public convertObjectsToArray(
    objectArray: Array<object>,
    deepClone: boolean = false
--- a/client/src/app/services/sas-viya.service.ts
+++ b/client/src/app/services/sas-viya.service.ts
@@ -120,9 +120,12 @@ export class SasViyaService {
  }

  getComputeContexts(): Observable<ViyaComputeContexts> {
-    return this.get<ViyaComputeContexts>(`${this.serverUrl}/compute/contexts`, {
-      withCredentials: true
-    })
+    return this.get<ViyaComputeContexts>(
+      `${this.serverUrl}/compute/contexts?limit=1000`,
+      {
+        withCredentials: true
+      }
+    )
  }

  getComputeContextById(id: string): Observable<ComputeContextDetails> {
--- a/client/src/app/shared/dc-validator/dc-validator.ts
+++ b/client/src/app/shared/dc-validator/dc-validator.ts
@@ -15,6 +15,7 @@ import {
 } from './models/dc-validation.model'
 import { DQRule, DQRuleTypes } from './models/dq-rules.model'
 import { getDqDataCols } from './utils/getDqDataCols'
+import { getNotNullDefault } from './utils/getNotNullDefault'
 import { mergeColsRules } from './utils/mergeColsRules'
 import { parseColType } from './utils/parseColType'
 import { dqValidate } from './validations/dq-validation'
@@ -133,6 +134,19 @@ export class DcValidator {
    }
  }

+  /**
+   * Returns the RULE_VALUE for a NOTNULL rule on the given column.
+   * Used for auto-populating default values when cells are empty.
+   * Converts to number for numeric columns.
+   *
+   * @param col column name
+   * @returns RULE_VALUE (string or number) if NOTNULL rule exists, otherwise undefined
+   */
+  getNotNullDefaultValue(col: string): string | number | undefined {
+    const colRule = this.getRule(col)
+    return getNotNullDefault(col, this.dqrules, colRule?.type)
+  }
+
  /**
   * Retrieves dropdown source for given dc validation rule
   * The values comes from MPE_SELECTBOX table
--- a/client/src/app/shared/dc-validator/tests/getHotDataSchema.spec.ts
+++ b/client/src/app/shared/dc-validator/tests/getHotDataSchema.spec.ts
@@ -1,3 +1,4 @@
+import { DQRule } from '../models/dq-rules.model'
 import { getHotDataSchema } from '../utils/getHotDataSchema'

 describe('DC Validator - hot data schema', () => {
@@ -8,4 +9,58 @@ describe('DC Validator - hot data schema', () => {
    ).toEqual(1)
    expect(getHotDataSchema('missing')).toEqual('')
  })
+
+  describe('NOTNULL defaults', () => {
+    const dqRules: DQRule[] = [
+      {
+        BASE_COL: 'TEXT_COL',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: 'default_text',
+        X: 1
+      },
+      { BASE_COL: 'NUM_COL', RULE_TYPE: 'NOTNULL', RULE_VALUE: '42', X: 1 }
+    ]
+
+    it('should return NOTNULL default for text column', () => {
+      expect(getHotDataSchema('text', { data: 'TEXT_COL' }, dqRules)).toEqual(
+        'default_text'
+      )
+    })
+
+    it('should return NOTNULL default as number for numeric column', () => {
+      expect(getHotDataSchema('numeric', { data: 'NUM_COL' }, dqRules)).toEqual(
+        42
+      )
+    })
+
+    it('should fall back to type default when no NOTNULL rule exists', () => {
+      expect(
+        getHotDataSchema('numeric', { data: 'OTHER_COL' }, dqRules)
+      ).toEqual('')
+    })
+
+    it('should prioritize NOTNULL over autocomplete first option', () => {
+      const rulesWithAutocomplete: DQRule[] = [
+        {
+          BASE_COL: 'SELECT_COL',
+          RULE_TYPE: 'NOTNULL',
+          RULE_VALUE: 'priority_value',
+          X: 1
+        },
+        {
+          BASE_COL: 'SELECT_COL',
+          RULE_TYPE: 'HARDSELECT',
+          RULE_VALUE: 'ignored',
+          X: 1
+        }
+      ]
+      expect(
+        getHotDataSchema(
+          'autocomplete',
+          { data: 'SELECT_COL', source: ['first', 'second'] },
+          rulesWithAutocomplete
+        )
+      ).toEqual('priority_value')
+    })
+  })
 })
--- a/client/src/app/shared/dc-validator/tests/getNotNullDefault.spec.ts
+++ b/client/src/app/shared/dc-validator/tests/getNotNullDefault.spec.ts
@@ -0,0 +1,65 @@
+import { DQRule } from '../models/dq-rules.model'
+import { getNotNullDefault } from '../utils/getNotNullDefault'
+
+describe('DC Validator - getNotNullDefault', () => {
+  const dqRules: DQRule[] = [
+    {
+      BASE_COL: 'TEXT_COL',
+      RULE_TYPE: 'NOTNULL',
+      RULE_VALUE: 'default_text',
+      X: 1
+    },
+    { BASE_COL: 'NUM_COL', RULE_TYPE: 'NOTNULL', RULE_VALUE: '42', X: 1 },
+    { BASE_COL: 'EMPTY_RULE', RULE_TYPE: 'NOTNULL', RULE_VALUE: '  ', X: 1 },
+    {
+      BASE_COL: 'OTHER_COL',
+      RULE_TYPE: 'HARDSELECT',
+      RULE_VALUE: 'some_value',
+      X: 1
+    }
+  ]
+
+  it('should return string value for text columns', () => {
+    expect(getNotNullDefault('TEXT_COL', dqRules, 'text')).toEqual(
+      'default_text'
+    )
+  })
+
+  it('should return number for numeric columns when RULE_VALUE is numeric', () => {
+    expect(getNotNullDefault('NUM_COL', dqRules, 'numeric')).toEqual(42)
+  })
+
+  it('should return string for numeric columns when RULE_VALUE is not numeric', () => {
+    const rulesWithNonNumeric: DQRule[] = [
+      {
+        BASE_COL: 'NUM_COL',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: 'not_a_number',
+        X: 1
+      }
+    ]
+    expect(
+      getNotNullDefault('NUM_COL', rulesWithNonNumeric, 'numeric')
+    ).toEqual('not_a_number')
+  })
+
+  it('should return undefined for empty RULE_VALUE', () => {
+    expect(getNotNullDefault('EMPTY_RULE', dqRules, 'text')).toBeUndefined()
+  })
+
+  it('should return undefined for columns without NOTNULL rule', () => {
+    expect(getNotNullDefault('OTHER_COL', dqRules, 'text')).toBeUndefined()
+  })
+
+  it('should return undefined for non-existent columns', () => {
+    expect(getNotNullDefault('MISSING_COL', dqRules, 'text')).toBeUndefined()
+  })
+
+  it('should return undefined for empty dqRules array', () => {
+    expect(getNotNullDefault('TEXT_COL', [], 'text')).toBeUndefined()
+  })
+
+  it('should return string when colType is undefined', () => {
+    expect(getNotNullDefault('NUM_COL', dqRules, undefined)).toEqual('42')
+  })
+})
--- a/client/src/app/shared/dc-validator/tests/isEmpty.spec.ts
+++ b/client/src/app/shared/dc-validator/tests/isEmpty.spec.ts
@@ -0,0 +1,39 @@
+import { isEmpty } from '../utils/isEmpty'
+
+describe('DC Validator - isEmpty', () => {
+  it('should return true for null', () => {
+    expect(isEmpty(null)).toBe(true)
+  })
+
+  it('should return true for undefined', () => {
+    expect(isEmpty(undefined)).toBe(true)
+  })
+
+  it('should return true for empty string', () => {
+    expect(isEmpty('')).toBe(true)
+  })
+
+  it('should return true for whitespace-only string', () => {
+    expect(isEmpty('   ')).toBe(true)
+    expect(isEmpty('\t\n')).toBe(true)
+  })
+
+  it('should return false for non-empty string', () => {
+    expect(isEmpty('hello')).toBe(false)
+    expect(isEmpty(' hello ')).toBe(false)
+  })
+
+  it('should return false for number zero', () => {
+    expect(isEmpty(0)).toBe(false)
+  })
+
+  it('should return false for non-zero numbers', () => {
+    expect(isEmpty(42)).toBe(false)
+    expect(isEmpty(-1)).toBe(false)
+  })
+
+  it('should return false for boolean values', () => {
+    expect(isEmpty(true)).toBe(false)
+    expect(isEmpty(false)).toBe(false)
+  })
+})
--- a/client/src/app/shared/dc-validator/utils/getHotDataSchema.ts
+++ b/client/src/app/shared/dc-validator/utils/getHotDataSchema.ts
@@ -1,4 +1,6 @@
 import { DcValidation } from '../models/dc-validation.model'
+import { DQRule } from '../models/dq-rules.model'
+import { getNotNullDefault } from './getNotNullDefault'

 const schemaTypeMap: { [key: string]: any } = {
  numeric: '',
@@ -7,14 +9,25 @@ const schemaTypeMap: { [key: string]: any } = {

 /**
 * Schema defines the default values for given types. For example when new row is added.
+ * Priority: NOTNULL RULE_VALUE > autocomplete first option > type default
 */
-export const getHotDataSchema = (
+export function getHotDataSchema(
  type: string | undefined,
-  cellValidation?: DcValidation
-): any => {
+  cellValidation?: DcValidation,
+  dqRules?: DQRule[]
+): any {
+  // Check for NOTNULL default first
+  if (dqRules && cellValidation?.data) {
+    const defaultValue = getNotNullDefault(cellValidation.data, dqRules, type)
+    if (defaultValue !== undefined) {
+      return defaultValue
+    }
+  }
+
  if (!type) return schemaTypeMap.default

  switch (type) {
+    case 'dropdown':
    case 'autocomplete': {
      return cellValidation && cellValidation.source
        ? (cellValidation.source as string[] | number[])[0]
--- a/client/src/app/shared/dc-validator/utils/getNotNullDefault.ts
+++ b/client/src/app/shared/dc-validator/utils/getNotNullDefault.ts
@@ -0,0 +1,30 @@
+import { DQRule } from '../models/dq-rules.model'
+
+/**
+ * Returns the NOTNULL default value for a column from DQ rules.
+ * Converts to number for numeric columns based on colType parameter.
+ *
+ * @param colName column name to look up
+ * @param dqRules array of DQ rules
+ * @param colType column type (e.g., 'numeric', 'text')
+ * @returns default value (string or number) if NOTNULL rule exists with non-empty value, otherwise undefined
+ */
+export function getNotNullDefault(
+  colName: string,
+  dqRules: DQRule[],
+  colType?: string
+): string | number | undefined {
+  const notNullRule = dqRules.find(
+    (rule: DQRule) => rule.BASE_COL === colName && rule.RULE_TYPE === 'NOTNULL'
+  )
+
+  if (!notNullRule?.RULE_VALUE || notNullRule.RULE_VALUE.trim().length === 0) {
+    return undefined
+  }
+
+  if (colType === 'numeric' && !isNaN(Number(notNullRule.RULE_VALUE))) {
+    return Number(notNullRule.RULE_VALUE)
+  }
+
+  return notNullRule.RULE_VALUE
+}
--- a/client/src/app/shared/dc-validator/utils/isEmpty.ts
+++ b/client/src/app/shared/dc-validator/utils/isEmpty.ts
@@ -0,0 +1,8 @@
+/**
+ * Checks if a value is considered empty for NOTNULL validation purposes.
+ * A value is empty if it's null, undefined, or a string that is blank after trimming.
+ */
+export function isEmpty(value: unknown): boolean {
+  if (value === null || value === undefined) return true
+  return value.toString().trim().length === 0
+}
--- a/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts
+++ b/client/src/app/shared/spreadsheet-util/spreadsheet-util.ts
@@ -511,6 +511,21 @@ export class SpreadsheetUtil {
        return resolve(XLSX.read(data, opts))
      }

+      // TEMPORARILY DISABLED: Web Worker for XLSX parsing
+      // Worker is disabled because Angular/webpack bundles it as a separate chunk
+      // with a numeric filename (e.g., 411.hash.js). In SAS9/Viya streaming
+      // environments, all JS files need to be served through SASJobExecution
+      // with _program= parameter, but our post-build processor can't reliably
+      // find and replace the worker chunk reference in the minified output.
+      // FIX: Add "namedChunks": true to production config in angular.json
+      // (under projects.datacontroller.architect.build.configurations.production)
+      // This will output worker as "spreadsheet-worker.hash.js" instead of
+      // numeric ID, making it findable by post-processor.
+      // Trade-off: UI may briefly freeze when parsing large Excel files.
+
+      return resolve(XLSX.read(data, opts))
+
+      /*
      if (typeof Worker === 'undefined') {
        console.info(
          'Not using worker to parse the XLSX - no Worker available in this environment'
@@ -551,6 +566,7 @@ export class SpreadsheetUtil {
      setTimeout(() => {
        return resolve(XLSX.read(data, opts))
      }, 600 * 1000) // 10 minutes
+      */
    })
  }

--- a/client/src/app/spreadsheet.worker.ts
+++ b/client/src/app/spreadsheet.worker.ts
@@ -4,6 +4,9 @@
 * We use normal version of the XLSX (SheetJS)
 * Because at the moment "@sheet/crypto" can't work in the Web Worker environment
 * Because of the missing "global" variable.
+ *
+ * Version bumped to v0.20.3 (`libraries/xlsx-0.20.3.tgz`)
+ * @see https://cdn.sheetjs.com/
 */
 import * as XLSX from 'xlsx'

--- a/client/src/app/viewer/viewer.component.ts
+++ b/client/src/app/viewer/viewer.component.ts
@@ -160,11 +160,7 @@ export class ViewerComponent
    afterGetColHeader: (col: number, th: any, headerLevel: number) => {
      // CRITICAL: Prevent "colToProp method cannot be called because this Handsontable instance has been destroyed" error
      // This callback can be triggered even after the instance is destroyed during rapid table switching
-      if (
-        !this.hotInstance ||
-        this.hotInstance.isDestroyed ||
-        this.isTableSwitching
-      ) {
+      if (!this.hotInstance || this.hotInstance.isDestroyed) {
        // Graceful fallback: apply only dark mode styling when instance is unavailable
        th.classList.add(globals.handsontable.darkTableHeaderClass)
        return
@@ -761,10 +757,6 @@ export class ViewerComponent
    // This prevents callbacks from accessing destroyed instances during table switching
    this.isTableSwitching = true

-    // CLEANUP: Ensure any existing Handsontable instance is properly destroyed
-    // This prevents "instance destroyed" errors
-    this.cleanupHotInstance()
-
    this.loadingTableView = true

    let libDataset: any
@@ -1177,8 +1169,6 @@ export class ViewerComponent
   * Purpose: Prevents "instance destroyed" errors and memory leaks during table switching
   *
   * Called from:
-   * - viewData() - before loading new table data
-   * - setupHot() - before creating new instance
   * - ngOnDestroy() - component cleanup
   *
   * Safety features:
@@ -1195,107 +1185,113 @@ export class ViewerComponent
      }
    }
    this.hotInstance = null
+    this.hooksAttached = false
  }

  /**
   * PERFORMANCE: Configures Handsontable with enhanced error handling (workaround needed for HOT version 16 and above)
   *
   * 1. Duplicate call prevention (500ms window)
-   * 2. Reduced timeout delays (200ms + 50ms vs original 1000ms + 200ms)
-   * 3. Multiple validation checks to prevent race conditions
-   * 4. Forced render for immediate primary key styling
+   * 2. Multiple validation checks to prevent race conditions
+   * 3. Forced render for immediate primary key styling
   *
-   * Timeline: 50ms (viewData) + 200ms (main) + 50ms (component ready) = ~300ms total
-   * Previous: 100ms + 600ms + 100ms = 800ms (plus render delays = ~2 seconds)
+   * Instance lifecycle is managed by Angular's hot-table component via [data] and [settings] bindings.
+   * This method only applies additional config that can't go through bindings (hooks, PK styling).
   */
  private setupHot() {
-    // DUPLICATE PREVENTION: Avoid multiple setup calls during rapid table switching
    const now = Date.now()
    if (now - this.lastSetupTime < 500) {
      return
    }
    this.lastSetupTime = now

+    // VALIDATION: Don't setup if we're currently switching tables or still loading
+    if (this.loadingTableView || !this.libDataset) {
+      return
+    }
+
+    if (!this.hotInstance || this.hotInstance.isDestroyed) {
+      this.hotInstance = this.hotTableComponent?.hotInstance
+    }
+
+    if (this.hotInstance && !this.hotInstance.isDestroyed) {
+      this.configureHotInstance()
+      return
+    }
+
+    // Instance not ready yet — Angular may still be creating the component
    setTimeout(() => {
-      // VALIDATION: Don't setup if we're currently switching tables or still loading
-      if (this.loadingTableView || !this.libDataset) {
+      if (this.isTableSwitching || this.loadingTableView || !this.libDataset) {
        return
      }

-      // CLEANUP: Ensure clean slate before new setup
-      this.cleanupHotInstance()
+      this.hotInstance = this.hotTableComponent?.hotInstance
+      this.configureHotInstance()
+    }, 250)
+  }

-      // TIMING: Wait for Angular component to be ready (optimized from 100ms to 50ms)
-      setTimeout(() => {
-        // DOUBLE-CHECK: Ensure we're still in valid state after delays
-        if (
-          this.isTableSwitching ||
-          this.loadingTableView ||
-          !this.libDataset
-        ) {
+  private hooksAttached = false
+
+  /**
+   * Applies settings that can't go through Angular [settings] binding:
+   * - Primary key column header styling
+   * - Column width cap
+   * - ARIA accessibility hooks (attached once per instance)
+   */
+  private configureHotInstance() {
+    if (!this.hotInstance || this.hotInstance.isDestroyed) return
+
+    this.hotInstance.updateSettings({
+      height: this.hotTable.height,
+      modifyColWidth: (width: any, col: any) => {
+        if (width > 500) return 500
+        else return width
+      },
+      afterGetColHeader: (col: number, th: any) => {
+        // CRITICAL: Same safety checks as initial callback to prevent destroyed instance errors
+        if (!this.hotInstance || this.hotInstance.isDestroyed) {
+          th.classList.add(globals.handsontable.darkTableHeaderClass)
          return
        }

-        this.hotInstance = this.hotTableComponent?.hotInstance
+        try {
+          const column = this.hotInstance.colToProp(col) as string

-        if (this.hotInstance && !this.hotInstance.isDestroyed) {
-          this.hotInstance.updateSettings({
-            height: this.hotTable.height,
-            modifyColWidth: (width: any, col: any) => {
-              if (width > 500) return 500
-              else return width
-            },
-            afterGetColHeader: (col: number, th: any) => {
-              // CRITICAL: Same safety checks as initial callback to prevent destroyed instance errors
-              if (
-                !this.hotInstance ||
-                this.hotInstance.isDestroyed ||
-                this.isTableSwitching
-              ) {
-                th.classList.add(globals.handsontable.darkTableHeaderClass)
-                return
-              }
+          // PRIMARY KEY STYLING: Apply special styling to PK columns (populated from API response)
+          const isPKCol = column && this.headerPks.indexOf(column) > -1
+          if (isPKCol) th.classList.add('primaryKeyHeaderStyle')

-              try {
-                const column = this.hotInstance.colToProp(col) as string
-
-                // PRIMARY KEY STYLING: Apply special styling to PK columns (populated from API response)
-                const isPKCol = column && this.headerPks.indexOf(column) > -1
-                if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
-
-                // DARK MODE: Apply to all headers
-                th.classList.add(globals.handsontable.darkTableHeaderClass)
-              } catch (error) {
-                // SAFETY NET: Ensure basic styling is always applied
-                th.classList.add(globals.handsontable.darkTableHeaderClass)
-              }
-            }
-          })
-
-          // Add hooks for accessibility fixes
-          this.hotInstance.addHook('afterRender', () => {
-            // Fix ARIA accessibility issues after each render
-            this.fixAriaAccessibility()
-          })
-
-          this.hotInstance.addHook('afterChange', () => {
-            // Fix ARIA accessibility issues after any data change
-            setTimeout(() => {
-              this.fixAriaAccessibility()
-            }, 50)
-          })
-
-          // Force immediate render to apply primary key styling
-          // Without this, styling would wait for ~2 seconds to be applied
-          // With this, styling appears in ~300ms total (workaround needed for HOT version 16 and above)
-          setTimeout(() => {
-            if (this.hotInstance && !this.hotInstance.isDestroyed) {
-              this.hotInstance.render()
-            }
-          }, 10)
+          // DARK MODE: Apply to all headers
+          th.classList.add(globals.handsontable.darkTableHeaderClass)
+        } catch (error) {
+          // SAFETY NET: Ensure basic styling is always applied
+          th.classList.add(globals.handsontable.darkTableHeaderClass)
        }
-      }, 50) // Optimized Angular component readiness delay
-    }, 200) // Optimized main setup delay (was 600ms)
+      }
+    })
+
+    // Add hooks for accessibility fixes
+    // Hooks are attached once per instance to avoid accumulating duplicate listeners
+    if (!this.hooksAttached) {
+      this.hotInstance.addHook('afterRender', () => {
+        // Fix ARIA accessibility issues after each render
+        this.fixAriaAccessibility()
+      })
+
+      this.hotInstance.addHook('afterChange', () => {
+        // Fix ARIA accessibility issues after any data change
+        setTimeout(() => {
+          this.fixAriaAccessibility()
+        }, 50)
+      })
+
+      this.hooksAttached = true
+    }
+
+    // Force immediate render to apply primary key styling
+    // Without this, styling would wait for ~2 seconds to be applied
+    // (workaround needed for HOT version 16 and above)
+    this.hotInstance.render()
  }

  async loadWithParameters() {
--- a/client/tsconfig.app.json
+++ b/client/tsconfig.app.json
@@ -10,12 +10,6 @@
    "outDir": "./app",
    "types": []
  },
-  "files": [
-    "src/polyfills.ts",
-    "src/main.ts",
-    "src/app/app.d.ts"
-  ],
-  "include": [
-    "src/**/*.d.ts"
-  ]
-}
+  "files": ["src/polyfills.ts", "src/main.ts", "src/app/app.d.ts"],
+  "include": ["src/**/*.d.ts"]
+}
--- a/client/tsconfig.json
+++ b/client/tsconfig.json
@@ -1,55 +1,37 @@
 /* To learn more about this file see: https://angular.io/config/tsconfig. */
 {
-    "compileOnSave": false,
-    "compilerOptions": {
-      "baseUrl": "",
-      "outDir": "dist",
-      "downlevelIteration": true,
-      "declaration": false,
-      "experimentalDecorators": true,
-      "allowSyntheticDefaultImports": true,
-      "strict": true,
-      "lib": [
-        "ES2022",
-        "dom"
-      ],
-      "skipLibCheck": true,
-      "module": "ES2022",
-      "importHelpers": true,
-      "moduleResolution": "node",
-      "sourceMap": true,
-      "resolveJsonModule": true,
-      "target": "ES2022",
-      "paths": {
-        "crypto": [
-          "./node_modules/crypto-browserify"
-        ],
-        "stream": [
-          "./node_modules/stream-browserify"
-        ],
-        "assert": [
-          "./node_modules/assert"
-        ],
-        "http": [
-          "./node_modules/stream-http"
-        ],
-        "https": [
-          "./node_modules/https-browserify"
-        ],
-        "os": [
-          "./node_modules/os-browserify"
-        ]
-      },
-      "useDefineForClassFields": false
+  "compileOnSave": false,
+  "compilerOptions": {
+    "baseUrl": "",
+    "outDir": "dist",
+    "downlevelIteration": true,
+    "declaration": false,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "lib": ["ES2022", "dom"],
+    "skipLibCheck": true,
+    "module": "ES2022",
+    "importHelpers": true,
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "target": "ES2022",
+    "paths": {
+      "crypto": ["./node_modules/crypto-browserify"],
+      "stream": ["./node_modules/stream-browserify"],
+      "assert": ["./node_modules/assert"],
+      "http": ["./node_modules/stream-http"],
+      "https": ["./node_modules/https-browserify"],
+      "os": ["./node_modules/os-browserify"]
    },
-    "angularCompilerOptions": {
-      "enableI18nLegacyMessageIdFormat": false,
-      "strictInjectionParameters": true,
-      "strictInputAccessModifiers": true,
-      "strictTemplates": true,
-    },
-    "exclude": [
-      "cypress/**/*.ts",
-      "cypress.config.ts"
-    ]
+    "useDefineForClassFields": false
+  },
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  },
+  "exclude": ["cypress/**/*.ts", "cypress.config.ts"]
 }
--- a/client/tsconfig.spec.json
+++ b/client/tsconfig.spec.json
@@ -3,15 +3,8 @@
  "extends": "./tsconfig.json",
  "compilerOptions": {
    "outDir": "./out-tsc/spec",
-    "types": [
-      "jasmine"
-    ]
+    "types": ["jasmine"]
  },
-  "files": [
-    "src/polyfills.ts"
-  ],
-  "include": [
-    "src/**/*.spec.ts",
-    "src/**/*.d.ts"
-  ]
+  "files": ["src/polyfills.ts"],
+  "include": ["src/**/*.spec.ts", "src/**/*.d.ts"]
 }
--- a/sas/deploy/makeDataServer.json
+++ b/sas/deploy/makeDataServer.json
@@ -2,7 +2,7 @@
  "fromjs": [
    {
      "ADMIN": "DCDEFAULT",
-      "DCPATH": "/tmp/mihajlo/dcserverfrs"
+      "DCPATH": "/tmp/dcdata"
    }
  ]
 }
--- a/sas/package-lock.json
+++ b/sas/package-lock.json
@@ -6,8 +6,8 @@
    "": {
      "name": "dc-sas",
      "dependencies": {
-        "@sasjs/cli": "^4.12.15",
-        "@sasjs/core": "^4.59.9"
+        "@sasjs/cli": "^4.13.1",
+        "@sasjs/core": "^4.60.0"
      }
    },
    "node_modules/@coolaj86/urequest": {
@@ -31,29 +31,62 @@
      }
    },
    "node_modules/@sasjs/adapter": {
-      "version": "4.16.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.16.0.tgz",
-      "integrity": "sha512-DzF/+s++FtSfuBmONicBbgeKI8feiwDOm1iKWlcDlmHCPmHIoj1IbI0v2fGktzurnE37/vkyp6dvHO+FhwI87Q==",
-      "hasInstallScript": true,
+      "version": "4.16.3",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.16.3.tgz",
+      "integrity": "sha512-xcoZT9qZhF6pXvXx4bHxbmauLdEHng8pSlTK4F6asUkHNR5uzeSvY6znA1yJqK+8FFtsVILyvMQyGyhWw6WsOA==",
      "license": "ISC",
      "dependencies": {
-        "@sasjs/utils": "3.5.2",
-        "axios": "1.12.2",
+        "@sasjs/utils": "3.5.6",
+        "axios": "^1.13.5",
        "axios-cookiejar-support": "5.0.5",
        "form-data": "4.0.4",
        "https": "1.0.0",
        "tough-cookie": "4.1.3"
      }
    },
-    "node_modules/@sasjs/cli": {
-      "version": "4.12.15",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.12.15.tgz",
-      "integrity": "sha512-vA2YY+U9niquU7qkcDSxhOKcoased+4gePQyg8eWDRxCoWJx4mPv/y86J/cT2DmgAzjEt5JJAT8IUUtZcFJX+A==",
-      "hasInstallScript": true,
+    "node_modules/@sasjs/adapter/node_modules/@sasjs/utils": {
+      "version": "3.5.6",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.5.6.tgz",
+      "integrity": "sha512-jx8zWSOysDD66vTjA0BWiZ8bcFqmqh8F+56fUCgLmJhm89eDbKrGF3mDKMQx3UE7d2+gxp9xYhJCdaBWz0Dlxw==",
      "license": "ISC",
      "dependencies": {
-        "@sasjs/adapter": "^4.16.0",
-        "@sasjs/core": "4.59.9",
+        "@fast-csv/format": "4.3.5",
+        "@types/fs-extra": "11.0.4",
+        "@types/prompts": "2.0.13",
+        "chalk": "4.1.1",
+        "cli-table": "0.3.6",
+        "consola": "2.15.0",
+        "find": "0.3.0",
+        "fs-extra": "11.3.0",
+        "jwt-decode": "3.1.2",
+        "prompts": "2.4.1",
+        "valid-url": "1.0.9"
+      }
+    },
+    "node_modules/@sasjs/adapter/node_modules/chalk": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@sasjs/cli": {
+      "version": "4.13.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.13.1.tgz",
+      "integrity": "sha512-KvaKB551d3RjgD4upAy5c8iu7lxnL45pkUkueRf5cM1bneyyNIVR6lZ3E17UKnzFbeZxA/x3EPH9ObervpyHQw==",
+      "license": "ISC",
+      "dependencies": {
+        "@sasjs/adapter": "4.16.3",
+        "@sasjs/core": "4.60.1",
        "@sasjs/lint": "2.4.3",
        "@sasjs/utils": "3.5.2",
        "adm-zip": "0.5.10",
@@ -78,9 +111,9 @@
      }
    },
    "node_modules/@sasjs/core": {
-      "version": "4.59.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.59.9.tgz",
-      "integrity": "sha512-dHsEbMCHRjrUAGrYXH/a93F8iFogMEQuIZ2qFBFNzM0LwjwDdcTK0kCRMarBRc3oNo4JsaHqOltYbYaZ06GmCw==",
+      "version": "4.60.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.60.1.tgz",
+      "integrity": "sha512-PjHg0w8cV3q1ZLe1CxHUiSC+H44frTnovkFoGHxKPjNQ7KWYn5Hu7LkQuf2JsGoj85XCjeJjr/daB2yDrbJakQ==",
      "license": "MIT"
    },
    "node_modules/@sasjs/lint": {
@@ -239,13 +272,13 @@
      "license": "MIT"
    },
    "node_modules/axios": {
-      "version": "1.12.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz",
-      "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==",
+      "version": "1.13.5",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz",
+      "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==",
      "license": "MIT",
      "dependencies": {
-        "follow-redirects": "^1.15.6",
-        "form-data": "^4.0.4",
+        "follow-redirects": "^1.15.11",
+        "form-data": "^4.0.5",
        "proxy-from-env": "^1.1.0"
      }
    },
@@ -268,6 +301,22 @@
        "tough-cookie": ">=4.0.0"
      }
    },
+    "node_modules/axios/node_modules/form-data": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
+      "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
    "node_modules/balanced-match": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -1761,6 +1810,7 @@
      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
      "license": "BSD-3-Clause",
+      "peer": true,
      "dependencies": {
        "psl": "^1.1.33",
        "punycode": "^2.1.1",
--- a/sas/package.json
+++ b/sas/package.json
@@ -28,7 +28,7 @@
  },
  "private": true,
  "dependencies": {
-    "@sasjs/cli": "^4.12.15",
-    "@sasjs/core": "^4.59.9"
+    "@sasjs/cli": "^4.13.1",
+    "@sasjs/core": "^4.60.0"
  }
 }
--- a/sas/sasjs/macros/bitemporal_closeouts.sas
+++ b/sas/sasjs/macros/bitemporal_closeouts.sas
@@ -19,6 +19,8 @@
    given that BUS_FROM should be supplied in the PK.
  @param [in] tech_from= (tx_from_dttm) Technical FROM datetime variable.
    Required on BASE table only.
+  @param [in] AUDITFOLDER= (0) Unquoted path to a directory into which a copy of
+    the generated delete program will be written

  <h4> Global Variables </h4>
  @li `dc_dttmtfmt`
@@ -28,6 +30,9 @@
  @li mp_abort.sas
  @li mf_existvar.sas
  @li mf_getattrn.sas
+  @li mf_getengine.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
  @li mf_getuser.sas
  @li mf_getvartype.sas
  @li mp_lockanytable.sas
@@ -53,8 +58,7 @@
                              /* Should INCLUDE BUS_FROM field if relevant. */
      ,NOW=DEFINE
      ,FILTER=                /* supply a filter to limit the update */
-      ,outdest=               /* supply an unquoted filepath/filename.ext to get
-                                a text file containing the update statements */
+      ,AUDITFOLDER=0
      ,loadtype=
      ,loadtarget=YES      /* if <> YES will return without changing anything */
 );
@@ -70,13 +74,16 @@
  * perform basic checks
  */
 /* do tables exist? */
-%if not %sysfunc(exist(&base_lib..&base_dsn)) %then %do;
-  %mp_abort(msg=&base_lib..&base_dsn does not exist)
-%end;
-%else %if %sysfunc(exist(&append_lib..&append_dsn))=0
-          and %sysfunc(exist(&append_lib..&append_dsn,VIEW))=0 %then %do;
-  %mp_abort(msg=&append_lib..&append_dsn does not exist)
-%end;
+%mp_abort(
+  iftrue=(%sysfunc(exist(&base_lib..&base_dsn)) ne 1),
+  msg=&base_lib..&base_dsn does not exist
+)
+%mp_abort(
+  iftrue=(%sysfunc(exist(&append_lib..&append_dsn))=0
+          and %sysfunc(exist(&append_lib..&append_dsn,VIEW))=0 ),
+  msg=&append_lib..&append_dsn does not exist
+)
+
 /* do TX columns exist? */
 %if &loadtype ne UPDATE %then %do;
  %if not %mf_existvar(&base_lib..&base_dsn,&tech_from) %then %do;
@@ -111,61 +118,97 @@ data _null_;
  gap=intck('HOURS',now,datetime());
  call symputx('gap',gap,'l');
 run;
-%mf_abort(
+%mp_abort(
  iftrue=(&gap > 24),
  msg=NOW variable (&now) is not within a 24hr tolerance
 )

 /* have any warnings / errs occurred thus far?  If so, abort */
-%mf_abort(
+%mp_abort(
  iftrue=(&syscc>0),
  msg=Aborted due to SYSCC=&SYSCC status
 )

+/* set up folder */
+%local tmplib;%let tmplib=%mf_getuniquelibref();
+%if "&AUDITFOLDER"="0" %then %do;
+  filename tmp temp lrecl=10000;
+  libname &tmplib (work);
+%end;
+%else %do;
+  filename tmp "&AUDITFOLDER/deleterecords.sas" lrecl=10000;
+  libname &tmplib "&AUDITFOLDER";
+%end;
+
 /**
-  * Create closeout statements.  These are sent as individual SQL statements
+  * Create closeout statements.  If UPDATE approach and CAS engine, use the
+  * DeleteRows action (as regular SQL deletes are not supported).
+  * Otherwise, the deletions are sent as individual SQL statements
  * to ensure pass-through utilisation.  The update_cnt variable monitors
  * how many records were actually updated on the target table.
  */
-%local update_cnt;
+%local update_cnt etype;
 %let update_cnt=0;
-filename tmp temp;
-data _null_;
-  set ___closeout1;
-  file tmp;
-  if _n_=1 then put 'proc sql noprint;' ;
-  length string $32767.;
-  %if &loadtype=UPDATE %then %do;
-    put "delete from &base_lib..&base_dsn where 1";
-  %end;
-  %else %do;
-    now=symget('now');
-    put "update &base_lib..&base_dsn set &tech_to= " now @;
-    %if %mf_existvar(&base_lib..&base_dsn,PROCESSED_DTTM) %then %do;
-      put " ,PROCESSED_DTTM=" now @;
-    %end;
-    put " where " now " lt &tech_to ";
-  %end;
-  %do x=1 %to %sysfunc(countw(&PK));
-    %let var=%scan(&pk,&x,%str( ));
-    %if %mf_getvartype(&base_lib..&base_dsn,&var)=C %then %do;
-      /* use single quotes to avoid ampersand resolution in data */
-      string=" & &var='"!!trim(prxchange("s/'/''/",-1,&var))!!"'";
+%let etype=%mf_getengine(&base_lib);
+%put &=etype;
+
+%if &loadtype=UPDATE and &etype=CAS %then %do;
+  /* create temp table for deletions */
+  %local delds;%let delds=%mf_getuniquename(prefix=DC);
+  data casuser.&delds &tmplib..deleterecords;
+    set work.___closeout1;
+    keep &pk;
+  run;
+  /* build the proc */
+  data _null_;
+    file tmp;
+    put "/* libname approve '&AUDITFOLDER'; */";
+    put 'proc cas;table.deleteRows result=r/ table={' ;
+    put "  caslib='&base_lib',name='&base_dsn',where='1=1',";
+    put "  whereTable={caslib='CASUSER',name='&delds'}";
+    put "};";
+    put "call symputx('update_cnt',r.RowsDeleted);";
+    put "quit;";
+    put "data;set casuser.&delds;putlog (_all_)(=);run;";
+    put '%put &=update_cnt;';
+    put "proc sql;drop table CASUSER.&delds;";
+    stop;
+  run;
+
+%end;
+%else %do;
+  data _null_;
+    set ___closeout1;
+    file tmp;
+    if _n_=1 then put 'proc sql noprint;' ;
+    length string $32767.;
+    %if &loadtype=UPDATE %then %do;
+      put "delete from &base_lib..&base_dsn where 1";
    %end;
    %else %do;
-      string=cats(" & &var=",&var);
+      now=symget('now');
+      put "update &base_lib..&base_dsn set &tech_to= " now @;
+      %if %mf_existvar(&base_lib..&base_dsn,PROCESSED_DTTM) %then %do;
+        put " ,PROCESSED_DTTM=" now @;
+      %end;
+      put " where " now " lt &tech_to ";
    %end;
-    put string;
-  %end;
-  put "&filter ;";
-  put '%let update_cnt=%eval(&update_cnt+&sqlobs);%put update_cnt=&update_cnt;';
-run;
-
-data _null_;
-  infile tmp;
-  input;
-  putlog _infile_;
-run;
+    %do x=1 %to %sysfunc(countw(&PK));
+      %let var=%scan(&pk,&x,%str( ));
+      %if %mf_getvartype(&base_lib..&base_dsn,&var)=C %then %do;
+        /* use single quotes to avoid ampersand resolution in data */
+        string=" & &var='"!!trim(prxchange("s/'/''/",-1,&var))!!"'";
+      %end;
+      %else %do;
+        string=cats(" & &var=",&var);
+      %end;
+      put string;
+    %end;
+    put "&filter ;";
+    put '%let update_cnt=%eval(&update_cnt+&sqlobs);';
+    put '%put update_cnt=&update_cnt;';
+  run;
+%end;

 %if &loadtarget ne YES %then %return;

--- a/sas/sasjs/macros/bitemporal_dataloader.sas
+++ b/sas/sasjs/macros/bitemporal_dataloader.sas
@@ -512,6 +512,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
      ,NOW=&dbnow
      ,loadtarget=&loadtarget
      ,loadtype=&loadtype
+      ,AUDITFOLDER=&dc_staging_area/&ETLSOURCE
    )
  %end;
 %end;
@@ -574,6 +575,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
      ,NOW=&dbnow
      ,loadtarget=&loadtarget
      ,loadtype=&loadtype
+      ,AUDITFOLDER=&dc_staging_area/&ETLSOURCE
    )
  %end;
 %end;
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -1866,6 +1866,15 @@ insert into &lib..MPE_VALIDATIONS set
    ,rule_value='0'
    ,rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_ROW_LEVEL_SECURITY"
+    ,base_col="RLS_SUBGROUP_ID"
+    ,rule_type='NOTNULL'
+    ,rule_value='0'
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
    tx_from=0
    ,base_lib="&lib"
--- a/sas/sasjs/sasjsconfig.json
+++ b/sas/sasjs/sasjsconfig.json
@@ -124,7 +124,8 @@
        "serviceFolders": [
          "sasjs/targets/viya/services_viya/viya_users",
          "sasjs/targets/viya/services_viya/admin",
-          "sasjs/targets/viya/services_viya/public"
+          "sasjs/targets/viya/services_viya/public",
+          "sasjs/targets/viya/services_viya/validations"
        ],
        "initProgram": "sasjs/utils/serviceinitviya.sas",
        "termProgram": "",
@@ -202,7 +203,7 @@
    },
    {
      "name": "server",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
      "serverType": "SASJS",
      "httpsAgentOptions": {
        "rejectUnauthorized": false,
--- a/sas/sasjs/services/admin/demodata.sas
+++ b/sas/sasjs/services/admin/demodata.sas
@@ -0,0 +1,363 @@
+/**
+  @file
+  @brief Creates demo tables and associated config
+  @details Can be removed in prod installs.
+
+  To activate this job, add the following to SETTINGS:
+
+      %let demolib=PUBLIC;
+      libname &demolib "%sysfunc(pathname(&dc_libref))/&demolib";
+      %let joblib=HOOKLIB;
+      libname &joblib "%sysfunc(pathname(&dc_libref))/&joblib";
+      %let dcdemoflag=1;
+
+  Note that this will:
+    * REPLACE any tables named CARS_EXT or COUNTRIES in the PUBLIC library
+    * REPLACE all DC config for libraries named PUBLIC
+    * CREATE a folder called "demo" in the DC Apploc
+    * CREATE two BASE libraries (HOOKLIB & PUBLIC) in the DC (physical) folder
+
+  <h4> SAS Macros </h4>
+  @li mpeinit.sas
+  @li mf_getengine.sas
+  @li mf_getuser.sas
+  @li mf_increment.sas
+  @li mf_nobs.sas
+  @li mf_uid.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mp_replace.sas
+  @li mx_createjob.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let dcdemoflag=0;
+options dlcreatedir;
+%global joblib demolib;
+%mpeinit()
+
+%mp_abort(iftrue= (&dcdemoflag ne 1)
+  ,mac=&_program
+  ,msg=%str(Job not configured.  See comments in the code.)
+)
+
+data work.cars_ext(index=(carspk=(make model PRODUCTIONDATE) /unique));
+  attrib
+    MAKE                             length= $13
+    MODEL                            length= $40
+    TYPE                             length= $8
+    ORIGIN                           length= $6
+    COUNTRY                          length= $30
+    POTENTIALBUY                     length= $6
+    COMMENT                          length= $30
+    NOTES                            length= $30
+    CHECKBOXVAR                      length= $3
+    PRODUCTIONDATE                   length= 8      format=DATE9.
+  ;
+  set sashelp.cars;
+  retain comment 'n/a';
+  if mod(ceil(ranuni(1)*100),3)=0 then notes=catx(' ',make,type);
+  call missing(notes);
+  /* random / reproducible date between 1960 and 2020 */
+  PRODUCTIONDATE=ceil(ranuni(1)*365*60);
+  if mod(ceil(ranuni(1)*1000),2)=0 then CHECKBOXVAR='YES';
+  else CHECKBOXVAR='No';
+  if mod(ceil(ranuni(1)*1000),3)=0 then POTENTIALBUY='Maybe';
+  else if mod(ceil(ranuni(1)*1000),2)=0 then POTENTIALBUY='Yes';
+  else POTENTIALBUY='No';
+  make=cats(make);
+  model=cats(model);
+
+  array cntrs (4) $ 60 _temporary_ ( "Germany" "France" "Poland" "Italy");
+  if origin='USA' then country='USA';
+  else if origin='Asia' then do;
+    if mod(_n_,2)=0 then country='Japan';
+    else country='Korea';
+  end;
+  else COUNTRY = cntrs[ ceil(dim(cntrs) * ranuni(1))];
+
+  *put (_all_)(=);
+run;
+
+data work.COUNTRIES(index=(countriespk=(origin country) /unique));
+  attrib
+    ORIGIN                           length= $6
+    COUNTRY                          length= $30
+  ;
+  infile cards dsd;
+  input
+    ORIGIN                           :$char.
+    COUNTRY                          :$char.
+  ;
+datalines4;
+Europe,Germany
+Europe,France
+Europe,Poland
+Europe,Italy
+USA,USA
+Asia,Japan
+Asia,Korea
+;;;;
+run;
+
+data work.jobdata;
+  length message job $100;
+  call missing(of _all_);
+  stop;
+run;
+
+%let engine_type=%mf_getengine(&demolib);
+%put &=engine_type;
+%if &engine_type=CAS %then %do;
+  proc cas;
+    table.tableExists result=r / name="CARS_EXT" caslib="PUBLIC";
+    if r.exists then
+      table.dropTable / name="CARS_EXT" caslib="PUBLIC" quiet=TRUE;
+
+    table.tableExists result=r2 / name="COUNTRIES" caslib="PUBLIC";
+    if r2.exists then
+      table.dropTable / name="COUNTRIES" caslib="PUBLIC" quiet=TRUE;
+
+    table.tableExists result=r2 / name="MPE_AUDIT" caslib="PUBLIC";
+    if r2.exists then
+      table.dropTable / name="MPE_AUDIT" caslib="PUBLIC" quiet=TRUE;
+  quit;
+  proc casutil;
+    load data=work.CARS_EXT outcaslib="PUBLIC" casout="CARS_EXT" promote;
+    load data=work.COUNTRIES outcaslib="PUBLIC" casout="COUNTRIES" promote;
+    load data=&dc_libref..MPE_AUDIT
+      outcaslib="PUBLIC" casout="MPE_AUDIT" promote;
+  run;
+  data &joblib..JOBDATA; set work.JOBDATA;run;
+%end;
+%else %do;
+  options replace;
+  data &demolib..CARS_EXT; set work.cars_ext;
+  data &demolib..COUNTRIES; set work.countries;
+  data &joblib..JOBDATA; set work.JOBDATA;run;
+%end;
+
+%let apploc=%mf_getapploc(&_program);
+%let demolib=%upcase(&demolib);
+proc sql;
+delete from &dc_libref..mpe_tables
+  where libref="&demolib" and dsn in ('CARS_EXT','COUNTRIES');
+data append;
+  if 0 then set &dc_libref..mpe_tables;
+  TX_FROM=0;
+  TX_TO='31DEC9999:23:59:59'dt;
+  LIBREF="&demolib";
+  LOADTYPE='UPDATE';
+  NUM_OF_APPROVALS_REQUIRED=1;
+  PRE_EDIT_HOOK="&apploc/demo/PREEDIT";
+  POST_EDIT_HOOK="&apploc/demo/POSTEDIT";
+  PRE_APPROVE_HOOK="&apploc/demo/PREAPPROVE";
+  POST_APPROVE_HOOK="&apploc/demo/POSTAPPROVE";
+  DSN='CARS_EXT'; BUSKEY='MAKE MODEL PRODUCTIONDATE'; output;
+  DSN='COUNTRIES'; BUSKEY='ORIGIN COUNTRY'; output;
+run;
+proc append base=&dc_libref..MPE_TABLES data=&syslast;
+run;
+
+/* hard coded values for CHECKBOXVAR */
+%let rk=1e6;
+proc sql noprint;
+delete from &dc_libref..mpe_selectbox
+  where select_lib="&demolib"
+    and select_ds in ('CARS_EXT');
+select max(selectbox_rk) into: rk
+  from &dc_libref..mpe_selectbox;
+
+insert into &dc_libref..mpe_selectbox set
+  selectbox_rk=%mf_increment(rk)
+  ,ver_from_dttm=0
+  ,select_lib="&demolib"
+  ,select_ds="CARS_EXT"
+  ,base_column="CHECKBOXVAR"
+  ,selectbox_value='Yes'
+  ,selectbox_order=1
+  ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dc_libref..mpe_selectbox set
+  selectbox_rk=%mf_increment(rk)
+  ,ver_from_dttm=0
+  ,select_lib="&demolib"
+  ,select_ds="CARS_EXT"
+  ,base_column="CHECKBOXVAR"
+  ,selectbox_value='No'
+  ,selectbox_order=2
+  ,ver_to_dttm='31DEC5999:23:59:59'dt;
+
+/* Table driven values */
+delete from &dc_libref..MPE_VALIDATIONS
+  where base_lib="&demolib" and base_ds="CARS_EXT";
+
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="MAKE"
+    ,rule_type='HARDSELECT'
+    ,rule_value="SASHELP.CARS.MAKE"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="MODEL"
+    ,rule_type='HARDSELECT'
+    ,rule_value="SASHELP.CARS.MODEL"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="TYPE"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="SASHELP.CARS.TYPE"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="POTENTIALBUY"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="&demolib..CARS_EXT.POTENTIALBUY"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="COMMENT"
+    ,rule_type='NOTNULL'
+    ,rule_value="n/a"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="ENGINESIZE"
+    ,rule_type='MINVAL'
+    ,rule_value="1.3"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="ENGINESIZE"
+    ,rule_type='MAXVAL'
+    ,rule_value="8.3"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program
+  ,msg=%str(syscc=syscc=&syscc during param configuration)
+)
+
+/* programmatic values for COUNTRY (Dynamic Dropdown) */
+filename vldtr temp;
+data _null_;
+  file vldtr ;
+  put 'proc sql;';
+  put 'create table work.vals as';
+  put '  select distinct ORIGIN as display_value,';
+  put '  ORIGIN as raw_value';
+  put "  from &demolib..COUNTRIES";
+  put '  order by 1;';
+  put 'data work.DYNAMIC_VALUES; set work.vals;display_index=_n_;run;';
+  put ' ';
+  put 'proc sql;';
+  put 'create table work.dev as ';
+  put '  select a.display_index,b.country as display_value';
+  put '  from work.DYNAMIC_VALUES as a';
+  put "  left join &demolib..countries as b";
+  put "  on a.raw_value=b.origin";
+  put '  order by display_index;';
+  put 'data work.DYNAMIC_EXTENDED_VALUES; set work.dev;by display_index;';
+  put '  EXTRA_COL_NAME="COUNTRY";';
+  put '  DISPLAY_TYPE="C";';
+  put '  RAW_VALUE_CHAR=DISPLAY_VALUE;';
+  put '  RAW_VALUE_NUM=.;';
+  put '  if first.display_index then forced_value=1;';
+  put 'run;';
+run;
+%mx_createjob(path=&apploc/demo
+  ,name=origin,code=vldtr
+)
+proc sql;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="ORIGIN"
+    ,rule_type='HARDSELECT_HOOK'
+    ,rule_value="&apploc/demo/origin"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
+/* PRE_EDIT JOB */
+%let fvar=XXXXXXXXXXX; /* cannot substitute macvars in parmcards */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds (%mf_nobs(work.out) obs) fetched for editing %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f1 temp;
+%mp_binarycopy(inref=ft15f001, outref=f1)
+%mp_replace("%sysfunc(pathname(f1))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=PREEDIT,code=f1)
+filename ft15f001 clear;
+
+/* POST EDIT JOB */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds staged %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f2 temp;
+%mp_binarycopy(inref=ft15f001, outref=f2)
+%mp_replace("%sysfunc(pathname(f2))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=POSTEDIT,code=f2)
+filename ft15f001 clear;
+
+/* PRE APPROVE JOB */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds (%mf_nobs(work.staging_ds) obs) under review by %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f3 temp;
+%mp_binarycopy(inref=ft15f001, outref=f3)
+%mp_replace("%sysfunc(pathname(f3))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=PREAPPROVE,code=f3)
+filename ft15f001 clear;
+
+/* POST APPROVE JOB */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds (%mf_nobs(work.staging_ds) obs) approved by %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f4 temp;
+%mp_binarycopy(inref=ft15f001, outref=f4)
+%mp_replace("%sysfunc(pathname(f4))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=POSTAPPROVE,code=f4)
+filename ft15f001 clear;
--- a/sas/sasjs/services/editors/loadfile.sas
+++ b/sas/sasjs/services/editors/loadfile.sas
@@ -286,7 +286,17 @@ options mprint;
 )

 filename outref "&dir/BKP_&xlsname";
+
+data _null_;
+  if "&xlsref" ne "0" then do;
+    rc=fcopy("&xlsref","outref");
+  end;
+run;
+
+/**
+  * if running 9.3 or older, delete step above and enable macro below
 %mp_binarycopy(iftrue=("&xlsref" ne "0"),inref=&xlsref,outref=outref)
+  */

 %mp_abort(iftrue= (&syscc ne 0)
  ,mac=&sysmacroname
--- a/sas/sasjs/services/editors/restore.sas
+++ b/sas/sasjs/services/editors/restore.sas
@@ -5,9 +5,12 @@

  <h4> SAS Macros </h4>
  @li dc_assignlib.sas
+  @li mcf_getfmttype.sas
  @li mf_nobs.sas
  @li mp_abort.sas
+  @li mp_applyformats.sas
  @li mp_ds2csv.sas
+  @li mp_getcols.sas
  @li mp_stripdiffs.sas
  @li mpeinit.sas
  @li mpe_checkrestore.sas
@@ -112,8 +115,26 @@ data approve.jsdset;
  length _____DELETE__THIS__RECORD_____ $3;
  if 0 then call missing(_____DELETE__THIS__RECORD_____);
  set work.mp_stripdiffs;
+  format _all_;
 run;

+/* find all of the date / datetime / time vars */
+%mcf_getfmttype(wrap=YES)
+%mp_getcols(&tgtds,outds=work.cols)
+
+data work.applydtfmts;
+  set work.cols;
+  lib="APPROVE";
+  ds="JSDSET";
+  var=name;
+  fmt=coalescec(format,'0');
+  fmttype=mcf_getfmttype(fmt);
+  if fmttype in ('DATE','DATETIME','TIME');
+  keep lib ds var fmt;
+run;
+%mp_applyformats(work.applydtfmts)
+
+
 /* export to csv */
 %mp_ds2csv(approve.jsdset
  ,dlm=COMMA
--- a/sas/sasjs/services/validations/tables_all.sas
+++ b/sas/sasjs/services/validations/tables_all.sas
@@ -54,7 +54,7 @@
 %let vartgtlib=%mf_getuniquename();
 %let var_is_lib=%mf_getuniquename();
 data _null_;
-  length &varlibds $41 &vartgtlib $8 libref $8 rls_libref $8;
+  length &varlibds $41 &vartgtlib $8 libref $8 rls_libref cls_libref $8;
  if _n_=1 then call missing(of _all_);
  set work.source_row;
  &varlibds=upcase(symget('libds'));
--- a/sas/sasjs/targets/viya/macros_viya/dc_createdataset.sas
+++ b/sas/sasjs/targets/viya/macros_viya/dc_createdataset.sas
@@ -1,17 +1,14 @@
 /**
  @file dc_createdataset.sas

-  <h4> SAS Macros </h4>
-
-  @version 9.3
  @author 4GL Apps Ltd
  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
    and may not be re-distributed or re-sold without the express permission of
    4GL Apps Ltd.
 **/

-%macro dc_createdataset(libds=mm_getlibs);
-data viewdata;
+%macro dc_createdataset(libds=mm_getlibs,outds=viewdata);
+data &outds;
  var1='Table';
  var2="&libds";
  var3="does not exist!";
--- a/sas/sasjs/targets/viya/services_viya/validations/sas_groups.sas
+++ b/sas/sasjs/targets/viya/services_viya/validations/sas_groups.sas
@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief validating the mpe_security.sas_group column
+  @details The input table is simply one row from the target table in table
+  called "work.source_row".
+
+  Available macro variables:
+    @li LIBDS - The library.dataset being filtered
+    @li VARIABLE_NM - The column being filtered
+
+
+  <h4> Service Outputs </h4>
+  The values provided below are generic samples - we encourage you to replace
+  these with realistic values in your own deployments.
+
+  <h5>DYNAMIC_VALUES</h5>
+  The RAW_VALUE column may be charactor or numeric.  If DISPLAY_INDEX is not
+  provided, it is added automatically.
+
+  |DISPLAY_INDEX:best.|DISPLAY_VALUE:$|RAW_VALUE|
+  |---|---|---|
+  |1|$77.43|77.43|
+  |2|$88.43|88.43|
+
+  <h5>DYNAMIC_EXTENDED_VALUES</h5>
+  This table is optional.  If provided, it will map the DISPLAY_INDEX from the
+  DYNAMIC_VALUES table to additional column/value pairs, that will be used to
+  populate dropdowns for _other_ cells in the _same_ row.
+
+  Should be used sparingly!  The use of large tables here can slow down the
+  browser.
+
+  |DISPLAY_INDEX:best.|EXTRA_COL_NAME:$32.|DISPLAY_VALUE:$|DISPLAY_TYPE:$1.|RAW_VALUE_NUM|RAW_VALUE_CHAR:$5000|
+  |---|---|---|
+  |1|DISCOUNT_RT|"50%"|N|0.5||
+  |1|DISCOUNT_RT|"40%"|N|0.4||
+  |1|DISCOUNT_RT|"30%"|N|0.3||
+  |1|CURRENCY_SYMBOL|"GBP"|C||"GBP"|
+  |1|CURRENCY_SYMBOL|"RSD"|C||"RSD"|
+  |2|DISCOUNT_RT|"50%"|N|0.5||
+  |2|DISCOUNT_RT|"40%"|N|0.4||
+  |2|CURRENCY_SYMBOL|"EUR"|C||"EUR"|
+  |2|CURRENCY_SYMBOL|"HKD"|C||"HKD"|
+
+
+
+  <h4> SAS Macros </h4>
+  @li dc_getgroups.sas
+
+
+**/
+
+
+%dc_getgroups(outds=groups)
+
+proc sql;
+create table work.DYNAMIC_VALUES as
+  select distinct groupname as display_value,
+    groupuri as raw_value
+  from work.groups
+  order by 1;