chore(release): 7.6.0 [skip ci]

# [7.6.0](https://git.datacontroller.io/dc/dc/compare/v7.5.0...v7.6.0) (2026-04-03)

### Bug Fixes

* add label and tooltip for libref download, sanitise input ([52d5803](52d58036a4))

### Features

* configurable email alerts.  Closes [#217](#217) ([2ccf0d1](2ccf0d1100))

.gitea/workflows/build.yaml

@@ -2,32 +2,31 @@ name: Build
 run-name: Running Lint Check and Licence checker on Pull Request
 on: [pull_request]
 
+env:
+  NODE_VERSION: '24.5.0'
+
 jobs:
   Build-and-ng-test:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 24.5.0
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Install Google Chrome
         run: |
-          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
-          echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
           apt-get update
-          apt-get install -y google-chrome-stable xvfb
+          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+          apt install -y ./google-chrome*.deb
 
       - name: Write .npmrc file
-        run: echo "$NPMRC" > client/.npmrc
+        run: echo "$NPMRC" >> client/.npmrc
         shell: bash
         env:
           NPMRC: ${{ secrets.NPMRC}}
 
-      - name: Lint check
-        run: npm run lint:check
-
       - name: Install dependencies
         run: |
           cd client
@@ -35,6 +34,18 @@ jobs:
           echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
           npm ci
 
+      - name: Check audit
+        # Audit should fail and stop the CI if critical vulnerability found
+        run: |
+          npm audit --audit-level=critical --omit=dev
+          cd ./sas
+          npm audit --audit-level=critical --omit=dev
+          cd ../client
+          npm audit --audit-level=critical --omit=dev
+
+      - name: Lint check
+        run: npm run lint:check
+
       - name: Licence checker
         run: |
           cd client
@@ -52,26 +63,27 @@ jobs:
 
   Build-and-test-development:
     runs-on: ubuntu-latest
-    needs: Build-production-and-ng-test
+    needs: Build-and-ng-test
+    env:
+      CHROME_BIN: /usr/bin/google-chrome
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 24.5.0
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Write .npmrc file
         run: |
           touch client/.npmrc
           echo '${{ secrets.NPMRC}}' > client/.npmrc
 
-      - run: apt-get update
-      - run: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-      - run: apt install -y ./google-chrome*.deb;
-      - run: export CHROME_BIN=/usr/bin/google-chrome
-      - run: apt-get update -y
-      - run: apt-get -y install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb
-      - run: apt -y install jq
+      - name: Install system dependencies
+        run: |
+          apt-get update
+          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+          apt install -y ./google-chrome*.deb
+          apt-get -y install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb jq zip
 
       - name: Write cypress credentials
         run: echo "$CYPRESS_CREDS" > ./client/cypress.env.json
@@ -86,17 +98,18 @@ jobs:
           echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
           npm ci
 
-      # Install pm2 and prepare SASJS server
-      - run: npm i -g pm2
-      - run: curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
-      - run: unzip linux.zip
-      - run: touch .env
-      - run: echo RUN_TIMES=js >> .env
-      - run: echo NODE_PATH=node >> .env
-      - run: echo CORS=enable >> .env
-      - run: echo WHITELIST=http://localhost:4200 >> .env
-      - run: cat .env
-      - run: pm2 start api-linux --wait-ready
+      - name: Setup and start SASjs server
+        run: |
+          npm i -g pm2
+          curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
+          unzip linux.zip
+          touch .env
+          echo RUN_TIMES=js >> .env
+          echo NODE_PATH=node >> .env
+          echo CORS=enable >> .env
+          echo WHITELIST=http://localhost:4200 >> .env
+          cat .env
+          pm2 start api-linux --wait-ready
 
       - name: Deploy mocked services
         run: |
@@ -106,11 +119,6 @@ jobs:
           sasjs cbd -t server-ci
           # sasjs request services/admin/makedata -t server-ci -d ./deploy/makeData4GL.json -c ./deploy/requestConfig.json -o ./output.json
 
-      - name: Install ZIP
-        run: |
-          apt-get update
-          apt-get install zip
-
       - name: Prepare and run frontend and cypress
         run: |
           cd ./client
@@ -126,7 +134,7 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

.gitea/workflows/lighthouse.yaml

@@ -2,38 +2,31 @@ name: Lighthouse Checks
 run-name: Running Lighthouse Performance and Accessibility Checks on Pull Request
 on: [pull_request]
 
+env:
+  NODE_VERSION: '24.5.0'
+
 jobs:
   lighthouse:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [24.5.0]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Use Node.js ${{ matrix.node-version }}
+      - name: Use Node.js ${{ env.NODE_VERSION }}
         uses: actions/setup-node@v4
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Install Google Chrome
         run: |
-          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
-          echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
           apt-get update
-          apt-get install -y google-chrome-stable xvfb
+          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+          apt install -y ./google-chrome*.deb
 
-      - name: Install pm2 for process management
-        run: npm i -g pm2
+      - name: Install global packages
+        run: npm i -g pm2 @sasjs/cli wait-on
 
-      - name: Install @sasjs/cli
-        run: npm i -g @sasjs/cli
-
-      - name: Install wait-on globally
-        run: npm install -g wait-on
-
-      - name: Create .env file for sasjs/server
+      - name: Setup and start SASjs server
         run: |
           touch .env
           echo RUN_TIMES=js >> .env
@@ -41,15 +34,9 @@ jobs:
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
           cat .env
-
-      - name: Download sasjs/server package from github using curl
-        run: curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
-
-      - name: Unzip downloaded package
-        run: unzip linux.zip
-
-      - name: Run sasjs server
-        run: pm2 start api-linux --wait-ready
+          curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
+          unzip linux.zip
+          pm2 start api-linux --wait-ready
 
       - name: Write .npmrc file
         run: echo "$NPMRC" > client/.npmrc

.gitea/workflows/release.yaml

@@ -5,15 +5,20 @@ on:
     branches:
       - main
 
+env:
+  NODE_VERSION: '24.5.0'
+
 jobs:
   Build-production-and-ng-test:
     runs-on: ubuntu-latest
+    env:
+      CHROME_BIN: /usr/bin/google-chrome
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 24.5.0
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Write .npmrc file
         run: |
@@ -24,8 +29,7 @@ jobs:
         run: |
           apt-get update
           wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-          apt install -y ./google-chrome*.deb;
-          export CHROME_BIN=/usr/bin/google-chrome
+          apt install -y ./google-chrome*.deb
 
       - name: Write cypress credentials
         run: echo "$CYPRESS_CREDS" > ./client/cypress.env.json
@@ -43,9 +47,9 @@ jobs:
       - name: Check audit
         # Audit should fail and stop the CI if critical vulnerability found
         run: |
-          npm audit --audit-level=critical --omit=dev
+          npm audit --omit=dev
           cd ./sas
-          npm audit --audit-level=critical --omit=dev
+          npm audit --omit=dev
           cd ../client
           npm audit --audit-level=critical --omit=dev
 
@@ -63,25 +67,26 @@ jobs:
   Build-and-test-development:
     runs-on: ubuntu-latest
     needs: Build-production-and-ng-test
+    env:
+      CHROME_BIN: /usr/bin/google-chrome
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 24.5.0
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Write .npmrc file
         run: |
           touch client/.npmrc
           echo '${{ secrets.NPMRC}}' > client/.npmrc
 
-      - run: apt-get update
-      - run: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-      - run: apt install -y ./google-chrome*.deb;
-      - run: export CHROME_BIN=/usr/bin/google-chrome
-      - run: apt-get update -y
-      - run: apt-get -y install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb
-      - run: apt -y install jq
+      - name: Install system dependencies
+        run: |
+          apt-get update
+          wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+          apt install -y ./google-chrome*.deb
+          apt-get -y install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb jq zip
 
       - name: Write cypress credentials
         run: echo "$CYPRESS_CREDS" > ./client/cypress.env.json
@@ -96,17 +101,18 @@ jobs:
           echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
           npm ci
 
-      # Install pm2 and prepare SASJS server
-      - run: npm i -g pm2
-      - run: curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
-      - run: unzip linux.zip
-      - run: touch .env
-      - run: echo RUN_TIMES=js >> .env
-      - run: echo NODE_PATH=node >> .env
-      - run: echo CORS=enable >> .env
-      - run: echo WHITELIST=http://localhost:4200 >> .env
-      - run: cat .env
-      - run: pm2 start api-linux --wait-ready
+      - name: Setup and start SASjs server
+        run: |
+          npm i -g pm2
+          curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
+          unzip linux.zip
+          touch .env
+          echo RUN_TIMES=js >> .env
+          echo NODE_PATH=node >> .env
+          echo CORS=enable >> .env
+          echo WHITELIST=http://localhost:4200 >> .env
+          cat .env
+          pm2 start api-linux --wait-ready
 
       - name: Deploy mocked services
         run: |
@@ -116,11 +122,6 @@ jobs:
           sasjs cbd -t server-ci
           # sasjs request services/admin/makedata -t server-ci -d ./deploy/makeData4GL.json -c ./deploy/requestConfig.json -o ./output.json
 
-      - name: Install ZIP
-        run: |
-          apt-get update
-          apt-get install zip
-
       - name: Prepare and run frontend and cypress
         run: |
           cd ./client
@@ -136,7 +137,7 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts"
 
       - name: Zip Cypress videos
         if: always()
@@ -155,10 +156,10 @@ jobs:
     needs: [Build-production-and-ng-test, Build-and-test-development]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: 24.5.0
+          node-version: ${{ env.NODE_VERSION }}
 
       - name: Write .npmrc file
         run: |
@@ -168,17 +169,11 @@ jobs:
         env:
           NPMRC: ${{ secrets.NPMRC}}
 
-      - name: Install packages
+      - name: Install system packages
         run: |
           apt-get update
-          apt-get install zip -y
-          # sasjs cli is used to compile & build the SAS services
+          apt-get install -y zip jq doxygen
           npm i -g @sasjs/cli
-          # jq is used to parse the release JSON
-          apt-get install jq -y
-          # doxygen is used for the SASJS docs
-          apt-get update
-          apt-get install doxygen -y
 
       - name: Frontend Preliminary Build
         description: We want to prevent creating empty release if frontend fails
@@ -228,6 +223,8 @@ jobs:
           cp sasjs/utils/favicon.ico ../client/dist/favicon.ico
           sasjs c -t server
           rm -rf sasjsbuild/tests
+          server_apploc="/Public/app/dc"
+          sed -i "s|apploc=\"[^\"]*\"|apploc=\"${server_apploc}\"|g" sasjsbuild/services/web/index.html
           sasjs b -t server
           cp sasjsbuild/server.json.zip ./sasjs_server.json.zip
 

.npmrc

@@ -1 +1,3 @@
 legacy-peer-deps=true
+ignore-scripts=true
+save-exact=true

CHANGELOG.md

@@ -1,3 +1,89 @@
+# [7.6.0](https://git.datacontroller.io/dc/dc/compare/v7.5.0...v7.6.0) (2026-04-03)
+
+
+### Bug Fixes
+
+* add label and tooltip for libref download, sanitise input ([52d5803](https://git.datacontroller.io/dc/dc/commit/52d58036a40e25847e900f9b04a77dbcc409c12b))
+
+
+### Features
+
+* configurable email alerts.  Closes [#217](https://git.datacontroller.io/dc/dc/issues/217) ([2ccf0d1](https://git.datacontroller.io/dc/dc/commit/2ccf0d11000129629a0665421135b7530af9892f))
+
+# [7.5.0](https://git.datacontroller.io/dc/dc/compare/v7.4.1...v7.5.0) (2026-04-03)
+
+
+### Bug Fixes
+
+* add workflow audits, update deps ([66e98a9](https://git.datacontroller.io/dc/dc/commit/66e98a96cbd092e762b94a04660f8e17ca003ceb))
+* allow CSV uploads with licence row limit ([5b260e4](https://git.datacontroller.io/dc/dc/commit/5b260e49153dd85bc0023ad94d8a5f57b8ffa6dc)), closes [#213](https://git.datacontroller.io/dc/dc/issues/213)
+* bumping cli and pinning versions in .npmrc ([80039f4](https://git.datacontroller.io/dc/dc/commit/80039f4876c8e09dc477678e1eff58329094c9e9))
+* guard CSV upload with fileUpload licence flag ([ed40df6](https://git.datacontroller.io/dc/dc/commit/ed40df62953c3055770b5cbf50738f4a48b943cd))
+* parse embed param from window.location.hash for hash router compatibility ([0269c24](https://git.datacontroller.io/dc/dc/commit/0269c2421db245f7f5405678605cb4d4587e2a67))
+* quote CSV char values.  Closes [#215](https://git.datacontroller.io/dc/dc/issues/215) ([d9980e8](https://git.datacontroller.io/dc/dc/commit/d9980e866d1a2fe7a731ff279d73accd35003e67))
+* resolve outer promise in parseCsvFile for non-WLATIN1 path ([4ee15e1](https://git.datacontroller.io/dc/dc/commit/4ee15e1b6e83f27f279fc345e6998452a8f64d7e))
+* use XLSX for CSV row truncation to handle new lines in values ([6d590c0](https://git.datacontroller.io/dc/dc/commit/6d590c050dcd593a73464fae5604f774f016b10d))
+
+
+### Features
+
+* add embed URL parameter to hide header and back button ([b0dc441](https://git.datacontroller.io/dc/dc/commit/b0dc441d681369e06eee58288dbdbb236f930bdc)), closes [#214](https://git.datacontroller.io/dc/dc/issues/214)
+* add target libref input to config download ([a89657b](https://git.datacontroller.io/dc/dc/commit/a89657b0b81b9c531f64c0dda2714b4eb16c4bc9)), closes [#212](https://git.datacontroller.io/dc/dc/issues/212)
+* export config service to allow dclib swapping.  Closes [#212](https://git.datacontroller.io/dc/dc/issues/212) ([326c26f](https://git.datacontroller.io/dc/dc/commit/326c26fddfa88a0dc4ca79d3bd0c77c4d807f37c))
+
+## [7.4.1](https://git.datacontroller.io/dc/dc/compare/v7.4.0...v7.4.1) (2026-03-12)
+
+
+### Bug Fixes
+
+* support for SASIOSNF engine (SNOW alias) plus meta assignment ([7694d1b](https://git.datacontroller.io/dc/dc/commit/7694d1b0fb2bd0407c8598147fbae87a00d889a8))
+
+# [7.4.0](https://git.datacontroller.io/dc/dc/compare/v7.3.0...v7.4.0) (2026-02-20)
+
+
+### Bug Fixes
+
+* cli bump for mf_getscheme support ([a84ba41](https://git.datacontroller.io/dc/dc/commit/a84ba41ea9f0c97ae24f0a572b8cf5ec200f2132))
+* missing upcase on SNOW section, plus local sasjs target ([dc20064](https://git.datacontroller.io/dc/dc/commit/dc200646f7df2fd1910841f392c314532aae7581))
+
+
+### Features
+
+* SAS code changes for snowflake support ([e273e87](https://git.datacontroller.io/dc/dc/commit/e273e870efbf7875db869b760f2c7b1f39d571ae))
+
+# [7.3.0](https://git.datacontroller.io/dc/dc/compare/v7.2.8...v7.3.0) (2026-02-10)
+
+
+### Bug Fixes
+
+* bump xlsx, add crypto-shim ([8dc18b1](https://git.datacontroller.io/dc/dc/commit/8dc18b155abfc20fd0b043e0d70bbbc17e6b6811))
+* correctly applying deletes on viya, also ([46cdeb0](https://git.datacontroller.io/dc/dc/commit/46cdeb0babee6870553a41877cbe85204e7099c4))
+* crypto module requirement for sheetjs/crypto package ([505d0af](https://git.datacontroller.io/dc/dc/commit/505d0af2b3b3c1c79c65045dcaffc263e0f8e796))
+* disable parsing excel in web worker beacuse it breaks in the stream apps ([280bdee](https://git.datacontroller.io/dc/dc/commit/280bdeeb1b82f00689f46c68a3cde3f2d24bc18f))
+* Display all contexts when installing DC on Viya ([d41f88f](https://git.datacontroller.io/dc/dc/commit/d41f88f8bf5bb2c725ee3edba085e8961ab8c727))
+* **edit:** use cellValidation keys and hotDataSchema to fill in defaults on add row ([4957548](https://git.datacontroller.io/dc/dc/commit/495754816c0e757b8f8b1c0ad51246dc7b65d957))
+* enabling closeouts for UPDATE in CAS tables ([8b8e8ae](https://git.datacontroller.io/dc/dc/commit/8b8e8aec159ff2f50cfa4683bcd7a25aabb75bf8))
+* enabling rollback when the table has formatted values ([815d6e9](https://git.datacontroller.io/dc/dc/commit/815d6e97a8e304d79d48cc949ba126e02a318dc1))
+* improvements to validations ([6ceb681](https://git.datacontroller.io/dc/dc/commit/6ceb6814633691b6d4ac2cb898cfb75e9d609102))
+* remove IE checks and conditions ([ece6bd1](https://git.datacontroller.io/dc/dc/commit/ece6bd1d787d722531334fc4f1396a94cf6d92ec))
+* updates to demodata to enable auto CAS promote ([7740d2a](https://git.datacontroller.io/dc/dc/commit/7740d2ac8694295b33b40a30603d8239818896f5))
+* upgrade angular core and compiler ([aecd597](https://git.datacontroller.io/dc/dc/commit/aecd5976875a7c01189248c5f5aa3478b28c1ab2))
+* using fcopy instead of binary copy for file upload, for Viya 2026 compatibility ([716ee6e](https://git.datacontroller.io/dc/dc/commit/716ee6eba0a28f4f0a7a96b0719caf15da9b6e78))
+* **viewer:** search causing blank Handsontable ([338c7a2](https://git.datacontroller.io/dc/dc/commit/338c7a2e418c47e34331bd04718cd816f978837c)), closes [#206](https://git.datacontroller.io/dc/dc/issues/206)
+
+
+### Features
+
+* adding demo data job ([8c2aeac](https://git.datacontroller.io/dc/dc/commit/8c2aeacc85da5c106c356709cefcb412ed0a71db))
+* **dq rules:** notnull validation when invalid cell, will auto populate a default value ([96f2518](https://git.datacontroller.io/dc/dc/commit/96f2518af9e547956be5862a1322d9ab8e07369b))
+
+## [7.2.8](https://git.datacontroller.io/dc/dc/compare/v7.2.7...v7.2.8) (2026-02-06)
+
+
+### Bug Fixes
+
+* bump adapter version ([f4c8699](https://git.datacontroller.io/dc/dc/commit/f4c8699aaf0b1e01b447296978a4f6dedc8903f9))
+
 ## [7.2.7](https://git.datacontroller.io/dc/dc/compare/v7.2.6...v7.2.7) (2026-02-05)
 
 

client/angular.json

@@ -41,6 +41,8 @@
               "zone.js",
               "text-encoding",
               "crypto-js/md5",
+              "crypto-js/sha1",
+              "crypto-js/sha512",
               "buffer",
               "numbro",
               "@clr/icons",

client/cypress/e2e/csv-limited.cy.ts

@@ -0,0 +1,95 @@
+const username = Cypress.env('username')
+const password = Cypress.env('password')
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+const serverType = Cypress.env('serverType')
+const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
+const fixturePath = 'csvs/'
+
+context('csv file upload restriction (free tier): ', function () {
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    cy.get('body').then(($body) => {
+      const usernameInput = $body.find('input.username')[0]
+
+      if (usernameInput && !Cypress.dom.isHidden(usernameInput)) {
+        cy.get('input.username').type(username)
+        cy.get('input.password').type(password)
+        cy.get('.login-group button').click()
+      }
+    })
+
+    cy.get('.app-loading', { timeout: longerCommandTimeout }).should(
+      'not.exist'
+    )
+
+    // Skip licensing page if presented - continue with free tier
+    cy.url().then((url) => {
+      if (url.includes('licensing')) {
+        cy.get('button').contains('Continue with free tier').click()
+      }
+    })
+
+    visitPage('home')
+  })
+
+  it('1 | File upload is restricted on free tier', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    // Click upload button - should show feature locked modal
+    cy.get('.buttonBar button:last-child').should('exist').click()
+
+    cy.get('.modal-title').should('contain', 'Locked Feature (File Upload)')
+  })
+})
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let targetLib
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            targetLib = node
+            break
+          }
+        }
+
+        cy.get(targetLib).within(() => {
+          cy.get('.clr-tree-node-content-container > button').click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+}
+
+const attachFile = (filename: string, callback?: any) => {
+  cy.get('.buttonBar button:last-child')
+    .should('exist')
+    .click()
+    .then(() => {
+      cy.get('input[type="file"]#file-upload')
+        .attachFile(`/${fixturePath}/${filename}`)
+        .then(() => {
+          if (callback) callback()
+        })
+    })
+}
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}

client/cypress/e2e/excel.cy.ts

@@ -309,6 +309,83 @@ context('excel tests: ', function () {
     })
   })
 
+  it('22 | Uploads password protected Excel and unlocks with correct password', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    cy.get('.buttonBar button:last-child')
+      .should('exist')
+      .click()
+      .then(() => {
+        cy.get('input[type="file"]#file-upload')
+          .attachFile(`/${fixturePath}/regular_excel_password.xlsx`)
+          .then(() => {
+            // Wait for password modal to appear
+            cy.get('#filePasswordInput', { timeout: 10000 })
+              .should('be.visible')
+              .type('123123')
+
+            // Click Unlock button
+            cy.get('.btn.btn-success-outline').should('not.be.disabled').click()
+
+            // Click away the overlay
+            cy.get('.modal-footer .btn.btn-primary', { timeout: 5000 }).click()
+
+            // Verify file loads successfully
+            cy.get('.btn-upload-preview', { timeout: 60000 })
+              .should('be.visible')
+              .then(() => {
+                submitExcel()
+                rejectExcel(done)
+              })
+          })
+      })
+  })
+
+  it('23 | Uploads password protected Excel and handles wrong password', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    cy.get('.buttonBar button:last-child')
+      .should('exist')
+      .click()
+      .then(() => {
+        cy.get('input[type="file"]#file-upload')
+          .attachFile(`/${fixturePath}/regular_excel_password.xlsx`)
+          .then(() => {
+            // First attempt: Enter wrong password
+            cy.get('#filePasswordInput', { timeout: 10000 })
+              .should('be.visible')
+              .type('wrongpassword')
+
+            cy.get('.btn.btn-success-outline').should('not.be.disabled').click()
+
+            // Verify error message appears
+            cy.get('.modal-footer .color-red', { timeout: 10000 })
+              .should('be.visible')
+              .should('contain', "Sorry that didn't work, try again.")
+
+            // Modal should still be open for retry
+            cy.get('#filePasswordInput')
+              .should('be.visible')
+              .clear()
+              .type('123123')
+
+            // Second attempt: Enter correct password
+            cy.get('.btn.btn-success-outline').should('not.be.disabled').click()
+
+            // Click away the overlay
+            cy.get('.modal-footer .btn.btn-primary', { timeout: 5000 }).click()
+
+            // Verify file loads successfully
+            cy.get('.btn-upload-preview', { timeout: 60000 })
+              .should('be.visible')
+              .then(() => {
+                submitExcel()
+                rejectExcel(done)
+              })
+          })
+      })
+  })
+
   // Large files break Cypress
 
   // it ('? | Uploads Excel with size of 5MB', (done) => {

client/cypress/fixtures/csvs/regular.csv

@@ -4,7 +4,11 @@ PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,SOME_NUM,SOME_DATE,SOME_DATETIME,SOME_
 2,even more dummy data,Option 3,42,12FEB1960,01JAN1960:00:00:42,0:02:22,3,44
 3,"It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told: It was a dark and stormy night.  The wind was blowing a gale!  The captain said to his mate - mate, tell us a tale.  And this, is the tale he told:",Option 2,1613.001,27FEB1961,01JAN1960:00:07:03,0:00:44,3,44
 4,if you can fill the unforgiving minute,Option 1,1613.0011235,02AUG1971,29MAY1973:06:12:03,0:06:52,3,44
-1010,10 bottles of beer on the wall,Option 1,0.9153696885,04MAR1962,01JAN1960:12:47:55,0:01:40,92,76
+1010,"10 bottles of beer
+
+
+
+ on the wall",Option 1,0.9153696885,04MAR1962,01JAN1960:12:47:55,0:01:40,92,76
 1011,11 bottles of beer on the wall,Option 1,0.3531217558,29MAR1960,01JAN1960:03:33:24,0:01:03,80,29
 1012,12 bottles of beer on the wall,Option 1,0.6743748717,02AUG1962,01JAN1960:07:25:59,0:00:10,16,98
 1013,13 bottles of beer on the wall,Option 1,0.1305445992,11SEP1960,01JAN1960:13:51:32,0:00:35,73,15

client/package.json

@@ -37,21 +37,21 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^19.2.17",
+    "@angular/animations": "^19.2.18",
     "@angular/cdk": "^19.2.19",
-    "@angular/common": "^19.2.17",
-    "@angular/compiler": "^19.2.17",
-    "@angular/core": "^19.2.17",
-    "@angular/forms": "^19.2.17",
-    "@angular/platform-browser": "^19.2.17",
-    "@angular/platform-browser-dynamic": "^19.2.17",
-    "@angular/router": "^19.2.17",
+    "@angular/common": "^19.2.18",
+    "@angular/compiler": "^19.2.18",
+    "@angular/core": "^19.2.18",
+    "@angular/forms": "^19.2.18",
+    "@angular/platform-browser": "^19.2.18",
+    "@angular/platform-browser-dynamic": "^19.2.18",
+    "@angular/router": "^19.2.18",
     "@cds/core": "^6.15.1",
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
     "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
     "@handsontable/angular-wrapper": "16.0.1",
-    "@sasjs/adapter": "^4.16.0",
+    "@sasjs/adapter": "^4.16.3",
     "@sasjs/utils": "^3.5.3",
     "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
     "@types/d3-graphviz": "^2.6.7",
@@ -67,7 +67,7 @@
     "hyperformula": "^2.5.0",
     "iconv-lite": "^0.5.0",
     "jquery-datetimepicker": "^2.5.21",
-    "jsrsasign": "^11.1.0",
+    "jsrsasign": "11.1.1",
     "marked": "^5.0.0",
     "moment": "^2.30.1",
     "ngx-clipboard": "^16.0.0",
@@ -82,7 +82,7 @@
     "tslib": "^2.3.0",
     "vm": "^0.1.0",
     "webpack": "^5.91.0",
-    "xlsx": "^0.18.5",
+    "xlsx": "file:libraries/xlsx-0.20.3.tgz",
     "zone.js": "~0.15.1"
   },
   "devDependencies": {
@@ -93,7 +93,7 @@
     "@angular-eslint/schematics": "19.8.1",
     "@angular-eslint/template-parser": "19.8.1",
     "@angular/cli": "^19.2.19",
-    "@angular/compiler-cli": "^19.2.17",
+    "@angular/compiler-cli": "^19.2.18",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@compodoc/compodoc": "^1.1.21",
     "@cypress/webpack-preprocessor": "^5.17.1",

client/src/app/_globals.ts

@@ -55,6 +55,7 @@ export interface HandsontableStaticConfig {
  * Cached viyaApi collections, search and selected endpoint
  */
 export const globals: {
+  embed: boolean
   rootParam: string
   dcLib: string
   xlmaps: XLMapListItem[]
@@ -69,6 +70,7 @@ export const globals: {
   handsontable: HandsontableStaticConfig
   [key: string]: any
 } = {
+  embed: false,
   rootParam: <string>'',
   dcLib: '',
   xlmaps: [],

client/src/app/app.component.html

@@ -107,7 +107,7 @@
     </div>
   </ng-container>
 
-  <header class="app-header">
+  <header class="app-header" *ngIf="!embed">
     <!-- <button
       *ngIf="
         isMainRoute('view') ||
@@ -213,9 +213,10 @@
   </header>
   <nav
     *ngIf="
-      router.url.includes('submitted') ||
-      router.url.includes('approve') ||
-      router.url.includes('history')
+      !embed &&
+      (router.url.includes('submitted') ||
+        router.url.includes('approve') ||
+        router.url.includes('history'))
     "
     class="subnav"
   >

client/src/app/app.component.ts

@@ -70,6 +70,7 @@ export class AppComponent {
 
   public syssite = this.appService.syssite
   public licenceState = this.licenceService.licenceState
+  public embed = globals.embed
 
   constructor(
     private appService: AppService,
@@ -143,6 +144,16 @@ export class AppComponent {
       }
     })
 
+    const hashQuery = window.location.hash.split('?')[1]
+    if (hashQuery) {
+      const embedParam = new URLSearchParams(hashQuery).get('embed')
+      if (embedParam !== null) {
+        const isEmbed = embedParam !== 'false'
+        globals.embed = isEmbed
+        this.embed = isEmbed
+      }
+    }
+
     this.subscribeToShowAbortModal()
     this.subscribeToRequestsModal()
     this.subscribeToStartupData()

client/src/app/editor/components/edit-record/edit-record.component.ts

@@ -14,6 +14,7 @@ import { HelperService } from 'src/app/services/helper.service'
 import { SasStoreService } from 'src/app/services/sas-store.service'
 import { DcValidator } from 'src/app/shared/dc-validator/dc-validator'
 import { DcValidation } from 'src/app/shared/dc-validator/models/dc-validation.model'
+import { isEmpty } from 'src/app/shared/dc-validator/utils/isEmpty'
 import {
   EditRecordDropdownChangeEvent,
   EditRecordInputFocusedEvent
@@ -146,23 +147,63 @@ export class EditRecordComponent implements OnInit {
     }, 0)
   }
 
-  async recordInputChange(event: any, colName: string) {
+  async recordInputChange(event: any, colName: string): Promise<void> {
     const colRules = this.currentRecordValidator?.getRule(colName)
     const value = event.target.value
 
     this.helperService.debounceCall(300, () => {
       this.validateRecordCol(colRules, value).then((valid: boolean) => {
-        const index = this.currentRecordInvalidCols.indexOf(colName)
+        this.updateValidationState(colName, valid)
 
-        if (valid) {
-          if (index > -1) this.currentRecordInvalidCols.splice(index, 1)
-        } else {
-          if (index < 0) this.currentRecordInvalidCols.push(colName)
+        if (!valid) {
+          this.tryAutoPopulateNotNull(event, colName, colRules, value)
         }
       })
     })
   }
 
+  /**
+   * Updates the invalid columns list based on validation result
+   */
+  private updateValidationState(colName: string, valid: boolean): void {
+    const index = this.currentRecordInvalidCols.indexOf(colName)
+
+    if (valid && index > -1) {
+      this.currentRecordInvalidCols.splice(index, 1)
+    } else if (!valid && index < 0) {
+      this.currentRecordInvalidCols.push(colName)
+    }
+  }
+
+  /**
+   * Auto-populates NOTNULL default value when the field is empty and has a default
+   */
+  private tryAutoPopulateNotNull(
+    event: any,
+    colName: string,
+    colRules: DcValidation | undefined,
+    value: any
+  ): void {
+    if (
+      !isEmpty(value) ||
+      !this.currentRecordValidator ||
+      !this.currentRecord
+    ) {
+      return
+    }
+
+    const defaultValue =
+      this.currentRecordValidator.getNotNullDefaultValue(colName)
+    if (defaultValue === undefined) return
+
+    this.currentRecord[colName] = defaultValue
+    event.target.value = defaultValue
+
+    this.validateRecordCol(colRules, defaultValue).then((isValid: boolean) => {
+      this.updateValidationState(colName, isValid)
+    })
+  }
+
   onNextRecordClick() {
     this.onNextRecord.emit()
   }

client/src/app/editor/editor.component.html

@@ -165,7 +165,7 @@
         class="card-header clr-row buttonBar headerBar clr-flex-md-row clr-justify-content-center clr-justify-content-lg-end"
       >
         <div
-          *ngIf="tableTrue"
+          *ngIf="tableTrue && !embed"
           class="clr-col-12 clr-col-md-3 clr-col-lg-4 backBtn"
         >
           <span

client/src/app/editor/editor.component.ts

@@ -13,6 +13,7 @@ import {
 import { ActivatedRoute, Router } from '@angular/router'
 import Handsontable from 'handsontable'
 import { Subject, Subscription } from 'rxjs'
+import { sanitiseForSas } from '../shared/utils/sanitise'
 import { SasStoreService } from '../services/sas-store.service'
 
 type AOA = any[][]
@@ -43,6 +44,7 @@ import { Col } from '../shared/dc-validator/models/col.model'
 import { DcValidation } from '../shared/dc-validator/models/dc-validation.model'
 import { DQRule } from '../shared/dc-validator/models/dq-rules.model'
 import { getHotDataSchema } from '../shared/dc-validator/utils/getHotDataSchema'
+import { isEmpty } from '../shared/dc-validator/utils/isEmpty'
 import { globals } from '../_globals'
 import { UploadStaterComponent } from './components/upload-stater/upload-stater.component'
 import { DynamicExtendedCellValidation } from './models/dynamicExtendedCellValidation'
@@ -263,6 +265,9 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
   public badEdit = false
   public badEditCause: string | undefined
   public badEditTitle: string | undefined
+  get embed() {
+    return globals.embed
+  }
   public tableTrue: boolean | undefined
   public saveLoading = false
   public approvers: string[] = []
@@ -1045,12 +1050,16 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
   }
 
   /**
-   * Creates a new empty row object with proper structure
+   * Creates a new empty row object with proper structure.
+   * Columns with NOTNULL DQ rules are pre-populated with their RULE_VALUE.
    */
   private createEmptyRow(): any {
     const newRow: any = {}
-    this.headerColumns.forEach((col: string) => {
-      newRow[col] = ''
+    this.cellValidation.forEach((rule: any) => {
+      const dataKey = rule.data
+      newRow[dataKey] = this.hotDataSchema.hasOwnProperty(dataKey)
+        ? this.hotDataSchema[dataKey]
+        : ''
     })
     newRow['noLinkOption'] = true
     return newRow
@@ -1661,7 +1670,7 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
       this.submit = true
       const updateParams: any = {}
       updateParams.ACTION = 'LOAD'
-      this.message = this.message.replace(/\n/g, '. ')
+      this.message = sanitiseForSas(this.message.replace(/\n/g, '. '))
       updateParams.MESSAGE = this.message
       // updateParams.APPROVER = this.approver;
       updateParams.LIBDS = this.libds
@@ -2676,13 +2685,14 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
     // Note: this.headerColumns and this.columnHeader contains same data
     // need to resolve redundancy
 
-    // default schema
+    // default schema - includes NOTNULL defaults from DQ rules
     for (let i = 0; i < this.headerColumns.length; i++) {
       const colType = this.cellValidation[i].type
 
       this.hotDataSchema[this.cellValidation[i].data] = getHotDataSchema(
         colType,
-        this.cellValidation[i]
+        this.cellValidation[i],
+        this.dcValidator?.getDqDetails()
       )
     }
 
@@ -2987,6 +2997,29 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
       }
     )
 
+    // Auto-populate NOTNULL default when validation fails due to empty value
+    hot.addHook(
+      'afterValidate',
+      (isValid: boolean, value: any, row: number, prop: string | number) => {
+        if (isValid || !isEmpty(value)) return
+
+        const colName =
+          typeof prop === 'string'
+            ? prop
+            : (hot.colToProp(prop as number) as string)
+
+        const defaultValue = this.dcValidator?.getNotNullDefaultValue(colName)
+        if (defaultValue === undefined) return
+
+        // Auto-populate using setTimeout to avoid modifying during validation
+        setTimeout(() => {
+          if (isEmpty(hot.getDataAtRowProp(row, colName))) {
+            hot.setDataAtRowProp(row, colName, defaultValue, 'autoPopulate')
+          }
+        }, 0)
+      }
+    )
+
     hot.addHook('beforePaste', (data: any, cords: any) => {
       const startCol = cords[0].startCol
 

client/src/app/free-tier.config.ts

@@ -30,7 +30,7 @@ export const freeTierConfig: LicenceState = {
   lineage_daily_limit: 3,
   tables_in_library_limit: 35,
   viewbox: true,
-  fileUpload: true,
+  fileUpload: false,
   editRecord: true,
   addRecord: true
 }

client/src/app/lineage/lineage.component.html

@@ -239,13 +239,7 @@
 
               <clr-dropdown-menu clrPosition="bottom-left" *clrIfOpen>
                 <div (click)="downloadSVG()" clrDropdownItem>SVG</div>
-                <div
-                  *ngIf="!helperService.isMicrosoft"
-                  (click)="downloadPNG()"
-                  clrDropdownItem
-                >
-                  PNG
-                </div>
+                <div (click)="downloadPNG()" clrDropdownItem>PNG</div>
                 <div (click)="downloadDot()" clrDropdownItem>Dot</div>
                 <div *ngIf="flatdata" (click)="downloadCSV()" clrDropdownItem>
                   CSV
@@ -366,13 +360,7 @@
 
         <clr-dropdown-menu clrPosition="bottom-left" *clrIfOpen>
           <div (click)="renderToDownload('SVG')" clrDropdownItem>SVG</div>
-          <div
-            *ngIf="!helperService.isMicrosoft"
-            (click)="renderToDownload('PNG')"
-            clrDropdownItem
-          >
-            PNG
-          </div>
+          <div (click)="renderToDownload('PNG')" clrDropdownItem>PNG</div>
           <div (click)="downloadDot(); cancelRenderingGraph()" clrDropdownItem>
             Dot
           </div>

client/src/app/review/approve-details/approve-details.component.ts

@@ -1,4 +1,5 @@
 import { ActivatedRoute } from '@angular/router'
+import { sanitiseForSas } from '../../shared/utils/sanitise'
 import { SasStoreService } from '../../services/sas-store.service'
 import {
   Component,
@@ -136,7 +137,7 @@ export class ApproveDetailsComponent implements AfterViewInit, OnDestroy {
 
   public async rejecting() {
     this.rejectLoading = true
-    this.submitReason = this.submitReason.replace(/\n/g, '. ')
+    this.submitReason = sanitiseForSas(this.submitReason.replace(/\n/g, '. '))
 
     let rejParams = {
       STP_ACTION: 'REJECT_TABLE',

client/src/app/services/helper.service.ts

@@ -11,12 +11,8 @@ const librariesToShow = 50
 export class HelperService {
   public shownLibraries: number = librariesToShow
   public loadMoreCount: number = librariesToShow
-  public isMicrosoft: boolean = false
 
-  constructor(private sasService: SasService) {
-    this.isMicrosoft = this.isIEorEDGE()
-    console.log('Is IE or Edge?', this.isMicrosoft)
-  }
+  constructor(private sasService: SasService) {}
 
   /**
    * Converts a JavaScript date object to a SAS Date or Datetime, given the logic below:
@@ -215,32 +211,6 @@ export class HelperService {
     })
   }
 
-  public isIEorEDGE() {
-    var ua = window.navigator.userAgent
-
-    var msie = ua.indexOf('MSIE ')
-    if (msie > 0) {
-      // IE 10 or older => return version number
-      return true
-    }
-
-    var trident = ua.indexOf('Trident/')
-    if (trident > 0) {
-      // IE 11 => return version number
-      var rv = ua.indexOf('rv:')
-      return true
-    }
-
-    var edge = ua.indexOf('Edge/')
-    if (edge > 0) {
-      // Edge (IE 12+) => return version number
-      return true
-    }
-
-    // other browser
-    return false
-  }
-
   public convertObjectsToArray(
     objectArray: Array<object>,
     deepClone: boolean = false

client/src/app/services/sas-viya.service.ts

@@ -120,9 +120,12 @@ export class SasViyaService {
   }
 
   getComputeContexts(): Observable<ViyaComputeContexts> {
-    return this.get<ViyaComputeContexts>(`${this.serverUrl}/compute/contexts`, {
-      withCredentials: true
-    })
+    return this.get<ViyaComputeContexts>(
+      `${this.serverUrl}/compute/contexts?limit=1000`,
+      {
+        withCredentials: true
+      }
+    )
   }
 
   getComputeContextById(id: string): Observable<ComputeContextDetails> {

client/src/app/shared/dc-validator/dc-validator.ts

@@ -15,6 +15,7 @@ import {
 } from './models/dc-validation.model'
 import { DQRule, DQRuleTypes } from './models/dq-rules.model'
 import { getDqDataCols } from './utils/getDqDataCols'
+import { getNotNullDefault } from './utils/getNotNullDefault'
 import { mergeColsRules } from './utils/mergeColsRules'
 import { parseColType } from './utils/parseColType'
 import { dqValidate } from './validations/dq-validation'
@@ -133,6 +134,19 @@ export class DcValidator {
     }
   }
 
+  /**
+   * Returns the RULE_VALUE for a NOTNULL rule on the given column.
+   * Used for auto-populating default values when cells are empty.
+   * Converts to number for numeric columns.
+   *
+   * @param col column name
+   * @returns RULE_VALUE (string or number) if NOTNULL rule exists, otherwise undefined
+   */
+  getNotNullDefaultValue(col: string): string | number | undefined {
+    const colRule = this.getRule(col)
+    return getNotNullDefault(col, this.dqrules, colRule?.type)
+  }
+
   /**
    * Retrieves dropdown source for given dc validation rule
    * The values comes from MPE_SELECTBOX table

client/src/app/shared/dc-validator/tests/getHotDataSchema.spec.ts

@@ -1,3 +1,4 @@
+import { DQRule } from '../models/dq-rules.model'
 import { getHotDataSchema } from '../utils/getHotDataSchema'
 
 describe('DC Validator - hot data schema', () => {
@@ -8,4 +9,58 @@ describe('DC Validator - hot data schema', () => {
     ).toEqual(1)
     expect(getHotDataSchema('missing')).toEqual('')
   })
+
+  describe('NOTNULL defaults', () => {
+    const dqRules: DQRule[] = [
+      {
+        BASE_COL: 'TEXT_COL',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: 'default_text',
+        X: 1
+      },
+      { BASE_COL: 'NUM_COL', RULE_TYPE: 'NOTNULL', RULE_VALUE: '42', X: 1 }
+    ]
+
+    it('should return NOTNULL default for text column', () => {
+      expect(getHotDataSchema('text', { data: 'TEXT_COL' }, dqRules)).toEqual(
+        'default_text'
+      )
+    })
+
+    it('should return NOTNULL default as number for numeric column', () => {
+      expect(getHotDataSchema('numeric', { data: 'NUM_COL' }, dqRules)).toEqual(
+        42
+      )
+    })
+
+    it('should fall back to type default when no NOTNULL rule exists', () => {
+      expect(
+        getHotDataSchema('numeric', { data: 'OTHER_COL' }, dqRules)
+      ).toEqual('')
+    })
+
+    it('should prioritize NOTNULL over autocomplete first option', () => {
+      const rulesWithAutocomplete: DQRule[] = [
+        {
+          BASE_COL: 'SELECT_COL',
+          RULE_TYPE: 'NOTNULL',
+          RULE_VALUE: 'priority_value',
+          X: 1
+        },
+        {
+          BASE_COL: 'SELECT_COL',
+          RULE_TYPE: 'HARDSELECT',
+          RULE_VALUE: 'ignored',
+          X: 1
+        }
+      ]
+      expect(
+        getHotDataSchema(
+          'autocomplete',
+          { data: 'SELECT_COL', source: ['first', 'second'] },
+          rulesWithAutocomplete
+        )
+      ).toEqual('priority_value')
+    })
+  })
 })

client/src/app/shared/dc-validator/tests/getNotNullDefault.spec.ts

@@ -0,0 +1,65 @@
+import { DQRule } from '../models/dq-rules.model'
+import { getNotNullDefault } from '../utils/getNotNullDefault'
+
+describe('DC Validator - getNotNullDefault', () => {
+  const dqRules: DQRule[] = [
+    {
+      BASE_COL: 'TEXT_COL',
+      RULE_TYPE: 'NOTNULL',
+      RULE_VALUE: 'default_text',
+      X: 1
+    },
+    { BASE_COL: 'NUM_COL', RULE_TYPE: 'NOTNULL', RULE_VALUE: '42', X: 1 },
+    { BASE_COL: 'EMPTY_RULE', RULE_TYPE: 'NOTNULL', RULE_VALUE: '  ', X: 1 },
+    {
+      BASE_COL: 'OTHER_COL',
+      RULE_TYPE: 'HARDSELECT',
+      RULE_VALUE: 'some_value',
+      X: 1
+    }
+  ]
+
+  it('should return string value for text columns', () => {
+    expect(getNotNullDefault('TEXT_COL', dqRules, 'text')).toEqual(
+      'default_text'
+    )
+  })
+
+  it('should return number for numeric columns when RULE_VALUE is numeric', () => {
+    expect(getNotNullDefault('NUM_COL', dqRules, 'numeric')).toEqual(42)
+  })
+
+  it('should return string for numeric columns when RULE_VALUE is not numeric', () => {
+    const rulesWithNonNumeric: DQRule[] = [
+      {
+        BASE_COL: 'NUM_COL',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: 'not_a_number',
+        X: 1
+      }
+    ]
+    expect(
+      getNotNullDefault('NUM_COL', rulesWithNonNumeric, 'numeric')
+    ).toEqual('not_a_number')
+  })
+
+  it('should return undefined for empty RULE_VALUE', () => {
+    expect(getNotNullDefault('EMPTY_RULE', dqRules, 'text')).toBeUndefined()
+  })
+
+  it('should return undefined for columns without NOTNULL rule', () => {
+    expect(getNotNullDefault('OTHER_COL', dqRules, 'text')).toBeUndefined()
+  })
+
+  it('should return undefined for non-existent columns', () => {
+    expect(getNotNullDefault('MISSING_COL', dqRules, 'text')).toBeUndefined()
+  })
+
+  it('should return undefined for empty dqRules array', () => {
+    expect(getNotNullDefault('TEXT_COL', [], 'text')).toBeUndefined()
+  })
+
+  it('should return string when colType is undefined', () => {
+    expect(getNotNullDefault('NUM_COL', dqRules, undefined)).toEqual('42')
+  })
+})

client/src/app/shared/dc-validator/tests/isEmpty.spec.ts

@@ -0,0 +1,39 @@
+import { isEmpty } from '../utils/isEmpty'
+
+describe('DC Validator - isEmpty', () => {
+  it('should return true for null', () => {
+    expect(isEmpty(null)).toBe(true)
+  })
+
+  it('should return true for undefined', () => {
+    expect(isEmpty(undefined)).toBe(true)
+  })
+
+  it('should return true for empty string', () => {
+    expect(isEmpty('')).toBe(true)
+  })
+
+  it('should return true for whitespace-only string', () => {
+    expect(isEmpty('   ')).toBe(true)
+    expect(isEmpty('\t\n')).toBe(true)
+  })
+
+  it('should return false for non-empty string', () => {
+    expect(isEmpty('hello')).toBe(false)
+    expect(isEmpty(' hello ')).toBe(false)
+  })
+
+  it('should return false for number zero', () => {
+    expect(isEmpty(0)).toBe(false)
+  })
+
+  it('should return false for non-zero numbers', () => {
+    expect(isEmpty(42)).toBe(false)
+    expect(isEmpty(-1)).toBe(false)
+  })
+
+  it('should return false for boolean values', () => {
+    expect(isEmpty(true)).toBe(false)
+    expect(isEmpty(false)).toBe(false)
+  })
+})

client/src/app/shared/dc-validator/utils/getHotDataSchema.ts

@@ -1,4 +1,6 @@
 import { DcValidation } from '../models/dc-validation.model'
+import { DQRule } from '../models/dq-rules.model'
+import { getNotNullDefault } from './getNotNullDefault'
 
 const schemaTypeMap: { [key: string]: any } = {
   numeric: '',
@@ -7,14 +9,25 @@ const schemaTypeMap: { [key: string]: any } = {
 
 /**
  * Schema defines the default values for given types. For example when new row is added.
+ * Priority: NOTNULL RULE_VALUE > autocomplete first option > type default
  */
-export const getHotDataSchema = (
+export function getHotDataSchema(
   type: string | undefined,
-  cellValidation?: DcValidation
-): any => {
+  cellValidation?: DcValidation,
+  dqRules?: DQRule[]
+): any {
+  // Check for NOTNULL default first
+  if (dqRules && cellValidation?.data) {
+    const defaultValue = getNotNullDefault(cellValidation.data, dqRules, type)
+    if (defaultValue !== undefined) {
+      return defaultValue
+    }
+  }
+
   if (!type) return schemaTypeMap.default
 
   switch (type) {
+    case 'dropdown':
     case 'autocomplete': {
       return cellValidation && cellValidation.source
         ? (cellValidation.source as string[] | number[])[0]

client/src/app/shared/dc-validator/utils/getNotNullDefault.ts

@@ -0,0 +1,30 @@
+import { DQRule } from '../models/dq-rules.model'
+
+/**
+ * Returns the NOTNULL default value for a column from DQ rules.
+ * Converts to number for numeric columns based on colType parameter.
+ *
+ * @param colName column name to look up
+ * @param dqRules array of DQ rules
+ * @param colType column type (e.g., 'numeric', 'text')
+ * @returns default value (string or number) if NOTNULL rule exists with non-empty value, otherwise undefined
+ */
+export function getNotNullDefault(
+  colName: string,
+  dqRules: DQRule[],
+  colType?: string
+): string | number | undefined {
+  const notNullRule = dqRules.find(
+    (rule: DQRule) => rule.BASE_COL === colName && rule.RULE_TYPE === 'NOTNULL'
+  )
+
+  if (!notNullRule?.RULE_VALUE || notNullRule.RULE_VALUE.trim().length === 0) {
+    return undefined
+  }
+
+  if (colType === 'numeric' && !isNaN(Number(notNullRule.RULE_VALUE))) {
+    return Number(notNullRule.RULE_VALUE)
+  }
+
+  return notNullRule.RULE_VALUE
+}

client/src/app/shared/dc-validator/utils/isEmpty.ts

@@ -0,0 +1,8 @@
+/**
+ * Checks if a value is considered empty for NOTNULL validation purposes.
+ * A value is empty if it's null, undefined, or a string that is blank after trimming.
+ */
+export function isEmpty(value: unknown): boolean {
+  if (value === null || value === undefined) return true
+  return value.toString().trim().length === 0
+}

client/src/app/shared/spreadsheet-util/spreadsheet-util.ts

@@ -375,38 +375,30 @@ export class SpreadsheetUtil {
     fileType: string
   ): Promise<ParseResult> {
     return new Promise((resolve, reject) => {
-      if (this.licenceState.value.submit_rows_limit !== Infinity) {
+      if (!this.licenceState.value.fileUpload) {
         uploader.queue.pop()
         return reject(
-          'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
+          'File uploads are not enabled for this licence. Please contact support@datacontroller.io'
         )
       }
 
-      if (parseParams.encoding === 'WLATIN1') {
-        let reader = new FileReader()
-        const self = this
-        // Closure to capture the file information.
-        reader.onload = (theFile: any) => {
-          let encoded = iconv.decode(
-            Buffer.from(theFile.target.result),
-            'CP-1252'
-          )
-          let blob = new Blob([encoded], { type: fileType })
-          let encodedFile: File = blobToFile(blob, parseParams.file.name)
-          uploader.queue.pop()
-          uploader.addToQueue([encodedFile])
+      if (parseParams.encoding !== 'WLATIN1') return resolve({ uploader })
 
-          return resolve({
-            uploader
-          })
-        }
+      const reader = new FileReader()
+      reader.onload = (theFile) => {
+        if (!theFile.target?.result) return resolve({ uploader })
 
-        reader.readAsArrayBuffer(parseParams.file)
-      } else {
-        return resolve({
-          uploader
-        })
+        const text = theFile.target.result as string
+        const encoded = iconv.encode(text, 'CP-1252')
+        const blob = new Blob([encoded], { type: fileType })
+        const encodedFile: File = blobToFile(blob, parseParams.file.name)
+        uploader.queue.pop()
+        uploader.addToQueue([encodedFile])
+
+        return resolve({ uploader })
       }
+
+      reader.readAsText(parseParams.file)
     })
   }
 
@@ -511,6 +503,21 @@ export class SpreadsheetUtil {
         return resolve(XLSX.read(data, opts))
       }
 
+      // TEMPORARILY DISABLED: Web Worker for XLSX parsing
+      // Worker is disabled because Angular/webpack bundles it as a separate chunk
+      // with a numeric filename (e.g., 411.hash.js). In SAS9/Viya streaming
+      // environments, all JS files need to be served through SASJobExecution
+      // with _program= parameter, but our post-build processor can't reliably
+      // find and replace the worker chunk reference in the minified output.
+      // FIX: Add "namedChunks": true to production config in angular.json
+      // (under projects.datacontroller.architect.build.configurations.production)
+      // This will output worker as "spreadsheet-worker.hash.js" instead of
+      // numeric ID, making it findable by post-processor.
+      // Trade-off: UI may briefly freeze when parsing large Excel files.
+
+      return resolve(XLSX.read(data, opts))
+
+      /*
       if (typeof Worker === 'undefined') {
         console.info(
           'Not using worker to parse the XLSX - no Worker available in this environment'
@@ -551,6 +558,7 @@ export class SpreadsheetUtil {
       setTimeout(() => {
         return resolve(XLSX.read(data, opts))
       }, 600 * 1000) // 10 minutes
+      */
     })
   }
 

client/src/app/shared/utils/sanitise.ts

@@ -0,0 +1,6 @@
+/**
+ * Strips characters that could cause SAS macro injection (& % ;).
+ */
+export function sanitiseForSas(input: string): string {
+  return input.replace(/[%&;]/g, '')
+}

client/src/app/spreadsheet.worker.ts

@@ -4,6 +4,9 @@
  * We use normal version of the XLSX (SheetJS)
  * Because at the moment "@sheet/crypto" can't work in the Web Worker environment
  * Because of the missing "global" variable.
+ *
+ * Version bumped to v0.20.3 (`libraries/xlsx-0.20.3.tgz`)
+ * @see https://cdn.sheetjs.com/
  */
 import * as XLSX from 'xlsx'
 

client/src/app/system/system.component.html

@@ -236,7 +236,36 @@
         <div class="admin-action">
           Download Configuration
 
-          <button (click)="downloadConfiguration()" class="btn btn-info btn-sm">
+          <div class="libref-group">
+            <clr-tooltip class="libref-tooltip">
+              <label clrTooltipTrigger class="libref-label">
+                Target DC Library
+                <cds-icon shape="info-circle" size="16"></cds-icon>
+              </label>
+              <clr-tooltip-content
+                clrPosition="bottom-left"
+                clrSize="md"
+                *clrIfOpen
+              >
+                Enter the target DC library and the downloaded files will
+                contain this, instead of the original.
+              </clr-tooltip-content>
+            </clr-tooltip>
+
+            <input
+              type="text"
+              class="clr-input libref-input"
+              maxlength="8"
+              [ngModel]="dcLib"
+              (ngModelChange)="targetLibref = $event.toUpperCase()"
+              placeholder="e.g. MYLIB"
+            />
+          </div>
+          <button
+            (click)="downloadConfiguration()"
+            [disabled]="targetLibref !== dcLib && !isValidLibref(targetLibref)"
+            class="btn btn-info btn-sm"
+          >
             DOWNLOAD
           </button>
         </div>

client/src/app/system/system.component.scss

@@ -0,0 +1,21 @@
+.libref-group {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  margin: 0 8px;
+}
+
+.libref-label {
+  cursor: pointer;
+  font-size: 0.55rem;
+  font-weight: 600;
+  color: var(--clr-p4-color, #565656);
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+}
+
+.libref-input {
+  width: 100px;
+  text-transform: uppercase;
+}

client/src/app/system/system.component.ts

@@ -10,6 +10,7 @@ import { EnvironmentInfo } from './models/environment-info.model'
 import { AppSettingsService } from '../services/app-settings.service'
 import { AppSettings } from '../models/AppSettings'
 import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
+import { globals } from '../_globals'
 
 @Component({
   selector: 'app-system',
@@ -39,6 +40,8 @@ export class SystemComponent implements OnInit {
   responseModal: boolean = false
 
   Infinity = Infinity
+  dcLib: string = globals.dcLib
+  targetLibref: string = globals.dcLib
 
   licenceState = this.licenceService.licenceState
   settings: AppSettings
@@ -71,13 +74,21 @@ export class SystemComponent implements OnInit {
     this.appSettingsService.setAppSettings(this.settings)
   }
 
+  isValidLibref(value: string): boolean {
+    return /^[A-Za-z_]\w{0,7}$/.test(value.trim())
+  }
+
   downloadConfiguration() {
     let sasjsConfig = this.sasService.getSasjsConfig()
     let storage = sasjsConfig.serverUrl
     let metaData = sasjsConfig.appLoc
     let path = this.sasService.getExecutionPath()
+    let lib = this.targetLibref.toUpperCase().trim()
     let downUrl =
       storage + path + '/?_program=' + metaData + '/services/admin/exportconfig'
+    if (lib && lib !== this.dcLib && this.isValidLibref(lib)) {
+      downUrl += '&dclib=' + encodeURIComponent(lib)
+    }
     window.open(downUrl)
   }
 

client/src/app/viewer/viewer.component.ts

@@ -160,11 +160,7 @@ export class ViewerComponent
     afterGetColHeader: (col: number, th: any, headerLevel: number) => {
       // CRITICAL: Prevent "colToProp method cannot be called because this Handsontable instance has been destroyed" error
       // This callback can be triggered even after the instance is destroyed during rapid table switching
-      if (
-        !this.hotInstance ||
-        this.hotInstance.isDestroyed ||
-        this.isTableSwitching
-      ) {
+      if (!this.hotInstance || this.hotInstance.isDestroyed) {
         // Graceful fallback: apply only dark mode styling when instance is unavailable
         th.classList.add(globals.handsontable.darkTableHeaderClass)
         return
@@ -761,10 +757,6 @@ export class ViewerComponent
     // This prevents callbacks from accessing destroyed instances during table switching
     this.isTableSwitching = true
 
-    // CLEANUP: Ensure any existing Handsontable instance is properly destroyed
-    // This prevents "instance destroyed" errors
-    this.cleanupHotInstance()
-
     this.loadingTableView = true
 
     let libDataset: any
@@ -1177,8 +1169,6 @@ export class ViewerComponent
    * Purpose: Prevents "instance destroyed" errors and memory leaks during table switching
    *
    * Called from:
-   * - viewData() - before loading new table data
-   * - setupHot() - before creating new instance
    * - ngOnDestroy() - component cleanup
    *
    * Safety features:
@@ -1195,107 +1185,113 @@ export class ViewerComponent
       }
     }
     this.hotInstance = null
+    this.hooksAttached = false
   }
 
   /**
    * PERFORMANCE: Configures Handsontable with enhanced error handling (workaround needed for HOT version 16 and above)
    *
    * 1. Duplicate call prevention (500ms window)
-   * 2. Reduced timeout delays (200ms + 50ms vs original 1000ms + 200ms)
-   * 3. Multiple validation checks to prevent race conditions
-   * 4. Forced render for immediate primary key styling
+   * 2. Multiple validation checks to prevent race conditions
+   * 3. Forced render for immediate primary key styling
    *
-   * Timeline: 50ms (viewData) + 200ms (main) + 50ms (component ready) = ~300ms total
-   * Previous: 100ms + 600ms + 100ms = 800ms (plus render delays = ~2 seconds)
+   * Instance lifecycle is managed by Angular's hot-table component via [data] and [settings] bindings.
+   * This method only applies additional config that can't go through bindings (hooks, PK styling).
    */
   private setupHot() {
-    // DUPLICATE PREVENTION: Avoid multiple setup calls during rapid table switching
     const now = Date.now()
     if (now - this.lastSetupTime < 500) {
       return
     }
     this.lastSetupTime = now
 
+    // VALIDATION: Don't setup if we're currently switching tables or still loading
+    if (this.loadingTableView || !this.libDataset) {
+      return
+    }
+
+    if (!this.hotInstance || this.hotInstance.isDestroyed) {
+      this.hotInstance = this.hotTableComponent?.hotInstance
+    }
+
+    if (this.hotInstance && !this.hotInstance.isDestroyed) {
+      this.configureHotInstance()
+      return
+    }
+
+    // Instance not ready yet — Angular may still be creating the component
     setTimeout(() => {
-      // VALIDATION: Don't setup if we're currently switching tables or still loading
-      if (this.loadingTableView || !this.libDataset) {
+      if (this.isTableSwitching || this.loadingTableView || !this.libDataset) {
         return
       }
 
-      // CLEANUP: Ensure clean slate before new setup
-      this.cleanupHotInstance()
+      this.hotInstance = this.hotTableComponent?.hotInstance
+      this.configureHotInstance()
+    }, 250)
+  }
 
-      // TIMING: Wait for Angular component to be ready (optimized from 100ms to 50ms)
-      setTimeout(() => {
-        // DOUBLE-CHECK: Ensure we're still in valid state after delays
-        if (
-          this.isTableSwitching ||
-          this.loadingTableView ||
-          !this.libDataset
-        ) {
+  private hooksAttached = false
+
+  /**
+   * Applies settings that can't go through Angular [settings] binding:
+   * - Primary key column header styling
+   * - Column width cap
+   * - ARIA accessibility hooks (attached once per instance)
+   */
+  private configureHotInstance() {
+    if (!this.hotInstance || this.hotInstance.isDestroyed) return
+
+    this.hotInstance.updateSettings({
+      height: this.hotTable.height,
+      modifyColWidth: (width: any, col: any) => {
+        if (width > 500) return 500
+        else return width
+      },
+      afterGetColHeader: (col: number, th: any) => {
+        // CRITICAL: Same safety checks as initial callback to prevent destroyed instance errors
+        if (!this.hotInstance || this.hotInstance.isDestroyed) {
+          th.classList.add(globals.handsontable.darkTableHeaderClass)
           return
         }
 
-        this.hotInstance = this.hotTableComponent?.hotInstance
+        try {
+          const column = this.hotInstance.colToProp(col) as string
 
-        if (this.hotInstance && !this.hotInstance.isDestroyed) {
-          this.hotInstance.updateSettings({
-            height: this.hotTable.height,
-            modifyColWidth: (width: any, col: any) => {
-              if (width > 500) return 500
-              else return width
-            },
-            afterGetColHeader: (col: number, th: any) => {
-              // CRITICAL: Same safety checks as initial callback to prevent destroyed instance errors
-              if (
-                !this.hotInstance ||
-                this.hotInstance.isDestroyed ||
-                this.isTableSwitching
-              ) {
-                th.classList.add(globals.handsontable.darkTableHeaderClass)
-                return
-              }
+          // PRIMARY KEY STYLING: Apply special styling to PK columns (populated from API response)
+          const isPKCol = column && this.headerPks.indexOf(column) > -1
+          if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
 
-              try {
-                const column = this.hotInstance.colToProp(col) as string
-
-                // PRIMARY KEY STYLING: Apply special styling to PK columns (populated from API response)
-                const isPKCol = column && this.headerPks.indexOf(column) > -1
-                if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
-
-                // DARK MODE: Apply to all headers
-                th.classList.add(globals.handsontable.darkTableHeaderClass)
-              } catch (error) {
-                // SAFETY NET: Ensure basic styling is always applied
-                th.classList.add(globals.handsontable.darkTableHeaderClass)
-              }
-            }
-          })
-
-          // Add hooks for accessibility fixes
-          this.hotInstance.addHook('afterRender', () => {
-            // Fix ARIA accessibility issues after each render
-            this.fixAriaAccessibility()
-          })
-
-          this.hotInstance.addHook('afterChange', () => {
-            // Fix ARIA accessibility issues after any data change
-            setTimeout(() => {
-              this.fixAriaAccessibility()
-            }, 50)
-          })
-
-          // Force immediate render to apply primary key styling
-          // Without this, styling would wait for ~2 seconds to be applied
-          // With this, styling appears in ~300ms total (workaround needed for HOT version 16 and above)
-          setTimeout(() => {
-            if (this.hotInstance && !this.hotInstance.isDestroyed) {
-              this.hotInstance.render()
-            }
-          }, 10)
+          // DARK MODE: Apply to all headers
+          th.classList.add(globals.handsontable.darkTableHeaderClass)
+        } catch (error) {
+          // SAFETY NET: Ensure basic styling is always applied
+          th.classList.add(globals.handsontable.darkTableHeaderClass)
         }
-      }, 50) // Optimized Angular component readiness delay
-    }, 200) // Optimized main setup delay (was 600ms)
+      }
+    })
+
+    // Add hooks for accessibility fixes
+    // Hooks are attached once per instance to avoid accumulating duplicate listeners
+    if (!this.hooksAttached) {
+      this.hotInstance.addHook('afterRender', () => {
+        // Fix ARIA accessibility issues after each render
+        this.fixAriaAccessibility()
+      })
+
+      this.hotInstance.addHook('afterChange', () => {
+        // Fix ARIA accessibility issues after any data change
+        setTimeout(() => {
+          this.fixAriaAccessibility()
+        }, 50)
+      })
+
+      this.hooksAttached = true
+    }
+
+    // Force immediate render to apply primary key styling
+    // Without this, styling would wait for ~2 seconds to be applied
+    // (workaround needed for HOT version 16 and above)
+    this.hotInstance.render()
   }
 
   async loadWithParameters() {

client/tsconfig.app.json

@@ -10,12 +10,6 @@
     "outDir": "./app",
     "types": []
   },
-  "files": [
-    "src/polyfills.ts",
-    "src/main.ts",
-    "src/app/app.d.ts"
-  ],
-  "include": [
-    "src/**/*.d.ts"
-  ]
-}
+  "files": ["src/polyfills.ts", "src/main.ts", "src/app/app.d.ts"],
+  "include": ["src/**/*.d.ts"]
+}

client/tsconfig.json

@@ -1,55 +1,37 @@
 /* To learn more about this file see: https://angular.io/config/tsconfig. */
 {
-    "compileOnSave": false,
-    "compilerOptions": {
-      "baseUrl": "",
-      "outDir": "dist",
-      "downlevelIteration": true,
-      "declaration": false,
-      "experimentalDecorators": true,
-      "allowSyntheticDefaultImports": true,
-      "strict": true,
-      "lib": [
-        "ES2022",
-        "dom"
-      ],
-      "skipLibCheck": true,
-      "module": "ES2022",
-      "importHelpers": true,
-      "moduleResolution": "node",
-      "sourceMap": true,
-      "resolveJsonModule": true,
-      "target": "ES2022",
-      "paths": {
-        "crypto": [
-          "./node_modules/crypto-browserify"
-        ],
-        "stream": [
-          "./node_modules/stream-browserify"
-        ],
-        "assert": [
-          "./node_modules/assert"
-        ],
-        "http": [
-          "./node_modules/stream-http"
-        ],
-        "https": [
-          "./node_modules/https-browserify"
-        ],
-        "os": [
-          "./node_modules/os-browserify"
-        ]
-      },
-      "useDefineForClassFields": false
+  "compileOnSave": false,
+  "compilerOptions": {
+    "baseUrl": "",
+    "outDir": "dist",
+    "downlevelIteration": true,
+    "declaration": false,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "lib": ["ES2022", "dom"],
+    "skipLibCheck": true,
+    "module": "ES2022",
+    "importHelpers": true,
+    "moduleResolution": "node",
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "target": "ES2022",
+    "paths": {
+      "crypto": ["./node_modules/crypto-browserify"],
+      "stream": ["./node_modules/stream-browserify"],
+      "assert": ["./node_modules/assert"],
+      "http": ["./node_modules/stream-http"],
+      "https": ["./node_modules/https-browserify"],
+      "os": ["./node_modules/os-browserify"]
     },
-    "angularCompilerOptions": {
-      "enableI18nLegacyMessageIdFormat": false,
-      "strictInjectionParameters": true,
-      "strictInputAccessModifiers": true,
-      "strictTemplates": true,
-    },
-    "exclude": [
-      "cypress/**/*.ts",
-      "cypress.config.ts"
-    ]
+    "useDefineForClassFields": false
+  },
+  "angularCompilerOptions": {
+    "enableI18nLegacyMessageIdFormat": false,
+    "strictInjectionParameters": true,
+    "strictInputAccessModifiers": true,
+    "strictTemplates": true
+  },
+  "exclude": ["cypress/**/*.ts", "cypress.config.ts"]
 }

client/tsconfig.spec.json

@@ -3,15 +3,8 @@
   "extends": "./tsconfig.json",
   "compilerOptions": {
     "outDir": "./out-tsc/spec",
-    "types": [
-      "jasmine"
-    ]
+    "types": ["jasmine"]
   },
-  "files": [
-    "src/polyfills.ts"
-  ],
-  "include": [
-    "src/**/*.spec.ts",
-    "src/**/*.d.ts"
-  ]
+  "files": ["src/polyfills.ts"],
+  "include": ["src/**/*.spec.ts", "src/**/*.d.ts"]
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "7.2.7",
+  "version": "7.6.0",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/.npmrc

@@ -0,0 +1,2 @@
+ignore-scripts=true
+save-exact=true

sas/deploy/makeDataServer.json

@@ -2,7 +2,7 @@
   "fromjs": [
     {
       "ADMIN": "DCDEFAULT",
-      "DCPATH": "/tmp/mihajlo/dcserverfrs"
+      "DCPATH": "/tmp/dcdata"
     }
   ]
 }

sas/package.json

@@ -28,7 +28,7 @@
   },
   "private": true,
   "dependencies": {
-    "@sasjs/cli": "^4.12.15",
-    "@sasjs/core": "^4.59.9"
+    "@sasjs/cli": "4.15.2",
+    "@sasjs/core": "4.63.0"
   }
 }

sas/sasjs/db/migrations/20260403_v7.6_release.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief migration script to move from v7.0 to v7.6 of data controller
+
+  OPTIONAL CHANGE - upload additional data as placeholders for modifying the
+  default email message
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+proc sql;
+insert into &dclib..mpe_config set
+    tx_from=%sysfunc(datetime())
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC_EMAIL"
+    ,var_name="SUBMITTED_TEMPLATE"
+    ,var_value='Dear user,'!!'0A'x!!'Please be advised that a change to table'
+      !!' &alert_lib..&alert_ds has been proposed by &from_user on the '
+      !!'&syshostname SAS server.'!!'0A'x!!'Reason provided: '
+      !!'%superq(SUBMITTED_TXT)'
+      !!'0A'x!!'This is an automated email by Data Controller for SAS. For '
+      !!'documentation, please visit https://docs.datacontroller.io'
+    ,var_active=1
+    ,var_desc='Template email, sent after submitting a change';
+insert into &dclib..mpe_config set
+    tx_from=%sysfunc(datetime())
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC_EMAIL"
+    ,var_name="APPROVED_TEMPLATE"
+    ,var_value='Dear user,'!!'0A'x!!'Please be advised that a change to table'
+      !!' &alert_lib..&alert_ds has been approved by &from_user on the '
+      !!'&syshostname SAS server.'!!'0A'x!!'This is an automated email by Data'
+      !!' Controller for SAS. For documentation, please visit '
+      !!'https://docs.datacontroller.io'
+    ,var_active=1
+    ,var_desc='Template email, sent after approving a change';
+insert into &dclib..mpe_config set
+    tx_from=%sysfunc(datetime())
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC_EMAIL"
+    ,var_name="REJECTED_TEMPLATE"
+    ,var_value='Dear user,'!!'0A'x!!'Please be advised that a change to table'
+      !!' &alert_lib..&alert_ds has been rejected by &from_user on the '
+      !!'&syshostname SAS server.'!!'0A'x!!'Reason provided: '
+      !!'%superq(REVIEW_REASON_TXT)'
+      !!'0A'x!!'This is an automated email by Data Controller for SAS. For '
+      !!'documentation, please visit https://docs.datacontroller.io'
+    ,var_active=1
+    ,var_desc='Template email, sent after rejecting a change';

sas/sasjs/macros/bitemporal_closeouts.sas

@@ -19,6 +19,8 @@
     given that BUS_FROM should be supplied in the PK.
   @param [in] tech_from= (tx_from_dttm) Technical FROM datetime variable.
     Required on BASE table only.
+  @param [in] AUDITFOLDER= (0) Unquoted path to a directory into which a copy of
+    the generated delete program will be written
 
   <h4> Global Variables </h4>
   @li `dc_dttmtfmt`
@@ -28,6 +30,9 @@
   @li mp_abort.sas
   @li mf_existvar.sas
   @li mf_getattrn.sas
+  @li mf_getengine.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
   @li mf_getuser.sas
   @li mf_getvartype.sas
   @li mp_lockanytable.sas
@@ -53,8 +58,7 @@
                               /* Should INCLUDE BUS_FROM field if relevant. */
       ,NOW=DEFINE
       ,FILTER=                /* supply a filter to limit the update */
-      ,outdest=               /* supply an unquoted filepath/filename.ext to get
-                                a text file containing the update statements */
+      ,AUDITFOLDER=0
       ,loadtype=
       ,loadtarget=YES      /* if <> YES will return without changing anything */
 );
@@ -70,13 +74,16 @@
   * perform basic checks
   */
 /* do tables exist? */
-%if not %sysfunc(exist(&base_lib..&base_dsn)) %then %do;
-  %mp_abort(msg=&base_lib..&base_dsn does not exist)
-%end;
-%else %if %sysfunc(exist(&append_lib..&append_dsn))=0
-          and %sysfunc(exist(&append_lib..&append_dsn,VIEW))=0 %then %do;
-  %mp_abort(msg=&append_lib..&append_dsn does not exist)
-%end;
+%mp_abort(
+  iftrue=(%sysfunc(exist(&base_lib..&base_dsn)) ne 1),
+  msg=&base_lib..&base_dsn does not exist
+)
+%mp_abort(
+  iftrue=(%sysfunc(exist(&append_lib..&append_dsn))=0
+          and %sysfunc(exist(&append_lib..&append_dsn,VIEW))=0 ),
+  msg=&append_lib..&append_dsn does not exist
+)
+
 /* do TX columns exist? */
 %if &loadtype ne UPDATE %then %do;
   %if not %mf_existvar(&base_lib..&base_dsn,&tech_from) %then %do;
@@ -111,61 +118,97 @@ data _null_;
   gap=intck('HOURS',now,datetime());
   call symputx('gap',gap,'l');
 run;
-%mf_abort(
+%mp_abort(
   iftrue=(&gap > 24),
   msg=NOW variable (&now) is not within a 24hr tolerance
 )
 
 /* have any warnings / errs occurred thus far?  If so, abort */
-%mf_abort(
+%mp_abort(
   iftrue=(&syscc>0),
   msg=Aborted due to SYSCC=&SYSCC status
 )
 
+/* set up folder */
+%local tmplib;%let tmplib=%mf_getuniquelibref();
+%if "&AUDITFOLDER"="0" %then %do;
+  filename tmp temp lrecl=10000;
+  libname &tmplib (work);
+%end;
+%else %do;
+  filename tmp "&AUDITFOLDER/deleterecords.sas" lrecl=10000;
+  libname &tmplib "&AUDITFOLDER";
+%end;
+
 /**
-  * Create closeout statements.  These are sent as individual SQL statements
+  * Create closeout statements.  If UPDATE approach and CAS engine, use the
+  * DeleteRows action (as regular SQL deletes are not supported).
+  * Otherwise, the deletions are sent as individual SQL statements
   * to ensure pass-through utilisation.  The update_cnt variable monitors
   * how many records were actually updated on the target table.
   */
-%local update_cnt;
+%local update_cnt etype;
 %let update_cnt=0;
-filename tmp temp;
-data _null_;
-  set ___closeout1;
-  file tmp;
-  if _n_=1 then put 'proc sql noprint;' ;
-  length string $32767.;
-  %if &loadtype=UPDATE %then %do;
-    put "delete from &base_lib..&base_dsn where 1";
-  %end;
-  %else %do;
-    now=symget('now');
-    put "update &base_lib..&base_dsn set &tech_to= " now @;
-    %if %mf_existvar(&base_lib..&base_dsn,PROCESSED_DTTM) %then %do;
-      put " ,PROCESSED_DTTM=" now @;
-    %end;
-    put " where " now " lt &tech_to ";
-  %end;
-  %do x=1 %to %sysfunc(countw(&PK));
-    %let var=%scan(&pk,&x,%str( ));
-    %if %mf_getvartype(&base_lib..&base_dsn,&var)=C %then %do;
-      /* use single quotes to avoid ampersand resolution in data */
-      string=" & &var='"!!trim(prxchange("s/'/''/",-1,&var))!!"'";
+%let etype=%mf_getengine(&base_lib);
+%put &=etype;
+
+%if &loadtype=UPDATE and &etype=CAS %then %do;
+  /* create temp table for deletions */
+  %local delds;%let delds=%mf_getuniquename(prefix=DC);
+  data casuser.&delds &tmplib..deleterecords;
+    set work.___closeout1;
+    keep &pk;
+  run;
+  /* build the proc */
+  data _null_;
+    file tmp;
+    put "/* libname approve '&AUDITFOLDER'; */";
+    put 'proc cas;table.deleteRows result=r/ table={' ;
+    put "  caslib='&base_lib',name='&base_dsn',where='1=1',";
+    put "  whereTable={caslib='CASUSER',name='&delds'}";
+    put "};";
+    put "call symputx('update_cnt',r.RowsDeleted);";
+    put "quit;";
+    put "data;set casuser.&delds;putlog (_all_)(=);run;";
+    put '%put &=update_cnt;';
+    put "proc sql;drop table CASUSER.&delds;";
+    stop;
+  run;
+
+%end;
+%else %do;
+  data _null_;
+    set ___closeout1;
+    file tmp;
+    if _n_=1 then put 'proc sql noprint;' ;
+    length string $32767.;
+    %if &loadtype=UPDATE %then %do;
+      put "delete from &base_lib..&base_dsn where 1";
     %end;
     %else %do;
-      string=cats(" & &var=",&var);
+      now=symget('now');
+      put "update &base_lib..&base_dsn set &tech_to= " now @;
+      %if %mf_existvar(&base_lib..&base_dsn,PROCESSED_DTTM) %then %do;
+        put " ,PROCESSED_DTTM=" now @;
+      %end;
+      put " where " now " lt &tech_to ";
     %end;
-    put string;
-  %end;
-  put "&filter ;";
-  put '%let update_cnt=%eval(&update_cnt+&sqlobs);%put update_cnt=&update_cnt;';
-run;
-
-data _null_;
-  infile tmp;
-  input;
-  putlog _infile_;
-run;
+    %do x=1 %to %sysfunc(countw(&PK));
+      %let var=%scan(&pk,&x,%str( ));
+      %if %mf_getvartype(&base_lib..&base_dsn,&var)=C %then %do;
+        /* use single quotes to avoid ampersand resolution in data */
+        string=" & &var='"!!trim(prxchange("s/'/''/",-1,&var))!!"'";
+      %end;
+      %else %do;
+        string=cats(" & &var=",&var);
+      %end;
+      put string;
+    %end;
+    put "&filter ;";
+    put '%let update_cnt=%eval(&update_cnt+&sqlobs);';
+    put '%put update_cnt=&update_cnt;';
+  run;
+%end;
 
 %if &loadtarget ne YES %then %return;
 

sas/sasjs/macros/bitemporal_dataloader.sas

@@ -26,8 +26,7 @@ NOTES:
 
   One cannot use BETWEEN
   One cannot use &xx_from LE [tstamp] LE &xx_from (equivalent to above).
-  Background:
-http://stackoverflow.com/questions/20005950/best-practice-for-scd-date-pairs-closing-opening-timestamps
+  Background: https://stackoverflow.com/questions/20005950
 
 Areas for optimisation
   - loading temporal history (currently experimental)
@@ -220,7 +219,8 @@ Areas for optimisation
 
 %local engine_type;
 %let engine_type=%mf_getengine(&base_lib);
-%if (&engine_type=REDSHIFT or &engine_type=POSTGRES) and %length(&CLOSE_VARS)>0
+%if %length(&CLOSE_VARS)>0 and (&engine_type=REDSHIFT or &engine_type=POSTGRES
+or &engine_type=SNOW or &engine_type=SASIOSNF)
 %then %do;
   %put NOTE:; %put NOTE-;%put NOTE-;%put NOTE-;
   %put NOTE- CLOSE_VARS functionality not yet supported in &engine_type;
@@ -512,6 +512,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
       ,NOW=&dbnow
       ,loadtarget=&loadtarget
       ,loadtype=&loadtype
+      ,AUDITFOLDER=&dc_staging_area/&ETLSOURCE
     )
   %end;
 %end;
@@ -574,6 +575,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
       ,NOW=&dbnow
       ,loadtarget=&loadtarget
       ,loadtype=&loadtype
+      ,AUDITFOLDER=&dc_staging_area/&ETLSOURCE
     )
   %end;
 %end;
@@ -634,7 +636,9 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
   %dc_assignlib(WRITE,&base_lib,passthru=myAlias)
   create table work.bitemp0_base as select * from connection to myAlias(
 %end;
-%else %if &engine_type=REDSHIFT or &engine_type=POSTGRES %then %do;
+%else %if &engine_type=REDSHIFT or &engine_type=POSTGRES or &engine_type=SNOW
+or &engine_type=SASIOSNF
+%then %do;
   /* grab schema */
   %let baselib_schema=%mf_getschema(&base_lib);
   %if &baselib_schema.X ne X %then %let baselib_schema=&baselib_schema..;
@@ -650,18 +654,24 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
       call symputx('redcnt',x,'l');
     run;
   %end;
-  /* cannot persist temp tables so must create a temporary permanent table */
-  %let temp_table=%mf_getuniquename(prefix=XDCTEMP);
+  %let temp_table=%upcase(%mf_getuniquename(prefix=XDCTEMP));
   %if &loadtype=BITEMPORAL or &loadtype=TXTEMPORAL %then
         %let base_table=(select * from &baselib_schema.&base_dsn
                         where timestamp &sqlnow < &tech_to );
   %else %let base_table=&baselib_schema.&base_dsn;
-  /* make empty table first - must clone & drop extra cols as autoload is bad */
+  /* make in-db empty table with PK + MD5 only */
   %dc_assignlib(WRITE,&base_lib,passthru=myAlias)
-
-  exec (create table &temp_table (like &baselib_schema.&base_dsn)) by myAlias;
-  %if &engine_type=REDSHIFT %then %do;
-    exec (alter table &temp_table alter sortkey none) by myAlias;
+  %if &engine_type=SNOW or &engine_type=SASIOSNF %then %do;
+    exec (create transient table &baselib_schema.&temp_table
+      like &baselib_schema.&base_dsn
+    ) by myAlias;
+  %end;
+  %else %do;
+    /* cannot persist temp tables so must create a temporary permanent table */
+    exec (create table &temp_table (like &baselib_schema.&base_dsn)) by myAlias;
+    %if &engine_type=REDSHIFT %then %do;
+      exec (alter table &temp_table alter sortkey none) by myAlias;
+    %end;
   %end;
   %local dropcols;
   %let dropcols=%mf_wordsinstr1butnotstr2(
@@ -676,9 +686,12 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
   exec (alter table &temp_table add column &md5_col varchar(32);) by myAlias;
   /* create view to strip formats and avoid warns in log */
   data work.vw_bitemp0/view=work.vw_bitemp0;
+    /* inherit remote length to handle byte expansion */
+    if 0 then set &base_lib..&temp_table(keep=&md5_col);
     set work.bitemp0_append(keep=&pk &md5_col);
     format _all_;
   run;
+
   proc append base=&base_lib..&temp_table
   %if &engine_type=REDSHIFT %then %do;
     (
@@ -731,6 +744,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
 
 
 %if &engine_type=OLEDB or &engine_type=REDSHIFT or &engine_type=POSTGRES
+or &engine_type=SNOW or &engine_type=SASIOSNF
 %then %do;
   ); proc sql; drop table &base_lib.."&temp_table"n;
 %end;
@@ -1185,7 +1199,7 @@ run;
 %else %if (&loadtype=BITEMPORAL or &loadtype=TXTEMPORAL or &loadtype=UPDATE)
 %then %do;
   data _null_;
-    putlog "&sysmacroname: &loadtype operation using &engine_type engine";
+    putlog "&sysmacroname: &loadtype operation using *&engine_type* engine";
   run;
   %local flexinow;
   proc sql;
@@ -1201,16 +1215,27 @@ run;
     %dc_assignlib(WRITE,&base_lib,passthru=myAlias)
     execute(
   %end;
-  %else %if &engine_type=REDSHIFT or &engine_type=POSTGRES %then %do;
-    %let innertable=%mf_getuniquename(prefix=XDCTEMP);
+  %else %if &engine_type=REDSHIFT or &engine_type=POSTGRES or &engine_type=SNOW
+  or &engine_type=SASIOSNF
+  %then %do;
+    %let innertable=%upcase(%mf_getuniquename(prefix=XDCTEMP));
     %let top_table=&baselib_schema.&base_dsn;
     %let flexinow=timestamp &SQLNOW;
     /* make empty table first - must clone & drop extra cols
       as autoload is bad */
     %dc_assignlib(WRITE,&base_lib,passthru=myAlias)
-    exec (create table &innertable (like &baselib_schema.&base_dsn)) by myAlias;
-    %if &engine_type=REDSHIFT %then %do;
-      exec (alter table &innertable alter sortkey none) by myAlias;
+    %if &engine_type=SNOW or &engine_type=SASIOSNF %then %do;
+      exec (create transient table &baselib_schema.&innertable
+        like &baselib_schema.&base_dsn
+      ) by myAlias;
+    %end;
+    %else %do;
+      exec (create table &innertable
+        (like &baselib_schema.&base_dsn)
+      ) by myAlias;
+      %if &engine_type=REDSHIFT %then %do;
+        exec (alter table &innertable alter sortkey none) by myAlias;
+      %end;
     %end;
     %let dropcols=%mf_wordsinstr1butnotstr2(
                     str1=%upcase(%mf_getvarlist(&basecopy))
@@ -1238,6 +1263,7 @@ run;
     execute(
   %end;
   %else %do;
+    %put Not using passthrough for *&engine_type* engine;
     %let innertable=bitemp5d_subquery;
     %let top_table=&base_lib..&base_dsn;
     %let flexinow=&now;
@@ -1290,6 +1316,7 @@ run;
   1=1);
 
   %if &engine_type=OLEDB or &engine_type=REDSHIFT or &engine_type=POSTGRES
+  or &engine_type=SNOW or &engine_type=SASIOSNF
   %then %do;
     ) by myAlias;
     execute (drop table &baselib_schema.&innertable) by myAlias;

sas/sasjs/macros/mpe_alerts.sas

@@ -127,6 +127,11 @@ run;
 filename __out email (&emails)
   subject="Table &alert_lib..&alert_ds has been &alert_event";
 
+data work.alertmessage;
+  set &mpelib..mpe_config;
+  where &dc_dttmtfmt. lt tx_to;
+  where also var_scope='DC_EMAIL' and var_name="&alert_event._TEMPLATE";
+run;
 %local SUBMITTED_TXT;
 %if &alert_event=SUBMITTED %then %do;
   data _null_;
@@ -136,30 +141,54 @@ filename __out email (&emails)
   run;
   data _null_;
     File __out lrecl=32000;
+    length txt $2048;
+  %if %mf_getattrn(alertmessage,NLOBS)=0 %then %do;
     put 'Dear user,';
     put ' ';
     put "Please be advised that a change to table &alert_lib..&alert_ds has "
-        "been proposed by &from_user on the '&syshostname' SAS server.";
+        "been proposed by &from_user on the &syshostname SAS server.";
     put " ";
-    length txt $2048;
     txt=symget('SUBMITTED_TXT');
     put "Reason provided: " txt;
     put " ";
     put "This is an automated email by Data Controller for SAS. For "
       "documentation, please visit https://docs.datacontroller.io";
+  %end;
+  %else %do;
+    /* take template from config table */
+    set work.alertmessage;
+    cnt=countw(var_value,'0A'x);
+    do i=1 to cnt;
+      txt=resolve(scan(var_value,i,'0A'x));
+      put txt /;
+    end;
+  %end;
   run;
 %end;
 %else %if &alert_event=APPROVED %then %do;
   /* there is no approval message */
   data _null_;
     File __out lrecl=32000;
+    length txt $2048;
+  %if %mf_getattrn(alertmessage,NLOBS)=0 %then %do;
+    /* fallback message */
     put 'Dear user,';
     put ' ';
     put "Please be advised that a change to table &alert_lib..&alert_ds has "
-        "been approved by &from_user on the '&syshostname' SAS server.";
+        "been approved by &from_user on the &syshostname SAS server.";
     put " ";
     put "This is an automated email by Data Controller for SAS. For "
       "documentation, please visit https://docs.datacontroller.io";
+  %end;
+  %else %do;
+    /* take template from config table */
+    set work.alertmessage;
+    cnt=countw(var_value,'0A'x);
+    do i=1 to cnt;
+      txt=resolve(scan(var_value,i,'0A'x));
+      put txt /;
+    end;
+  %end;
   run;
 %end;
 %else %if &alert_event=REJECTED %then %do;
@@ -170,17 +199,29 @@ filename __out email (&emails)
   run;
   data _null_;
     File __out lrecl=32000;
+    length txt $2048;
+  %if %mf_getattrn(alertmessage,NLOBS)=0 %then %do;
+    /* fallback message */
     put 'Dear user,';
     put ' ';
     put "Please be advised that a change to table &alert_lib..&alert_ds has "
-        "been rejected by &from_user on the '&syshostname' SAS server.";
+        "been rejected by &from_user on the &syshostname SAS server.";
     put " ";
-    length txt $2048;
     txt=symget('REVIEW_REASON_TXT');
     put "Reason provided: " txt;
     put " ";
     put "This is an automated email by Data Controller for SAS. For "
       "documentation, please visit https://docs.datacontroller.io";
+  %end;
+  %else %do;
+    /* take template from config table */
+    set work.alertmessage;
+    cnt=countw(var_value,'0A'x);
+    do i=1 to cnt;
+      txt=resolve(scan(var_value,i,'0A'x));
+      put txt /;
+    end;
+  %end;
   run;
 %end;
 

sas/sasjs/macros/mpe_makedata.sas

@@ -201,6 +201,44 @@ insert into &lib..mpe_config set
     ,var_value=' '
     ,var_active=1
     ,var_desc='Activation Key';
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC_EMAIL"
+    ,var_name="SUBMITTED_TEMPLATE"
+    ,var_value='Dear user,'!!'0A'x!!'Please be advised that a change to table'
+      !!' &alert_lib..&alert_ds has been proposed by &from_user on the '
+      !!'&syshostname SAS server.'!!'0A'x!!'Reason provided: '
+      !!'%superq(SUBMITTED_TXT)'
+      !!'0A'x!!'This is an automated email by Data Controller for SAS. For '
+      !!'documentation, please visit https://docs.datacontroller.io'
+    ,var_active=1
+    ,var_desc='Template email, sent after submitting a change';
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC_EMAIL"
+    ,var_name="APPROVED_TEMPLATE"
+    ,var_value='Dear user,'!!'0A'x!!'Please be advised that a change to table'
+      !!' &alert_lib..&alert_ds has been approved by &from_user on the '
+      !!'&syshostname SAS server.'!!'0A'x!!'This is an automated email by Data'
+      !!' Controller for SAS. For documentation, please visit '
+      !!'https://docs.datacontroller.io'
+    ,var_active=1
+    ,var_desc='Template email, sent after approving a change';
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC_EMAIL"
+    ,var_name="REJECTED_TEMPLATE"
+    ,var_value='Dear user,'!!'0A'x!!'Please be advised that a change to table'
+      !!' &alert_lib..&alert_ds has been rejected by &from_user on the '
+      !!'&syshostname SAS server.'!!'0A'x!!'Reason provided: '
+      !!'%superq(REVIEW_REASON_TXT)'
+      !!'0A'x!!'This is an automated email by Data Controller for SAS. For '
+      !!'documentation, please visit https://docs.datacontroller.io'
+    ,var_active=1
+    ,var_desc='Template email, sent after rejecting a change';
 
 
 insert into &lib..mpe_datadictionary set
@@ -213,7 +251,6 @@ insert into &lib..mpe_datadictionary set
     ,DD_RESPONSIBLE="&sysuserid"
     ,DD_SENSITIVITY="Low"
     ,tx_to='31DEC5999:23:59:59'dt;
-
 insert into &lib..mpe_datadictionary set
     tx_from=0
     ,DD_TYPE='TABLE'
@@ -224,7 +261,6 @@ insert into &lib..mpe_datadictionary set
     ,DD_RESPONSIBLE="&sysuserid"
     ,DD_SENSITIVITY="Low"
     ,tx_to='31DEC5999:23:59:59'dt;
-
 insert into &lib..mpe_datadictionary set
     tx_from=0
     ,DD_TYPE='COLUMN'
@@ -235,7 +271,6 @@ insert into &lib..mpe_datadictionary set
     ,DD_RESPONSIBLE="&sysuserid"
     ,DD_SENSITIVITY="Low"
     ,tx_to='31DEC5999:23:59:59'dt;
-
 insert into &lib..mpe_datadictionary set
     tx_from=0
     ,DD_TYPE='DIRECTORY'
@@ -1866,6 +1901,15 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value='0'
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&lib"
+    ,base_ds="MPE_ROW_LEVEL_SECURITY"
+    ,base_col="RLS_SUBGROUP_ID"
+    ,rule_type='NOTNULL'
+    ,rule_value='0'
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"

sas/sasjs/macros/mpe_refreshcatalogs.sas

@@ -1,5 +1,5 @@
 /**
-  @file mpe_refreshtables.sas
+  @file
   @brief Refreshes the data catalog
   @details Assumes library is already assigned.
   Usage:
@@ -11,6 +11,7 @@
 
   @version 9.3
   @author 4GL Apps Ltd
+
 **/
 
 %macro mpe_refreshcatalogs(lib,cat=#all);

sas/sasjs/sasjsconfig.json

@@ -124,7 +124,8 @@
         "serviceFolders": [
           "sasjs/targets/viya/services_viya/viya_users",
           "sasjs/targets/viya/services_viya/admin",
-          "sasjs/targets/viya/services_viya/public"
+          "sasjs/targets/viya/services_viya/public",
+          "sasjs/targets/viya/services_viya/validations"
         ],
         "initProgram": "sasjs/utils/serviceinitviya.sas",
         "termProgram": "",
@@ -202,7 +203,44 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas9.4gl.io",
+      "serverUrl": "https://sas.4gl.io",
+      "serverType": "SASJS",
+      "httpsAgentOptions": {
+        "rejectUnauthorized": false,
+        "allowInsecureRequests": true
+      },
+      "appLoc": "/Public/app/dc",
+      "deployConfig": {
+        "deployServicePack": true,
+        "deployScripts": []
+      },
+      "macroFolders": [
+        "sasjs/targets/server/macros_server"
+      ],
+      "programFolders": [
+        "sasjs/db/datactrl"
+      ],
+      "serviceConfig": {
+        "serviceFolders": [
+          "sasjs/targets/server/services_server/admin",
+          "sasjs/targets/server/services_server/usernav"
+        ],
+        "initProgram": "sasjs/utils/serviceinitserver.sas",
+        "termProgram": "",
+        "macroVars": {}
+      },
+      "streamConfig": {
+        "streamWeb": true,
+        "streamWebFolder": "web",
+        "webSourcePath": "../client/dist",
+        "streamServiceName": "DataController",
+        "streamLogo": "favicon.ico",
+        "assetPaths": []
+      }
+    },
+    {
+      "name": "local",
+      "serverUrl": "http://localhost:5000",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "rejectUnauthorized": false,

sas/sasjs/services/admin/demodata.sas

@@ -0,0 +1,363 @@
+/**
+  @file
+  @brief Creates demo tables and associated config
+  @details Can be removed in prod installs.
+
+  To activate this job, add the following to SETTINGS:
+
+      %let demolib=PUBLIC;
+      libname &demolib "%sysfunc(pathname(&dc_libref))/&demolib";
+      %let joblib=HOOKLIB;
+      libname &joblib "%sysfunc(pathname(&dc_libref))/&joblib";
+      %let dcdemoflag=1;
+
+  Note that this will:
+    * REPLACE any tables named CARS_EXT or COUNTRIES in the PUBLIC library
+    * REPLACE all DC config for libraries named PUBLIC
+    * CREATE a folder called "demo" in the DC Apploc
+    * CREATE two BASE libraries (HOOKLIB & PUBLIC) in the DC (physical) folder
+
+  <h4> SAS Macros </h4>
+  @li mpeinit.sas
+  @li mf_getengine.sas
+  @li mf_getuser.sas
+  @li mf_increment.sas
+  @li mf_nobs.sas
+  @li mf_uid.sas
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+  @li mp_replace.sas
+  @li mx_createjob.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let dcdemoflag=0;
+options dlcreatedir;
+%global joblib demolib;
+%mpeinit()
+
+%mp_abort(iftrue= (&dcdemoflag ne 1)
+  ,mac=&_program
+  ,msg=%str(Job not configured.  See comments in the code.)
+)
+
+data work.cars_ext(index=(carspk=(make model PRODUCTIONDATE) /unique));
+  attrib
+    MAKE                             length= $13
+    MODEL                            length= $40
+    TYPE                             length= $8
+    ORIGIN                           length= $6
+    COUNTRY                          length= $30
+    POTENTIALBUY                     length= $6
+    COMMENT                          length= $30
+    NOTES                            length= $30
+    CHECKBOXVAR                      length= $3
+    PRODUCTIONDATE                   length= 8      format=DATE9.
+  ;
+  set sashelp.cars;
+  retain comment 'n/a';
+  if mod(ceil(ranuni(1)*100),3)=0 then notes=catx(' ',make,type);
+  call missing(notes);
+  /* random / reproducible date between 1960 and 2020 */
+  PRODUCTIONDATE=ceil(ranuni(1)*365*60);
+  if mod(ceil(ranuni(1)*1000),2)=0 then CHECKBOXVAR='YES';
+  else CHECKBOXVAR='No';
+  if mod(ceil(ranuni(1)*1000),3)=0 then POTENTIALBUY='Maybe';
+  else if mod(ceil(ranuni(1)*1000),2)=0 then POTENTIALBUY='Yes';
+  else POTENTIALBUY='No';
+  make=cats(make);
+  model=cats(model);
+
+  array cntrs (4) $ 60 _temporary_ ( "Germany" "France" "Poland" "Italy");
+  if origin='USA' then country='USA';
+  else if origin='Asia' then do;
+    if mod(_n_,2)=0 then country='Japan';
+    else country='Korea';
+  end;
+  else COUNTRY = cntrs[ ceil(dim(cntrs) * ranuni(1))];
+
+  *put (_all_)(=);
+run;
+
+data work.COUNTRIES(index=(countriespk=(origin country) /unique));
+  attrib
+    ORIGIN                           length= $6
+    COUNTRY                          length= $30
+  ;
+  infile cards dsd;
+  input
+    ORIGIN                           :$char.
+    COUNTRY                          :$char.
+  ;
+datalines4;
+Europe,Germany
+Europe,France
+Europe,Poland
+Europe,Italy
+USA,USA
+Asia,Japan
+Asia,Korea
+;;;;
+run;
+
+data work.jobdata;
+  length message job $100;
+  call missing(of _all_);
+  stop;
+run;
+
+%let engine_type=%mf_getengine(&demolib);
+%put &=engine_type;
+%if &engine_type=CAS %then %do;
+  proc cas;
+    table.tableExists result=r / name="CARS_EXT" caslib="PUBLIC";
+    if r.exists then
+      table.dropTable / name="CARS_EXT" caslib="PUBLIC" quiet=TRUE;
+
+    table.tableExists result=r2 / name="COUNTRIES" caslib="PUBLIC";
+    if r2.exists then
+      table.dropTable / name="COUNTRIES" caslib="PUBLIC" quiet=TRUE;
+
+    table.tableExists result=r2 / name="MPE_AUDIT" caslib="PUBLIC";
+    if r2.exists then
+      table.dropTable / name="MPE_AUDIT" caslib="PUBLIC" quiet=TRUE;
+  quit;
+  proc casutil;
+    load data=work.CARS_EXT outcaslib="PUBLIC" casout="CARS_EXT" promote;
+    load data=work.COUNTRIES outcaslib="PUBLIC" casout="COUNTRIES" promote;
+    load data=&dc_libref..MPE_AUDIT
+      outcaslib="PUBLIC" casout="MPE_AUDIT" promote;
+  run;
+  data &joblib..JOBDATA; set work.JOBDATA;run;
+%end;
+%else %do;
+  options replace;
+  data &demolib..CARS_EXT; set work.cars_ext;
+  data &demolib..COUNTRIES; set work.countries;
+  data &joblib..JOBDATA; set work.JOBDATA;run;
+%end;
+
+%let apploc=%mf_getapploc(&_program);
+%let demolib=%upcase(&demolib);
+proc sql;
+delete from &dc_libref..mpe_tables
+  where libref="&demolib" and dsn in ('CARS_EXT','COUNTRIES');
+data append;
+  if 0 then set &dc_libref..mpe_tables;
+  TX_FROM=0;
+  TX_TO='31DEC9999:23:59:59'dt;
+  LIBREF="&demolib";
+  LOADTYPE='UPDATE';
+  NUM_OF_APPROVALS_REQUIRED=1;
+  PRE_EDIT_HOOK="&apploc/demo/PREEDIT";
+  POST_EDIT_HOOK="&apploc/demo/POSTEDIT";
+  PRE_APPROVE_HOOK="&apploc/demo/PREAPPROVE";
+  POST_APPROVE_HOOK="&apploc/demo/POSTAPPROVE";
+  DSN='CARS_EXT'; BUSKEY='MAKE MODEL PRODUCTIONDATE'; output;
+  DSN='COUNTRIES'; BUSKEY='ORIGIN COUNTRY'; output;
+run;
+proc append base=&dc_libref..MPE_TABLES data=&syslast;
+run;
+
+/* hard coded values for CHECKBOXVAR */
+%let rk=1e6;
+proc sql noprint;
+delete from &dc_libref..mpe_selectbox
+  where select_lib="&demolib"
+    and select_ds in ('CARS_EXT');
+select max(selectbox_rk) into: rk
+  from &dc_libref..mpe_selectbox;
+
+insert into &dc_libref..mpe_selectbox set
+  selectbox_rk=%mf_increment(rk)
+  ,ver_from_dttm=0
+  ,select_lib="&demolib"
+  ,select_ds="CARS_EXT"
+  ,base_column="CHECKBOXVAR"
+  ,selectbox_value='Yes'
+  ,selectbox_order=1
+  ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dc_libref..mpe_selectbox set
+  selectbox_rk=%mf_increment(rk)
+  ,ver_from_dttm=0
+  ,select_lib="&demolib"
+  ,select_ds="CARS_EXT"
+  ,base_column="CHECKBOXVAR"
+  ,selectbox_value='No'
+  ,selectbox_order=2
+  ,ver_to_dttm='31DEC5999:23:59:59'dt;
+
+/* Table driven values */
+delete from &dc_libref..MPE_VALIDATIONS
+  where base_lib="&demolib" and base_ds="CARS_EXT";
+
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="MAKE"
+    ,rule_type='HARDSELECT'
+    ,rule_value="SASHELP.CARS.MAKE"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="MODEL"
+    ,rule_type='HARDSELECT'
+    ,rule_value="SASHELP.CARS.MODEL"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="TYPE"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="SASHELP.CARS.TYPE"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="POTENTIALBUY"
+    ,rule_type='SOFTSELECT'
+    ,rule_value="&demolib..CARS_EXT.POTENTIALBUY"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="COMMENT"
+    ,rule_type='NOTNULL'
+    ,rule_value="n/a"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="ENGINESIZE"
+    ,rule_type='MINVAL'
+    ,rule_value="1.3"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="ENGINESIZE"
+    ,rule_type='MAXVAL'
+    ,rule_value="8.3"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program
+  ,msg=%str(syscc=syscc=&syscc during param configuration)
+)
+
+/* programmatic values for COUNTRY (Dynamic Dropdown) */
+filename vldtr temp;
+data _null_;
+  file vldtr ;
+  put 'proc sql;';
+  put 'create table work.vals as';
+  put '  select distinct ORIGIN as display_value,';
+  put '  ORIGIN as raw_value';
+  put "  from &demolib..COUNTRIES";
+  put '  order by 1;';
+  put 'data work.DYNAMIC_VALUES; set work.vals;display_index=_n_;run;';
+  put ' ';
+  put 'proc sql;';
+  put 'create table work.dev as ';
+  put '  select a.display_index,b.country as display_value';
+  put '  from work.DYNAMIC_VALUES as a';
+  put "  left join &demolib..countries as b";
+  put "  on a.raw_value=b.origin";
+  put '  order by display_index;';
+  put 'data work.DYNAMIC_EXTENDED_VALUES; set work.dev;by display_index;';
+  put '  EXTRA_COL_NAME="COUNTRY";';
+  put '  DISPLAY_TYPE="C";';
+  put '  RAW_VALUE_CHAR=DISPLAY_VALUE;';
+  put '  RAW_VALUE_NUM=.;';
+  put '  if first.display_index then forced_value=1;';
+  put 'run;';
+run;
+%mx_createjob(path=&apploc/demo
+  ,name=origin,code=vldtr
+)
+proc sql;
+insert into &dc_libref..MPE_VALIDATIONS set
+    tx_from=0
+    ,base_lib="&demolib"
+    ,base_ds="CARS_EXT"
+    ,base_col="ORIGIN"
+    ,rule_type='HARDSELECT_HOOK'
+    ,rule_value="&apploc/demo/origin"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+
+/* PRE_EDIT JOB */
+%let fvar=XXXXXXXXXXX; /* cannot substitute macvars in parmcards */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds (%mf_nobs(work.out) obs) fetched for editing %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f1 temp;
+%mp_binarycopy(inref=ft15f001, outref=f1)
+%mp_replace("%sysfunc(pathname(f1))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=PREEDIT,code=f1)
+filename ft15f001 clear;
+
+/* POST EDIT JOB */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds staged %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f2 temp;
+%mp_binarycopy(inref=ft15f001, outref=f2)
+%mp_replace("%sysfunc(pathname(f2))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=POSTEDIT,code=f2)
+filename ft15f001 clear;
+
+/* PRE APPROVE JOB */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds (%mf_nobs(work.staging_ds) obs) under review by %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f3 temp;
+%mp_binarycopy(inref=ft15f001, outref=f3)
+%mp_replace("%sysfunc(pathname(f3))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=PREAPPROVE,code=f3)
+filename ft15f001 clear;
+
+/* POST APPROVE JOB */
+filename ft15f001 temp;
+parmcards4;
+  proc sql;
+  insert into XXXXXXXXXXX.JOBDATA values(
+    "&orig_libds (%mf_nobs(work.staging_ds) obs) approved by %trim(
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
+;;;;
+filename f4 temp;
+%mp_binarycopy(inref=ft15f001, outref=f4)
+%mp_replace("%sysfunc(pathname(f4))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=POSTAPPROVE,code=f4)
+filename ft15f001 clear;

sas/sasjs/services/admin/exportconfig.sas

@@ -9,10 +9,12 @@
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
   @li mf_nobs.sas
-  @li mp_ds2cards.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
+  @li mp_ds2cards.sas
+  @li mp_ds2csv.sas
   @li mp_streamfile.sas
+  @li mp_validatecol.sas
 
   @author 4GL Apps Ltd
   @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
@@ -21,23 +23,33 @@
 
 **/
 
+%global dclib islib newlib;
 %mpeinit()
+
+data _null_;
+  newlib=coalescec(symget('dclib'),"&mpelib");
+  %mp_validatecol(newlib,ISLIB,islib)
+  call symputx('islib',islib);
+  call symputx('newlib',upcase(newlib));
+  putlog (_all_)(=);
+run;
+
+%mp_abort(iftrue= (&islib ne 1)
+  ,mac=&_program
+  ,msg=%nrstr(&newlib is not a valid libref)
+)
+
 %let work=%sysfunc(pathname(work));
-
-/* excel does not work in all envs */
-%let mime=application/vnd.ms-excel;
-%let dbms=EXCEL;
-
 %let mime=application/csv;
 %let dbms=CSV;
 %let ext=csv;
 
 %macro conditional_export(ds);
 %if %mf_nobs(&ds)>0 %then %do;
-  PROC EXPORT DATA= &ds OUTFILE= "&work/&ds..&ext"
-    DBMS=&dbms REPLACE;
-  RUN;
-  ods package(ProdOutput) add file="&work/&ds..&ext" mimetype="&mime";
+  /* cannot use PROC EXPORT as we need to wrap all csv char values in quotes */
+  /* cannot use excel as it does not work consistently in all SAS envs */
+  %mp_ds2csv(&ds,outfile="&work/&newlib..&ds..csv",headerformat=NAME)
+  ods package(ProdOutput) add file="&work/&newlib..&ds..&ext" mimetype="&mime";
 %end;
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=&_program
@@ -52,6 +64,7 @@ data MPE_ALERTS;
   set &mpelib..MPE_ALERTS;
   where &dc_dttmtfmt. le tx_to;
   drop tx_: ;
+  if alert_lib="&mpelib" then alert_lib="&newlib";
 run;
 %conditional_export(MPE_ALERTS)
 
@@ -61,6 +74,7 @@ data MPE_COLUMN_LEVEL_SECURITY;
   where &dc_dttmtfmt. le tx_to;
   where also CLS_LIBREF ne "&mpelib";
   drop tx_: ;
+  CLS_LIBREF="&newlib";
 run;
 %conditional_export(MPE_COLUMN_LEVEL_SECURITY)
 
@@ -68,6 +82,7 @@ data MPE_CONFIG;
   set &mpelib..MPE_CONFIG;
   where &dc_dttmtfmt. le tx_to;
   drop tx_: ;
+  if var_name='DC_MACROS' then var_value=tranwrd(var_value,"&mpelib","&newlib");
 run;
 %conditional_export(MPE_CONFIG)
 
@@ -93,6 +108,7 @@ data MPE_EXCEL_CONFIG;
   set &mpelib..MPE_EXCEL_CONFIG;
   where &dc_dttmtfmt. le tx_to;
   drop tx_: ;
+  if xl_libref="&mpelib" then xl_libref="&newlib";
 run;
 %conditional_export(MPE_EXCEL_CONFIG)
 
@@ -107,6 +123,7 @@ data MPE_ROW_LEVEL_SECURITY;
   set &mpelib..MPE_ROW_LEVEL_SECURITY;
   where &dc_dttmtfmt. le tx_to;
   drop tx_: ;
+  if rls_libref="&mpelib" then rls_libref="&newlib";
 run;
 %conditional_export(MPE_ROW_LEVEL_SECURITY)
 
@@ -115,6 +132,7 @@ data MPE_SECURITY;
   set &mpelib..MPE_SECURITY;
   where &dc_dttmtfmt. le TX_TO;
   drop tx_: ;
+  if libref="&mpelib" then libref="&newlib";
 run;
 %conditional_export(MPE_SECURITY)
 
@@ -142,6 +160,23 @@ data MPE_VALIDATIONS;
 run;
 %conditional_export(MPE_VALIDATIONS)
 
+data MPE_XLMAP_INFO;
+  set &mpelib..MPE_XLMAP_INFO;
+  where &dc_dttmtfmt. le TX_TO;
+  drop tx_: ;
+  if XLMAP_TARGETLIBDS=:"&mpelib.." then
+    XLMAP_TARGETLIBDS=tranwrd(XLMAP_TARGETLIBDS,"&mpelib..","&newlib..");
+run;
+%conditional_export(MPE_XLMAP_INFO)
+
+data MPE_XLMAP_RULES;
+  set &mpelib..MPE_XLMAP_RULES;
+  where &dc_dttmtfmt. le TX_TO;
+  drop tx_: ;
+run;
+%conditional_export(MPE_XLMAP_RULES)
+
+
 /* finish up zip file */
 ods package(ProdOutput) publish archive properties
         (archive_name="DCBACKUP.zip" archive_path="&work");

sas/sasjs/services/approvers/rejection.sas

@@ -84,7 +84,8 @@ data work.reject;
   REVIEW_STATUS_ID="REJECTED";
   REVIEWED_BY_NM="&user";
   REVIEWED_ON_DTTM=&now;
-  REVIEW_REASON_TXT=symget('STP_REASON');
+  /* sanitise message to prevent code injection */
+  REVIEW_REASON_TXT=compress(symget('STP_REASON'), '&%;');
 run;
 
 %mp_lockanytable(LOCK,

sas/sasjs/services/editors/loadfile.sas

@@ -286,7 +286,17 @@ options mprint;
 )
 
 filename outref "&dir/BKP_&xlsname";
+
+data _null_;
+  if "&xlsref" ne "0" then do;
+    rc=fcopy("&xlsref","outref");
+  end;
+run;
+
+/**
+  * if running 9.3 or older, delete step above and enable macro below
 %mp_binarycopy(iftrue=("&xlsref" ne "0"),inref=&xlsref,outref=outref)
+  */
 
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=&sysmacroname

sas/sasjs/services/editors/restore.sas

@@ -5,9 +5,12 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mcf_getfmttype.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_applyformats.sas
   @li mp_ds2csv.sas
+  @li mp_getcols.sas
   @li mp_stripdiffs.sas
   @li mpeinit.sas
   @li mpe_checkrestore.sas
@@ -112,8 +115,26 @@ data approve.jsdset;
   length _____DELETE__THIS__RECORD_____ $3;
   if 0 then call missing(_____DELETE__THIS__RECORD_____);
   set work.mp_stripdiffs;
+  format _all_;
 run;
 
+/* find all of the date / datetime / time vars */
+%mcf_getfmttype(wrap=YES)
+%mp_getcols(&tgtds,outds=work.cols)
+
+data work.applydtfmts;
+  set work.cols;
+  lib="APPROVE";
+  ds="JSDSET";
+  var=name;
+  fmt=coalescec(format,'0');
+  fmttype=mcf_getfmttype(fmt);
+  if fmttype in ('DATE','DATETIME','TIME');
+  keep lib ds var fmt;
+run;
+%mp_applyformats(work.applydtfmts)
+
+
 /* export to csv */
 %mp_ds2csv(approve.jsdset
   ,dlm=COMMA

sas/sasjs/services/editors/stagedata.sas

@@ -51,6 +51,8 @@
 data _null_;
   set work.sascontroltable;
   call symputx('action',action);
+  /* sanitise message to prevent code injection */
+  message=compress(message, '&%;');
   call symputx('message',message);
   libds=upcase(libds);
   call symputx('orig_libds',libds);

sas/sasjs/services/public/getrawdata.sas

@@ -16,6 +16,7 @@
   @li mf_existfeature.sas
   @li dc_assignlib.sas
   @li mp_ds2cards.sas
+  @li mp_ds2csv.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_cntlout.sas
@@ -117,10 +118,8 @@ options validvarname=upcase;
   /* cannot proc export excel if PC Files is not licensed */
   %then %do;
     %let outfile=%sysfunc(pathname(work))/&table..csv;
-    PROC EXPORT DATA= staged
-        OUTFILE= "&outfile"
-        DBMS=csv REPLACE;
-    RUN;
+    /* cannot use PROC EXPORT as we need to wrap all char values in quotes */
+    %mp_ds2csv(work.staged,outfile="&outfile",headerformat=NAME)
     %let ext=csv;
     %let mimetype=csv;
   %end;

sas/sasjs/services/validations/tables_all.sas

@@ -54,7 +54,7 @@
 %let vartgtlib=%mf_getuniquename();
 %let var_is_lib=%mf_getuniquename();
 data _null_;
-  length &varlibds $41 &vartgtlib $8 libref $8 rls_libref $8;
+  length &varlibds $41 &vartgtlib $8 libref $8 rls_libref cls_libref $8;
   if _n_=1 then call missing(of _all_);
   set work.source_row;
   &varlibds=upcase(symget('libds'));

sas/sasjs/targets/viya/macros_viya/dc_createdataset.sas

@@ -1,17 +1,14 @@
 /**
   @file dc_createdataset.sas
 
-  <h4> SAS Macros </h4>
-
-  @version 9.3
   @author 4GL Apps Ltd
   @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
     and may not be re-distributed or re-sold without the express permission of
     4GL Apps Ltd.
 **/
 
-%macro dc_createdataset(libds=mm_getlibs);
-data viewdata;
+%macro dc_createdataset(libds=mm_getlibs,outds=viewdata);
+data &outds;
   var1='Table';
   var2="&libds";
   var3="does not exist!";

sas/sasjs/targets/viya/services_viya/validations/sas_groups.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief validating the mpe_security.sas_group column
+  @details The input table is simply one row from the target table in table
+  called "work.source_row".
+
+  Available macro variables:
+    @li LIBDS - The library.dataset being filtered
+    @li VARIABLE_NM - The column being filtered
+
+
+  <h4> Service Outputs </h4>
+  The values provided below are generic samples - we encourage you to replace
+  these with realistic values in your own deployments.
+
+  <h5>DYNAMIC_VALUES</h5>
+  The RAW_VALUE column may be charactor or numeric.  If DISPLAY_INDEX is not
+  provided, it is added automatically.
+
+  |DISPLAY_INDEX:best.|DISPLAY_VALUE:$|RAW_VALUE|
+  |---|---|---|
+  |1|$77.43|77.43|
+  |2|$88.43|88.43|
+
+  <h5>DYNAMIC_EXTENDED_VALUES</h5>
+  This table is optional.  If provided, it will map the DISPLAY_INDEX from the
+  DYNAMIC_VALUES table to additional column/value pairs, that will be used to
+  populate dropdowns for _other_ cells in the _same_ row.
+
+  Should be used sparingly!  The use of large tables here can slow down the
+  browser.
+
+  |DISPLAY_INDEX:best.|EXTRA_COL_NAME:$32.|DISPLAY_VALUE:$|DISPLAY_TYPE:$1.|RAW_VALUE_NUM|RAW_VALUE_CHAR:$5000|
+  |---|---|---|
+  |1|DISCOUNT_RT|"50%"|N|0.5||
+  |1|DISCOUNT_RT|"40%"|N|0.4||
+  |1|DISCOUNT_RT|"30%"|N|0.3||
+  |1|CURRENCY_SYMBOL|"GBP"|C||"GBP"|
+  |1|CURRENCY_SYMBOL|"RSD"|C||"RSD"|
+  |2|DISCOUNT_RT|"50%"|N|0.5||
+  |2|DISCOUNT_RT|"40%"|N|0.4||
+  |2|CURRENCY_SYMBOL|"EUR"|C||"EUR"|
+  |2|CURRENCY_SYMBOL|"HKD"|C||"HKD"|
+
+
+
+  <h4> SAS Macros </h4>
+  @li dc_getgroups.sas
+
+
+**/
+
+
+%dc_getgroups(outds=groups)
+
+proc sql;
+create table work.DYNAMIC_VALUES as
+  select distinct groupname as display_value,
+    groupuri as raw_value
+  from work.groups
+  order by 1;