chore(release): 7.18.0 [skip ci]

# [7.18.0](https://git.datacontroller.io/dc/dc/compare/v7.17.0...v7.18.0) (2026-10-19)

### Bug Fixes

* **deps:** bump @sasjs/core to 5.4.0 ([19fb032](19fb032f0d))
* **deps:** pin @sasjs/utils and drop the unused jsrsasign ([05d2d1e](05d2d1e08e))
* **editor:** own the delete-record column client-side ([56025cc](56025cc052))
* **editor:** surface the reserved-column refusal on every getdata path ([b26f557](b26f5571b6))
* **filters:** do not turn a fractional temporal value into a literal ([f710d87](f710d871ff))
* **filters:** fix the dynamic where clause deref and map its temporal values ([ece67d3](ece67d37ba))
* **filters:** keep the range field readable and cover a BETWEEN in the rewrite ([be79f3a](be79f3a1ea))
* **filters:** map an IN entry to its unformatted value before the literal ([8f98ebe](8f98ebe326))
* **filters:** stop the temporal rewrite corrupting a typed value, and bind the range dropdown to the unformatted value ([a5f61d4](a5f61d4a73))
* **filters:** use the unformatted value in every IN branch ([cc01f54](cc01f540c2))
* **makedata:** shorten the DC_MAXCELLS_WEBEDIT description to fit var_desc ([a51d9e5](a51d9e5574))
* **multi-dataset:** surface the reserved-column guard, use the name constant ([2b8a2fa](2b8a2fa103))
* **release:** emit the release body as raw text, not a JSON string ([a05554e](a05554e67a))

### Features

* **filters:** show dates as date literals in the filter query ([662d000](662d00032c))

.gitea/scripts/verify-section.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Emits the "Verifying the download" section appended to every release body
+# by the "Upload assets to release" step of release.yaml.
+# Keep this text in sync with the docs page:
+#   docs.datacontroller.io/docs/downloads.md
+set -euo pipefail
+cat <<'EOF'
+
+## Verifying the download
+
+Every asset on this release is covered by the `SHA256SUMS` file. To verify a download, fetch `SHA256SUMS` from the release assets alongside the files you downloaded, then run:
+
+```
+sha256sum --check SHA256SUMS --ignore-missing
+```
+
+The command reports `OK` for each asset that matches. `--ignore-missing` lets you check just the files you downloaded rather than all of them.
+
+What this protects: a matching checksum confirms the asset is byte-for-byte the file this pipeline uploaded, guarding against corrupted or tampered downloads (mirrors, proxies, interrupted transfers). It is not a signature: the hashes travel in the same release as the files, so a compromise of the forge itself could alter both. Treat it as an integrity check, not a trust anchor.
+EOF

.gitea/workflows/build.yaml

@@ -15,6 +15,22 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
+      - name: Scan for secrets (gitleaks)
+        # The local pre-commit hook is suppressed entirely when .npmrc sets
+        # ignore-scripts=true (the prepare script that would activate the
+        # hooks never runs), so CI is the enforcement point: a leaked
+        # credential must fail the build, not just a commit.
+        # Uses the same @nogoo9/gitleaks binary the pre-commit hook pins - the
+        # gitleaks-action requires a license key for organization repos.
+        # Runs before client/.npmrc is written below, because that file is
+        # created from the NPMRC secret and a later scan would flag the
+        # registry token this job injects itself.
+        # --no-git scans the working tree, which is what a pull request gate
+        # is about.
+        run: |
+          npm ci
+          node_modules/.bin/gitleaks detect --no-git --source . --redact --no-banner -v
+
       - name: Install Google Chrome
         run: |
           apt-get update
@@ -107,6 +123,15 @@ jobs:
           echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
           npm ci
 
+      - name: Build frontend for web streaming
+        # The server-ci target streams client/dist as the 'clickme' web app (see
+        # CONTRIBUTING.md), so the production build has to exist before the mock
+        # services are deployed. Without it the deploy logs
+        # "webSourcePath: .../client/dist ... doesn't exist" and streams no frontend.
+        run: |
+          cd client
+          npm run build
+
       - name: Setup and start SASjs server
         run: |
           npm i -g pm2
@@ -117,8 +142,27 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          # The server refuses to run as root unless RUN_AS names an account
+          # (a security change in the recent releases). This job runs as root
+          # in a throwaway container, so opt in explicitly.
+          echo RUN_AS=root >> .env
           cat .env
           pm2 start api-linux --wait-ready
+          # `pm2 start --wait-ready` can return while the app is still
+          # launching (it does not always wait for the ready signal), and the
+          # next step hits :5000 immediately. Wait for the server to answer,
+          # and dump its log if it never does.
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5000/; then
+              echo "SASjs server answered on attempt ${i}"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "SASjs server did not come up within 120s"
+          pm2 list
+          pm2 logs api-linux --lines 100 --nostream
+          exit 1
 
       - name: Deploy mocked services
         run: |
@@ -154,7 +198,7 @@ jobs:
           # Start frontend and run cypress
           # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

.gitea/workflows/lighthouse.yaml

@@ -33,10 +33,29 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          # The server refuses to run as root unless RUN_AS names an account
+          # (a security change in the recent releases). This job runs as root
+          # in a throwaway container, so opt in explicitly.
+          echo RUN_AS=root >> .env
           cat .env
           curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
           unzip linux.zip
           pm2 start api-linux --wait-ready
+          # `pm2 start --wait-ready` can return while the app is still
+          # launching (it does not always wait for the ready signal), and the
+          # next step hits :5000 immediately. Wait for the server to answer,
+          # and dump its log if it never does.
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5000/; then
+              echo "SASjs server answered on attempt ${i}"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "SASjs server did not come up within 120s"
+          pm2 list
+          pm2 logs api-linux --lines 100 --nostream
+          exit 1
 
       - name: Write .npmrc file
         run: echo "$NPMRC" > client/.npmrc

.gitea/workflows/release.yaml

@@ -117,8 +117,27 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          # The server refuses to run as root unless RUN_AS names an account
+          # (a security change in the recent releases). This job runs as root
+          # in a throwaway container, so opt in explicitly.
+          echo RUN_AS=root >> .env
           cat .env
           pm2 start api-linux --wait-ready
+          # `pm2 start --wait-ready` can return while the app is still
+          # launching (it does not always wait for the ready signal), and the
+          # next step hits :5000 immediately. Wait for the server to answer,
+          # and dump its log if it never does.
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5000/; then
+              echo "SASjs server answered on attempt ${i}"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "SASjs server did not come up within 120s"
+          pm2 list
+          pm2 logs api-linux --lines 100 --nostream
+          exit 1
 
       - name: Deploy mocked services
         run: |
@@ -338,23 +357,47 @@ jobs:
           RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
           RELEASE_BODY=$(echo "$RELEASE_JSON" | jq -r '.body')
 
-          # Update body (also confirms the token has contents:write on this repo)
+          # Generate SHA-256 checksums over the release assets, so downloads can
+          # be verified. The file uses basename-only paths so that a plain
+          # "sha256sum -c SHA256SUMS" works in the folder the assets were
+          # downloaded to (the upload below stores them under their basename).
+          ASSETS=(frontend.zip
+                  sas/demostream_sas9.sas
+                  sas/viya.sas
+                  sas/sasjs_server.json.zip
+                  sas/sas9.sas
+                  sas/viya_noweb.sas
+                  sas/viya_noweb.json)
+
+          : > SHA256SUMS
+          for f in "${ASSETS[@]}"; do
+            sum=$(sha256sum "$f" | cut -d' ' -f1)
+            printf '%s  %s\n' "$sum" "${f##*/}" >> SHA256SUMS
+          done
+          cat SHA256SUMS
+
+          # Update body: keep the existing notes, add a verification section
+          # ahead of the installation footer. The section text lives in a
+          # dedicated script below (.gitea/scripts/verify-section.sh) so this
+          # YAML scalar stays clean.
+          # `jq -r` is required: without it jq emits the result as a JSON string
+          # literal (quoted, with \n escapes), and the PATCH below then encodes
+          # that literal a second time, so the release body renders as one
+          # escaped line instead of markdown.
+          VERIFY_SECTION=$(./.gitea/scripts/verify-section.sh)
+          NEW_BODY=$(jq -nr --arg body "$RELEASE_BODY" --arg verify "$VERIFY_SECTION" \
+            '$body + "\n" + $verify + "\n\nFor installation instructions, please visit https://docs.datacontroller.io/"')
           curl -k --fail-with-body -sS -X PATCH \
             -H "$AUTH_HEADER" \
             -H 'Content-Type: application/json' \
-            --data "$(jq -n --arg body "$RELEASE_BODY"$'\n\nFor installation instructions, please visit https://docs.datacontroller.io/' \
-                        '{draft:false, body:$body}')" \
+            --data "$(jq -n --arg body "$NEW_BODY" '{draft:false, body:$body}')" \
             "$BASE/releases/$RELEASE_ID"
 
           # Upload assets
           URL="$BASE/releases/$RELEASE_ID/assets"
-          for f in frontend.zip \
-                   sas/demostream_sas9.sas \
-                   sas/viya.sas \
-                   sas/sasjs_server.json.zip \
-                   sas/sas9.sas \
-                   sas/viya_noweb.sas \
-                   sas/viya_noweb.json; do
+          for f in "${ASSETS[@]}"; do
             echo "Uploading $f ..."
             curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@$f"
           done
+          echo "Uploading SHA256SUMS ..."
+          curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@SHA256SUMS"

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+# Hooks for the pre-commit framework - https://pre-commit.com
+#
+# The repo also ships native hooks in .git-hooks/, covering the secret scan, a
+# 2MB commit size limit, a prettier check and Conventional Commits. But .npmrc
+# sets ignore-scripts=true, so the `prepare` script that points git at them
+# never runs on `npm i` - a fresh clone commits unchecked until the one-time
+# command in CONTRIBUTING.md is run by hand.
+#
+# This file is the standard route instead: install the framework once with
+# `pre-commit install`, and the secret scan runs on every commit. The hook is
+# pinned to an exact gitleaks release tag.
+repos:
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.30.1
+    hooks:
+      - id: gitleaks

CHANGELOG.md

@@ -1,3 +1,108 @@
+# [7.18.0](https://git.datacontroller.io/dc/dc/compare/v7.17.0...v7.18.0) (2026-10-19)
+
+
+### Bug Fixes
+
+* **deps:** bump @sasjs/core to 5.4.0 ([19fb032](https://git.datacontroller.io/dc/dc/commit/19fb032f0dbdd612651ca4ff3b55d1aefa4850d5))
+* **deps:** pin @sasjs/utils and drop the unused jsrsasign ([05d2d1e](https://git.datacontroller.io/dc/dc/commit/05d2d1e08e2c3082f0dd5aa4629d91907c0401fc))
+* **editor:** own the delete-record column client-side ([56025cc](https://git.datacontroller.io/dc/dc/commit/56025cc0524d34d3589992e9f1059c13fb5581ed))
+* **editor:** surface the reserved-column refusal on every getdata path ([b26f557](https://git.datacontroller.io/dc/dc/commit/b26f5571b66b65166c1158817f2df3027ae45cdc))
+* **filters:** do not turn a fractional temporal value into a literal ([f710d87](https://git.datacontroller.io/dc/dc/commit/f710d871fffc0fa451fd8cc4d037a26414725654))
+* **filters:** fix the dynamic where clause deref and map its temporal values ([ece67d3](https://git.datacontroller.io/dc/dc/commit/ece67d37ba2c4e2037dea626b9730a692388f025))
+* **filters:** keep the range field readable and cover a BETWEEN in the rewrite ([be79f3a](https://git.datacontroller.io/dc/dc/commit/be79f3a1ea5e4c4a157396d276f2c6654f615d39))
+* **filters:** map an IN entry to its unformatted value before the literal ([8f98ebe](https://git.datacontroller.io/dc/dc/commit/8f98ebe326978b8da58d8ac895ec946e562b5de4))
+* **filters:** stop the temporal rewrite corrupting a typed value, and bind the range dropdown to the unformatted value ([a5f61d4](https://git.datacontroller.io/dc/dc/commit/a5f61d4a73051237b671695f67122c237770c95a))
+* **filters:** use the unformatted value in every IN branch ([cc01f54](https://git.datacontroller.io/dc/dc/commit/cc01f540c29d3d5f7dd86943b6eed82b332ce868))
+* **makedata:** shorten the DC_MAXCELLS_WEBEDIT description to fit var_desc ([a51d9e5](https://git.datacontroller.io/dc/dc/commit/a51d9e5574383e5d1e1943c95261fbce63e8788d))
+* **multi-dataset:** surface the reserved-column guard, use the name constant ([2b8a2fa](https://git.datacontroller.io/dc/dc/commit/2b8a2fa1039da3ab888af8c3ffae7e1b848b250a))
+* **release:** emit the release body as raw text, not a JSON string ([a05554e](https://git.datacontroller.io/dc/dc/commit/a05554e67a8d44e32e2d9f5e4a959a9d0592be5e))
+
+
+### Features
+
+* **filters:** show dates as date literals in the filter query ([662d000](https://git.datacontroller.io/dc/dc/commit/662d00032c27634bbc2d82c392e7875a2e174eff))
+
+# [7.17.0](https://git.datacontroller.io/dc/dc/compare/v7.16.0...v7.17.0) (2026-10-15)
+
+
+### Bug Fixes
+
+* **client:** do not publish an empty getcols fallback to the picker ([9f8fc6f](https://git.datacontroller.io/dc/dc/commit/9f8fc6f2eb83c78ea9cb220bb74059bab39e66ad))
+* **client:** map the getcols fallback onto the picker vocabulary ([7cecf79](https://git.datacontroller.io/dc/dc/commit/7cecf795153138d6f2ad1f87f7bdd12b5d861341))
+* **client:** pin the dependency entries this release changes ([b034fe5](https://git.datacontroller.io/dc/dc/commit/b034fe5335c41ec0289e459270839ee7caa8e126))
+* **deploy:** apply the same identity-call fix to the automatic deploy ([5693b44](https://git.datacontroller.io/dc/dc/commit/5693b4488b0328022c69dd34f98662d660612578))
+* **deploy:** disable the manual deploy actions without an admin group ([6ffbdba](https://git.datacontroller.io/dc/dc/commit/6ffbdbadf9804e0128694ed1c2bdb8f957743ad9))
+* **deploy:** disable the manual deploy file inputs while admin groups load ([7094bfd](https://git.datacontroller.io/dc/dc/commit/7094bfdf7cdab4c70e5f1a7763a8a8aeb8cc61a7))
+* **deploy:** guard the manual deploy form and Viya identity calls ([6522fcb](https://git.datacontroller.io/dc/dc/commit/6522fcb28b67b910b2784b93251f591bbc79d294))
+* **deploy:** never bake an empty admin group into the deploy script ([b0e47fc](https://git.datacontroller.io/dc/dc/commit/b0e47fc4ed4e36878de6505e678667b14d19c5e6))
+* **deps:** pin shelljs to clear the braces advisory in the sas audit ([6e50938](https://git.datacontroller.io/dc/dc/commit/6e5093858be106a1c3545a16119d246f69aac3da))
+* **editor:** check the EDIT limits once, and skip switched-off options ([f08c4c2](https://git.datacontroller.io/dc/dc/commit/f08c4c231f019979a27fe71fe73798752c015710))
+* **filters:** carry TYPE on the fallback list, and drop the temporal exclusion ([bcb8fde](https://git.datacontroller.io/dc/dc/commit/bcb8fde9868b8e1ded1590e204c34054f62d8250))
+* **filters:** source the picker variables from public/getcols when the table is not returned ([3a6a216](https://git.datacontroller.io/dc/dc/commit/3a6a216b9d4dfc462e05031039c47002a9156130))
+* **getcols:** use %global and %mp_getcols, and merge the base branch ([1385741](https://git.datacontroller.io/dc/dc/commit/1385741f2d303c1432f1f0ba637dd3cb39500a71))
+* **getcols:** widen TYPE before assigning the client vocabulary ([bcad05e](https://git.datacontroller.io/dc/dc/commit/bcad05e03291fc3e2b72a4465629e1e330d70c8a))
+* **getdata:** apply row level security to selectbox source values ([768d33c](https://git.datacontroller.io/dc/dc/commit/768d33c618a7ea453c3c19723f342a170ec576ea)), closes [#340](https://git.datacontroller.io/dc/dc/issues/340)
+* **mocks:** read MPE_CONFIG correctly, and pin the limits to it ([05126ae](https://git.datacontroller.io/dc/dc/commit/05126ae73a1ab2461066eb701b7475674b96f1ec))
+* repair the SAS suite for the row/cell limits and default the manual deploy admin group ([99ff732](https://git.datacontroller.io/dc/dc/commit/99ff732d67425b01a8a8f5e3e7c98b3e6d604323))
+* **requests-modal:** widen the dialog so log lines need less scrolling ([3c10d91](https://git.datacontroller.io/dc/dc/commit/3c10d915a5134247aff0e987dd4d5c83c197e4c5))
+* **sas:** keep format and label in the public getcols response ([9c32b1b](https://git.datacontroller.io/dc/dc/commit/9c32b1b1dae00e4b7bfd4acad94cbed831819d84))
+* **sas:** keep the SAS C/N vocabulary in the public getcols response ([e90bcde](https://git.datacontroller.io/dc/dc/commit/e90bcde51218c9e5f562f7aaa65506761b21d6bd))
+* **sas:** run the row/cell limit migration as a Viya job ([90af6b8](https://git.datacontroller.io/dc/dc/commit/90af6b851542a97f79172b655070cda559386a5c))
+* **startup:** stack the SASjs tag detail so the dump is readable ([1f4c4a4](https://git.datacontroller.io/dc/dc/commit/1f4c4a4d8297568f46bcc16ea208673fedad345b))
+
+
+### Features
+
+* **debug:** keep the debug toggle state across a page refresh ([3ce006d](https://git.datacontroller.io/dc/dc/commit/3ce006d6b469b4ece2bc769f0a32a1792eb1e774))
+* **editor:** raise the row limits and add a cell limit to the EDIT screen ([3de6655](https://git.datacontroller.io/dc/dc/commit/3de6655f26b7e895f60266e9931b46cfc0053975))
+* **mocks:** read the DC_MAX* options from MPE_CONFIG, and mock the DDL export ([edac9a9](https://git.datacontroller.io/dc/dc/commit/edac9a987aad3e4217641eed43f697f61c4bf739))
+* **startup:** call out a missing SASjs tag on the boot screen ([b24eb73](https://git.datacontroller.io/dc/dc/commit/b24eb731ce089acf496bed1e11a5e34208c1bd11))
+* **startup:** dump the SASjs tag contents in the boot screen checks ([75c1dcc](https://git.datacontroller.io/dc/dc/commit/75c1dccfd23557ab321cd7abd1508f20d817c7b8))
+* **system:** export the DC library DDL from the system screen ([05f3f49](https://git.datacontroller.io/dc/dc/commit/05f3f49e0c4d7005fa1f93852d73f2368395ab28)), closes [#179](https://git.datacontroller.io/dc/dc/issues/179)
+
+# [7.16.0](https://git.datacontroller.io/dc/dc/compare/v7.15.0...v7.16.0) (2026-09-28)
+
+
+### Bug Fixes
+
+* **ci:** correct server-ci webSourcePath and build frontend before mock deploy ([a4ad4cb](https://git.datacontroller.io/dc/dc/commit/a4ad4cba2861fe76f7ee5ea72ccf8c6e61c888a5))
+* **client:** parse iOS browsers in parseUserAgent ([eec3438](https://git.datacontroller.io/dc/dc/commit/eec3438de9735e0feb74c95d0b28128af6677477))
+* **diagnostics:** give the startup service a body for browser_info ([5538e0c](https://git.datacontroller.io/dc/dc/commit/5538e0c5752553a7c61903c62e4fdc82b6206826))
+* **diagnostics:** treat _debug=128 as debug on ([8de9978](https://git.datacontroller.io/dc/dc/commit/8de99782134e37c5e9c26c3117b083f49fa65189))
+* **loader:** abort when a post edit hook routes to an unregistered table ([171974b](https://git.datacontroller.io/dc/dc/commit/171974bd412be3efafd792e6c61c23f2fb442ace))
+* **mocks:** keep the mock hook reader inside the Drive ([7fd8ae4](https://git.datacontroller.io/dc/dc/commit/7fd8ae462993f0a1a8a034b5938d309d63920043))
+* **sas:** drop the duplicated abort in the target re-registration check ([48d45ac](https://git.datacontroller.io/dc/dc/commit/48d45ac7f8c32e39ff2d0e1ceac50252a49ebadd))
+* **sidebar:** drop the query string from the sub-page label ([d2dd46b](https://git.datacontroller.io/dc/dc/commit/d2dd46bfdfddd1695d1b3915d20ce79330cf275b))
+
+
+### Features
+
+* **client:** report the browser and both versions in browser_info ([a770f36](https://git.datacontroller.io/dc/dc/commit/a770f3695ceac441f48527f606843602c39fcc13))
+* **client:** send session_info with every service request ([50282f2](https://git.datacontroller.io/dc/dc/commit/50282f2ddb1250e048c55c04b395b428167142f7))
+* **filters:** make the applied-filter panel expandable ([fdaa249](https://git.datacontroller.io/dc/dc/commit/fdaa249f447b3a2c87da4b2aa1e5978c09f53ff8))
+* **mocks:** emulate the %mpeinit diagnostics dump ([200ee89](https://git.datacontroller.io/dc/dc/commit/200ee8931d3952624434567b5b95c9e7e53c6508))
+* **mocks:** run pre/post edit hook programs in the JS mock services ([b5f2283](https://git.datacontroller.io/dc/dc/commit/b5f228351bca18cbc8a3032e60846d05192668f6))
+* **mpeinit:** dump session_info to the log when debug is on ([fbd77a5](https://git.datacontroller.io/dc/dc/commit/fbd77a533417b1c8549a339b94785f7af0a30ca8))
+* **release:** publish SHA256SUMS and verify instructions with each release ([f1ae501](https://git.datacontroller.io/dc/dc/commit/f1ae501d49b510a1870f3682bb7d83de7446e061))
+
+# [7.15.0](https://git.datacontroller.io/dc/dc/compare/v7.14.2...v7.15.0) (2026-09-23)
+
+
+### Bug Fixes
+
+* **mocks:** keep special missings in the DIFF, and add the clip recording spec ([d124ce3](https://git.datacontroller.io/dc/dc/commit/d124ce3b36f8f5153e8d7a576d4bec3189524ed5))
+* **mocks:** list a column's own special missing in its dropdown ([b8f16a6](https://git.datacontroller.io/dc/dc/commit/b8f16a63827310f68629f2e016a4fb876593c37c))
+* **mocks:** make the approval path work, and record the approval scene ([12847cf](https://git.datacontroller.io/dc/dc/commit/12847cf07165d5aefa61592815cab0a37245e31f))
+* **validator:** a range rule ignores a missing value ([70dae4b](https://git.datacontroller.io/dc/dc/commit/70dae4b7013ac825eb77d3dc2b622980c0e4af05))
+* **validator:** keep the regular missing in a numeric dropdown source too ([a639bca](https://git.datacontroller.io/dc/dc/commit/a639bca70264e8a6a81795892b9ed63ba58d5f73))
+* **validator:** primary keys are NOT NULL, and a strict dropdown can match a special missing ([586a41a](https://git.datacontroller.io/dc/dc/commit/586a41ad49670ee795284a83cf8535711ce14d2c))
+* **validator:** reject a special missing on a NOT NULL column ([6d85bce](https://git.datacontroller.io/dc/dc/commit/6d85bce2a0c528e8eab9ee4abbade8c682ec3a1b))
+
+
+### Features
+
+* **validator:** compare MINVAL and MAXVAL in SAS's own order ([3467322](https://git.datacontroller.io/dc/dc/commit/3467322e99b0c3f6fda5654d35b507fdb2cb1c22))
+
 ## [7.14.2](https://git.datacontroller.io/dc/dc/compare/v7.14.1...v7.14.2) (2026-09-22)
 
 

CONTRIBUTING.md

@@ -19,6 +19,8 @@ git config core.hooksPath ./.git-hooks
 
 The pre-commit hook requires the gitleaks binary provided by the root `@nogoo9/gitleaks` devDependency, so make sure `npm i` has run in the repo root before your first commit.
 
+The standard [pre-commit](https://pre-commit.com) route is available as well, and does not depend on the lifecycle scripts above: install it once with `pre-commit install` and the secret scan in [`.pre-commit-config.yaml`](./.pre-commit-config.yaml) runs on every commit. The same scan also runs in CI, so a leaked credential fails the build regardless of which local hooks are active.
+
 ## Dependencies that requires licences
 
 [SheetJS Pro Version](https://www.npmjs.com/package/sheetjs)
@@ -48,15 +50,12 @@ npm i
 
 [Handsontable](https://www.npmjs.com/package/handsontable)
 
-Licence should be inserted in the `client/src/index.html` file:
+The licence key comes from the backend licence: the decrypted licence carries `hot_license_key`,
+which is published on the `hot_license_key` subject and pushed into every Handsontable instance.
+Until a licence is loaded the client falls back to the non-commercial key
+(`DEFAULT_HOT_LICENCE_KEY` in `client/src/app/services/licence.service.ts`), so there is nothing to
+configure in `index.html` for this.
 
-```
- <sasjs
-    ...
-    hotLicenceKey="LICENCE-GOES-HERE"
->
-</sasjs>
-```
 ## Development
 Update `client/src/index.html` so that it points to your SAS9, SASVIYA or SASJS backend.  
 Be aware that VIYA can be configured in such way that it would not work with cross origin frontend.  

client/cypress/clips/special-missings-clip.cy.ts

@@ -0,0 +1,592 @@
+// Clip script: special missings inside Data Controller's validation rules.
+//
+// This is not a test - it is the recording script for the companion clip, and
+// it deliberately pauses between steps so each beat is readable on video. It
+// lives in cypress/clips (outside cypress/e2e) so the default spec pattern does
+// not pick it up in CI.
+//
+// Record it with:
+//
+//   npx cypress run --spec cypress/clips/special-missings-clip.cy.ts \
+//     --config video=true,viewportWidth=1280,viewportHeight=720
+//
+// The table is TESTDATA.DEMO_01 (seven columns, one rule each):
+//   ID (NOTNULL), AMOUNT (MINVAL 1), SCORE (MAXVAL 100),
+//   REF (SOFTREGEX /^[0-9]+$/), STATUS (SOFTSELECT TESTDATA.DEMO_01.STATUS),
+//   RATING (no rule), LAST_REVIEWED (date9.)
+//
+// One editing session, one submission. The rule scenes are played on row 1 and
+// each value is put back afterwards, so the DIFF that follows carries only the
+// two changes the clip is about; the review screens are reached through the
+// app's own navigation rather than cy.visit, so nothing reloads mid-clip.
+//
+// The beats are deliberately short - the take is played back at ~1.4x on the
+// way out, so a hold that looks tight here reads as a normal pause on the
+// finished clip.
+
+// Caption track: each mark opens a caption and carries the text it shows.
+// The encoder turns consecutive marks into subtitle cues, so the captions are
+// timed by the recording itself rather than by guessed offsets.
+const BEATS = '/tmp/clip-beats.tsv'
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+// A beat long enough to read on video.
+const beat = (ms = 1500) => cy.wait(ms)
+
+context('special missings clip (DEMO_01)', function () {
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    // The mock estate carries a valid licence key (mock-storage/licence.json),
+    // so the app activates directly with no free-tier banner in shot.
+    cy.get('.nav-tree', { timeout: longerCommandTimeout }).should('exist')
+  })
+
+  it('records the demo in one editing session', () => {
+    cy.writeFile(BEATS, '', { flag: 'w' })
+
+    // ---- Scene 1: the table and its rules ----------------------------------
+    openTableFromTree('testdata', 'demo_01')
+    beat(1200)
+
+    clickOnEdit(() => {
+      // Seven columns, five rules: ID is the key, AMOUNT the floor, SCORE the
+      // ceiling, REF the pattern, STATUS the dropdown, and RATING and
+      // LAST_REVIEWED carry nothing.
+      cy.get('.ht_master tbody tr', { timeout: longerCommandTimeout }).should(
+        'have.length.greaterThan',
+        3
+      )
+      mark(
+        'grid',
+        'TESTDATA.DEMO_01 - eight columns, six rules. ID is the key, and RATING and LAST_REVIEWED carry none.'
+      )
+      beat(2600)
+
+      // ---- Scene 2: typing a special missing ------------------------------
+      // RATING has no rule, so this shows entry on its own. Every beat asserts
+      // the settled cell state before holding, so the recording always rests on
+      // the outcome (flagged or accepted) rather than on a blind pause that
+      // might land before the rule engine has run.
+      mark(
+        'entry',
+        'A numeric cell takes a special missing as a letter or an underscore, with or without a period - row 2 already holds .a.'
+      )
+      typeAndHold(0, 'RATING', 'a', 'accepted') // a single letter is taken
+      mark(
+        'reject',
+        'Two letters, or a letter mixed with a number, are refused. Red means it will not submit.'
+      )
+      typeAndHold(0, 'RATING', 'AB', 'rejected', 2200) // two letters are not
+      typeAndHold(0, 'RATING', '1a', 'rejected', 2200) // nor a number and a letter
+      typeAndHold(0, 'RATING', 'a', 'accepted') // leave it as a special missing
+
+      // ---- Scene 3: NOTNULL refuses both --------------------------------
+      // ID is the key, so NOTNULL applies. A special missing is not a value
+      // here: like a blank, it fails the rule (a real SAS NOT NULL constraint
+      // rejects a special missing as well), and only a number satisfies it -
+      // the original key, put back so the row carries no change into the DIFF.
+      mark(
+        'notnull_blank',
+        "ID is the table's primary key, so NOT NULL is applied to it automatically - a blank fails..."
+      )
+      typeAndHold(0, 'ID', '', 'rejected', 2200) // a blank fails NOTNULL
+      mark(
+        'notnull_missing',
+        '...and so does a special missing: a missing is not a value to NOT NULL.'
+      )
+      typeAndHold(0, 'ID', 'A', 'rejected', 2200) // so does a special missing
+      mark(
+        'notnull_number',
+        'A number satisfies it, and the row key goes back.'
+      )
+      typeAndHold(0, 'ID', '1', 'accepted') // a number is what it wants
+
+      // ---- Scene 4: the pattern still applies -----------------------------
+      // REF carries SOFTREGEX /^[0-9]+$/, and a special missing is not exempt
+      // from it. The amber cell is a soft rule warning rather than a block,
+      // which the caption on this beat says out loud - the pattern itself is
+      // only visible in the cell's native title, which a screencast does not
+      // capture.
+      mark(
+        'softregex',
+        'REF carries SOFTREGEX /^[0-9]+$/. Amber is a soft warning - it warns, it does not block.'
+      )
+      typeIntoCell(0, 'REF', 'A')
+      getCellByHeaderAndRow(0, 'REF').should('have.class', 'dc-warning-cell')
+      beat(3400)
+      typeIntoCell(0, 'REF', '1001') // put the reference back
+      getCellByHeaderAndRow(0, 'REF').should(
+        'not.have.class',
+        'dc-warning-cell'
+      )
+      beat(600)
+
+      // ---- Scene 5: the dropdown lists the missing -------------------------
+      // STATUS carries a SOFTSELECT whose list is taken from the column itself
+      // (the library.member.column form), and that column holds a special
+      // missing - so the dropdown offers it alongside the ordinary values, as
+      // the bare letter SAS produces for it.
+      mark(
+        'dropdown',
+        "STATUS carries a SOFTSELECT, and its list is the column's own values."
+      )
+      openDropdown(0, 'STATUS')
+        .should('have.length', 4)
+        .then(($items: any) => {
+          const texts = [...$items].map((td: any) => td.innerText.trim())
+          expect(texts).to.include('A')
+        })
+      beat(2400)
+
+      mark(
+        'dropdown_pick',
+        'So the special missing the column holds is offered as a bare letter, first - a missing sorts below every number.'
+      )
+      pickFromDropdown('A')
+      getCellByHeaderAndRow(0, 'STATUS').should('contain.text', 'A')
+      beat(1800)
+      typeIntoCell(0, 'STATUS', '1') // put the status back
+      beat(600)
+
+      // ---- Scene 6: a range rule compares in SAS order, missings included
+      mark(
+        'minval',
+        'AMOUNT has MINVAL 1. A missing sorts below every number, so it is below the floor.'
+      )
+      typeAndHold(0, 'AMOUNT', 'A', 'rejected', 2200) // a missing is below the floor
+      mark(
+        'maxval',
+        'SCORE has MAXVAL 100 - the same missing is below the ceiling, so it passes.'
+      )
+      typeAndHold(0, 'SCORE', 'A', 'accepted', 2200) // a missing is below the ceiling
+      mark(
+        'grade',
+        'GRADE takes MINVAL .A and MAXVAL .C. The missings have an order of their own: .B is inside the range, .D is outside it.'
+      )
+      typeAndHold(0, 'GRADE', '.B', 'accepted', 2400) // .B is between .A and .C
+      typeAndHold(0, 'GRADE', '.D', 'rejected', 2400) // .D is above .C
+
+      mark('abort', 'Submitting with an invalid cell aborts.')
+
+      // Submit while AMOUNT is invalid - the modal reports it.
+      submitTable(() => {
+        cy.get('.modal-body', { timeout: longerCommandTimeout }).should(
+          'contain.text',
+          'Invalid Values are Present'
+        )
+        beat(2400)
+        // Close the abort so the editor is clean for the next scene.
+        cy.get('clr-modal.clr-abort-modal .modal-footer button')
+          .contains('Close')
+          .click({ force: true })
+        beat(800)
+      })
+
+      // ---- Scene 7: put the test values back, then make the real change ----
+      mark(
+        'clean',
+        'Back to a clean row. Row 2 already holds .a, so this changes one special missing to another.'
+      )
+      typeIntoCell(0, 'AMOUNT', '120')
+      typeIntoCell(0, 'SCORE', '82')
+      typeIntoCell(0, 'GRADE', '.a')
+      typeIntoCell(0, 'RATING', '4')
+      beat(800)
+
+      // Row 2 (ID 2) already carries a special missing in RATING, so this is a
+      // change from one special missing to another, and the date column gives
+      // the formatted / unformatted switch something to switch.
+      typeAndHold(1, 'RATING', 'B', 'accepted')
+      typeIntoDateCell(1, 'LAST_REVIEWED', '2026-01-15')
+      beat(1500)
+
+      submitTable(() => {
+        cy.get('#submitBtn', { timeout: longerCommandTimeout })
+          .should('exist')
+          .should('not.be.disabled')
+          .click()
+        beat(2500)
+      })
+    })
+
+    // ---- Scene 8: the queue, then the DIFF (in-app navigation) ------------
+    mark('submitted', 'Submitted - the queue shows it waiting for approval.')
+    goToReviewNav()
+    beat(1800)
+
+    // The submit queue lists oldest first, so the row we just created is last.
+    cy.get('app-submitter clr-datagrid clr-dg-row', {
+      timeout: longerCommandTimeout
+    })
+      .should('exist')
+      .last()
+      .click({ force: true })
+    beat(1800)
+
+    cy.get('app-approve-details .card', { timeout: longerCommandTimeout })
+      .should('exist')
+      .should('be.visible')
+    beat(1000)
+
+    // The DIFF table is wider than the frame, so the two columns that matter
+    // (RATING and LAST_REVIEWED) sit off the right edge until it is scrolled.
+    scrollDiffToEnd()
+    beat(1200)
+
+    // Only those two cells changed: the rule scenes were put back, so the DIFF
+    // is the two changes the clip is about and nothing else.
+    getDiffCell('RATING')
+      .should('contain.text', '.b')
+      .should('have.class', 'ch')
+    getDiffCell('LAST_REVIEWED')
+      .should('contain.text', '15JAN2026')
+      .should('have.class', 'ch')
+    cy.get('app-approve-details .tableCont tbody tr td:not(.ch)').should(
+      'have.length.greaterThan',
+      3
+    )
+    mark(
+      'diff',
+      'The DIFF compares staged with base: one changed row, two changed cells.'
+    )
+    beat(2600)
+
+    // ---- Scene 9: the staged data -----------------------------------------
+    // What the approval is actually acting on: the staged row, still holding
+    // the special missing, before it reaches the base table.
+    mark(
+      'staged',
+      'The staged row, before approval - still holding the special missing.'
+    )
+    clickButton('VIEW STAGED DATA')
+    // 'Basic Submitted Details' is on the staged screen only - asserting
+    // 'Staged Data' alone would be satisfied by the button that was just
+    // clicked, which is how a beat can pass without ever leaving the DIFF.
+    cy.get('body', { timeout: longerCommandTimeout })
+      .should('contain.text', 'Basic Submitted Details')
+      .and('contain.text', 'Base Table')
+    beat(4200)
+    mark('staged_end', '')
+
+    // ---- Scene 10: the approver opens it, and switches the format ----------
+    goToReviewNav()
+    beat(1500)
+    openApproveTab()
+    beat(1500)
+
+    cy.get('app-approve clr-datagrid clr-dg-row a.color-green', {
+      timeout: longerCommandTimeout
+    })
+      .should('exist')
+      .last()
+      .click({ force: true })
+    beat(1800)
+
+    cy.get('#acceptBtn', { timeout: longerCommandTimeout })
+      .should('exist')
+      .should('not.be.disabled')
+    beat(1000)
+
+    // Same scroll as the submitter view, so the date column is in frame when
+    // the format is switched.
+    scrollDiffToEnd()
+    beat(1200)
+
+    mark(
+      'approve',
+      'The approver opens the same submission. Hovering a changed cell shows the value it replaced.'
+    )
+
+    // Hovering the two changed cells shows what each replaced - that is how the
+    // two special missings are told apart, not just the before/after of one.
+    hoverDiffCell('RATING', 'Original value is: .a')
+    beat(2800)
+
+    mark('hover_date', '...including the date it replaced.')
+    hoverDiffCell('LAST_REVIEWED', 'Original value is: 29FEB2024')
+    beat(2800)
+
+    mark(
+      'toggle',
+      'The formatted / unformatted switch shows the value as SAS stores it: 24121.'
+    )
+    cy.get('.formatted-values-toggle').should('have.text', 'Formatted').click()
+    cy.get('.formatted-values-toggle').should('have.text', 'Unformatted')
+    getDiffCell('LAST_REVIEWED').should('contain.text', '24121')
+    getDiffCell('RATING').should('contain.text', '.b')
+    beat(2800)
+
+    cy.get('.formatted-values-toggle').click()
+    cy.get('.formatted-values-toggle').should('have.text', 'Formatted')
+    getDiffCell('LAST_REVIEWED').should('contain.text', '15JAN2026')
+    beat(1000)
+
+    // ---- Scene 11: approve, and the change is in the history --------------
+    mark('accepted', 'Accepted - the history records it as APPROVED.')
+    cy.get('#acceptBtn').click()
+    cy.url({ timeout: longerCommandTimeout }).should(
+      'include',
+      '/review/history'
+    )
+    cy.get('app-history clr-datagrid clr-dg-row', {
+      timeout: longerCommandTimeout
+    })
+      .should('exist')
+      .first()
+      .should('contain.text', 'APPROVED')
+    beat(2800)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Helpers (mirrored from the e2e specs so the clip drives the same UI paths)
+// ---------------------------------------------------------------------------
+
+/**
+ * Caption track. Each mark opens a caption and carries the text it shows; the
+ * encoder turns consecutive marks into subtitle cues, so the captions are timed
+ * by the recording itself rather than by guessed offsets.
+ *
+ * The timestamp has to be taken inside a `cy.then()`: Cypress evaluates a
+ * command's arguments when the command is *queued*, so `Date.now()` passed to
+ * `cy.writeFile` directly would give every mark the same value - the moment the
+ * spec body ran.
+ */
+const mark = (label: string, text: string) => {
+  cy.then(() => {
+    cy.writeFile(BEATS, `${label}\t${text}\t${Date.now()}\n`, { flag: 'a+' })
+  })
+}
+
+const typeIntoCell = (rowIndex: number, header: string, value: string) => {
+  getCellByHeaderAndRow(rowIndex, header)
+    .dblclick({ force: true })
+    .then(() => {
+      cy.focused().clear().type(`${value}{enter}`)
+    })
+}
+
+/**
+ * A date-formatted column edits through an HTML date input, where cy.type()
+ * refuses anything but a bare YYYY-MM-DD string (so no {enter} in the same
+ * call). Set the value through the DOM and commit it with the Enter keydown
+ * that Handsontable listens for.
+ */
+const typeIntoDateCell = (rowIndex: number, header: string, value: string) => {
+  getCellByHeaderAndRow(rowIndex, header)
+    .dblclick({ force: true })
+    .then(() => {
+      cy.focused()
+        .then(($i: any) => {
+          const el = $i[0]
+          el.value = value
+          el.dispatchEvent(new Event('input', { bubbles: true }))
+          el.dispatchEvent(new Event('change', { bubbles: true }))
+        })
+        .trigger('keydown', { key: 'Enter', keyCode: 13, which: 13 })
+    })
+}
+
+/**
+ * Opens the selectbox on a cell and returns its list entries. Handsontable
+ * renders the arrow itself (`.htAutocompleteArrow`); the list is a
+ * `.handsontable.listbox` in the app document.
+ */
+const openDropdown = (rowIndex: number, header: string) => {
+  getCellByHeaderAndRow(rowIndex, header).within(() => {
+    cy.get('.htAutocompleteArrow').click({ force: true })
+  })
+
+  return cy.get('.handsontable.listbox td', { timeout: longerCommandTimeout })
+}
+
+/** Picks an entry from the open selectbox. */
+const pickFromDropdown = (value: string) => {
+  cy.get('.handsontable.listbox td').contains(value).click({ force: true })
+}
+
+/** Reaches the review area through the app's own navigation (no reload). */
+const goToReviewNav = () => {
+  cy.get('.nav-link', { timeout: longerCommandTimeout })
+    .contains('REVIEW')
+    .click({ force: true })
+}
+
+/** Opens the APPROVE tab within the review area. */
+const openApproveTab = () => {
+  cy.get('.nav-link', { timeout: longerCommandTimeout })
+    .contains('APPROVE')
+    .click({ force: true })
+}
+
+/**
+ * Clicks a button by its visible text. The match is case-insensitive: the app
+ * uppercases button labels in CSS, so the rendered text and the DOM's
+ * textContent differ.
+ */
+const clickButton = (text: string) => {
+  cy.contains('button', new RegExp(text, 'i'), {
+    timeout: longerCommandTimeout
+  }).click({ force: true })
+}
+
+/**
+ * The DIFF table is wider than the recording frame, so scroll its container to
+ * the end - that is what puts the changed RATING and the date column on screen.
+ */
+const scrollDiffToEnd = () => {
+  cy.get('app-approve-details .tableCont', { timeout: longerCommandTimeout })
+    .should('exist')
+    .scrollTo('right', { duration: 1200 })
+}
+
+/**
+ * The DIFF on the review screen is a plain HTML table (`.tableCont`), not
+ * Handsontable: headers are `th`, cells `td`, and a changed cell also carries
+ * the `ch` class whose tooltip holds the value it replaced.
+ */
+const getDiffCell = (headerText: string) => {
+  return cy
+    .get('app-approve-details .tableCont thead tr th', {
+      timeout: longerCommandTimeout
+    })
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => th.innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => th.innerText.trim() === headerText
+      )
+
+      return cy
+        .get('app-approve-details .tableCont tbody tr')
+        .first()
+        .then(($tr: any) => $tr[0].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}
+
+/**
+ * Reveals the value a changed DIFF cell replaced, and waits until it is really
+ * on screen.
+ *
+ * Clarity shows the tooltip through CSS :hover, which a synthetic
+ * `trigger('mouseover')` does NOT activate - the tooltip stays
+ * `visibility: hidden`, and a `contain.text` assertion still passes because the
+ * text is in the DOM. So this moves the real mouse (cypress-real-events) and
+ * then asserts the computed style. The first real move after another action can
+ * be swallowed, hence the repeat; if the tooltip ever fails to appear the
+ * recording fails rather than quietly showing nothing.
+ */
+const hoverDiffCell = (headerText: string, expected: string) => {
+  getDiffCell(headerText).realHover()
+  beat(300)
+  getDiffCell(headerText).realHover()
+
+  getDiffCell(headerText)
+    .find('.tooltip-content')
+    .should(($t) => {
+      const style = getComputedStyle($t[0] as HTMLElement)
+      expect(style.visibility, 'tooltip visibility').to.eq('visible')
+      expect(Number(style.opacity), 'tooltip opacity').to.be.greaterThan(0)
+      expect($t[0].textContent || '', 'tooltip text').to.contain(expected)
+    })
+}
+
+/**
+ * Types a value into a cell and holds on the settled result. The rule engine
+ * flags the cell (htInvalid) once the edit commits, so asserting the expected
+ * state before the hold means the recording always rests on the outcome, and
+ * waits for it however long the engine takes.
+ *
+ * @param expected 'rejected' when the rule engine should flag the cell,
+ *   'accepted' when the value should settle unflagged.
+ */
+const typeAndHold = (
+  rowIndex: number,
+  header: string,
+  value: string,
+  expected: 'accepted' | 'rejected',
+  hold = 1800
+) => {
+  typeIntoCell(rowIndex, header, value)
+
+  getCellByHeaderAndRow(rowIndex, header).should(
+    expected === 'rejected' ? 'have.class' : 'not.have.class',
+    'htInvalid'
+  )
+
+  beat(hold)
+}
+
+const getCellByHeaderAndRow = (rowIndex: number, headerText: string) => {
+  return cy
+    .get('.ht_clone_top .htCore thead tr th')
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => th.innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => th.innerText.trim() === headerText
+      )
+
+      return cy
+        .get('.ht_master tbody tr')
+        .then((rows: any) => rows[rowIndex].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}
+
+const clickOnEdit = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+const submitTable = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary')
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let libNode
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            libNode = node
+            break
+          }
+        }
+
+        cy.get(libNode).within(() => {
+          cy.get('.clr-tree-node-content-container > button').click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+}

client/cypress/e2e/browser-info.cy.ts

@@ -0,0 +1,165 @@
+// Marks this file as an ES module so its top-level consts don't collide with
+// other spec files under the TS type-checker.
+export {}
+//
+// The client sends `browser_info` (and `browser_url_vars`, when the page URL
+// has parameters) with the startup service and with the services that execute
+// customer-provided code - hook scripts and dynamic cell dropdown programs.
+// Other services do not receive the tables.
+//
+// Payloads are keyed by the _program value in the request URL, so the
+// assertions can tell one service's payload from another's. Asserted on the
+// adapter interface, which is the only place the payload is observable end
+// to end.
+//
+// Each test starts from a plain visit of the app root rather than assuming
+// the previous test's session: Cypress clears cookies between tests, the
+// SASjs Server session drops, and the app lands back on the evaluation
+// agreement card. The acceptance is guarded, so it is a no-op when the
+// session survived.
+const hostUrl = Cypress.env('hosturl')
+
+/** The services the app sends diagnostics to (see DIAGNOSTICS_SERVICES in
+ * client/src/app/services/sas.service.ts). */
+const DIAGNOSTICS_SERVICES = [
+  'public/startupservice',
+  'editors/getdata',
+  'editors/getdynamiccolvals',
+  'editors/stagedata',
+  'editors/loadfile',
+  'editors/restore',
+  'auditors/postdata'
+]
+
+/** Boots the app at the root, accepting the evaluation agreement if shown. */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: 60000 })
+  cy.get('body').then(($body) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Captures each service's payload, keyed by the _program URL parameter. */
+const capturePrograms = (): Record<string, string> => {
+  const programs: Record<string, string> = {}
+  cy.intercept('**/stp/execute/**', (req) => {
+    // The handler must never throw: an exception inside an intercept handler
+    // aborts the request, which breaks the app under test. A lone '%' in the
+    // URL would make decodeURIComponent throw, hence the guarded parse.
+    let program = req.url
+    try {
+      program = decodeURIComponent(
+        (req.url.match(/_program=([^&]+)/) || [])[1] || ''
+      )
+    } catch (e) {
+      program = (req.url.match(/_program=([^&]+)/) || [])[1] || req.url
+    }
+    if (typeof req.body === 'string') {
+      programs[program] = req.body
+    } else {
+      programs[program] = JSON.stringify(req.body)
+    }
+  })
+  return programs
+}
+
+context('browser_info input table: ', function () {
+  it('1 | services that run no customer code do not receive it', () => {
+    const programs = capturePrograms()
+    bootApp()
+
+    // The viewer route fires public/viewlibs and a usernav service - plain
+    // data services with no hook scripts and no customer includes. Taken
+    // from a fresh boot, before anything caches the library tree.
+    cy.visit(`${hostUrl}/#/view/data`, { timeout: 60000 })
+    cy.wait(8000)
+
+    cy.then(() => {
+      const captured = Object.keys(programs)
+      // The startup service is a diagnostics service by design - it runs on
+      // every boot, which is the point of it - so it is excluded from this
+      // sweep. On a real estate it would otherwise appear here carrying
+      // browser_info and fail the test.
+      const plain = captured.filter(
+        (program) => !DIAGNOSTICS_SERVICES.some((s) => program.includes(s))
+      )
+      expect(
+        plain.length,
+        `non-diagnostics service calls captured (${captured.join(', ')})`
+      ).to.be.greaterThan(0)
+
+      plain.forEach((program) => {
+        expect(
+          programs[program],
+          `${program} carries no browser_info`
+        ).not.to.contain('browser_info')
+      })
+    })
+  })
+
+  it('2 | hook-running services receive it', () => {
+    const programs = capturePrograms()
+    bootApp()
+
+    // Opening the editor directly makes the app call editors/getdata, which
+    // runs the pre-edit hook and so receives the diagnostics tables.
+    cy.visit(`${hostUrl}/#/editor/DC_JSLIB.MPE_X_TEST`, { timeout: 60000 })
+    cy.get('#hotTable', { timeout: 60000 }).should('exist')
+    cy.wait(8000)
+
+    cy.then(() => {
+      const captured = Object.keys(programs)
+      expect(
+        captured.length,
+        `service calls captured (${captured.join(', ')})`
+      ).to.be.greaterThan(0)
+
+      // The startup service is the other diagnostics recipient, but it is not
+      // reachable from a mock estate: `checkSasjsDeploy` finds makedata in
+      // services/admin and routes to the deploy screen instead, so the app
+      // never calls it on boot. Its null-payload path is covered by the unit
+      // spec (src/app/services/sas.service.spec.ts).
+      const getdata = captured.find((p) => p.includes('editors/getdata'))
+      expect(getdata, 'editors/getdata captured').to.not.be.undefined
+      expect(
+        programs[getdata!],
+        'getdata payload carries browser_info'
+      ).to.contain('browser_info')
+      // TIMEZONE is a column of browser_info and of no other input table, so
+      // this shows the row itself arrived, not just the table name.
+      expect(
+        programs[getdata!],
+        'browser_info row content in the payload'
+      ).to.contain('timezone')
+    })
+  })
+
+  it('3 | browser_url_vars carries the URL parameters as name/value pairs', () => {
+    const programs = capturePrograms()
+    bootApp()
+
+    // The labels parameter is read by the editor from the hash query string,
+    // so the diagnostics tables must carry it as a name/value row.
+    cy.visit(`${hostUrl}/#/editor/DC_JSLIB.MPE_X_TEST?labels=true`, {
+      timeout: 60000
+    })
+    cy.get('#hotTable', { timeout: 60000 }).should('exist')
+    cy.wait(8000)
+
+    cy.then(() => {
+      const getdata = Object.keys(programs).find((p) =>
+        p.includes('editors/getdata')
+      )
+      expect(getdata, 'editors/getdata captured').to.not.be.undefined
+      expect(
+        programs[getdata!],
+        'getdata payload carries browser_url_vars'
+      ).to.contain('browser_url_vars')
+      // the labels parameter, as a name in the name/value table
+      expect(programs[getdata!], 'labels parameter row').to.contain('labels')
+    })
+  })
+})

client/cypress/e2e/coltype-delete-record.cy.ts

@@ -25,11 +25,13 @@ const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
 //   2. Ordinary columns (e.g. SOME_DROPDOWN) still get their rule via the
 //      new per-row cols[].COLTYPE path — test 2 proves the migration didn't
 //      silently drop real columns' rules either.
-// Column positions below come from COLHEADERS in
-// sas/mocks/sasjs/services/editors/getdata.js:
-// "_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,..."
-// — childNodes[0] on a body <tr> is the row-header <th>, so childNodes[N+1]
-// is the Nth (0-indexed) column in that list.
+// Column positions below come from the mock's COLHEADERS
+// (sas/mocks/sasjs/services/editors/getdata.js):
+// "PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,..."
+// — the client-owned delete column is prepended by the editor itself (see
+// deleteRecordColumn.ts), so on screen it is still column 0. childNodes[0] on
+// a body <tr> is the row-header <th>, so childNodes[N+1] is the Nth
+// (0-indexed) column in that list.
 context('coltype vertical-array regression tests: ', function () {
   this.beforeAll(() => {
     cy.visit(`${hostUrl}/SASLogon/logout`)

client/cypress/e2e/ddl-export.cy.ts

@@ -0,0 +1,164 @@
+export {}
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('Export DC Library DDL: ', function () {
+  this.beforeAll(() => {
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+    visitPage('home')
+  })
+
+  it('1 | the admin action opens the flavour chooser and exports the SAS flavour by default', () => {
+    visitPage('system')
+
+    cy.get('.admin-action', { timeout: longerCommandTimeout })
+      .contains('Export DC Library DDL')
+      .should('exist')
+    cy.get('.admin-action').contains('button', 'EXPORT').should('exist')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().should('be.visible')
+    ddlModal().within(() => {
+      cy.get('select').should('have.value', 'SAS')
+      // the schema box only applies to the DB flavours
+      cy.contains('label', 'Schema (optional)').should('not.exist')
+    })
+
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').should(
+      'have.been.calledWithMatch',
+      /\/services\/admin\/exportdb&flavour=SAS$/
+    )
+  })
+
+  it('2 | the PGSQL flavour reveals the optional schema box and passes it to the service', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().within(() => {
+      cy.get('select').select('PGSQL')
+      cy.contains('label', 'Schema (optional)').should('be.visible')
+      cy.get('input[type="text"]').clear().type('DC')
+    })
+
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').should(
+      'have.been.calledWithMatch',
+      /\/services\/admin\/exportdb&flavour=PGSQL&schema=DC$/
+    )
+  })
+
+  it('3 | the TSQL flavour exports without a schema when the box is left empty', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().within(() => {
+      cy.get('select').select('TSQL')
+      cy.contains('label', 'Schema (optional)').should('be.visible')
+    })
+
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').should(
+      'have.been.calledWithMatch',
+      /\/services\/admin\/exportdb&flavour=TSQL$/
+    )
+  })
+
+  it('4 | the SAS export streams the DC library DDL', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().contains('button', 'Export').click()
+
+    // the URL the action opens is served by the backend - against the mocks
+    // that is services/admin/exportdb.js, which renders the library as basic
+    // DDL per flavour
+    cy.get('@open').then((open: any) => {
+      const url = open.getCall(0).args[0]
+      expect(url).to.match(/\/services\/admin\/exportdb&flavour=SAS$/)
+
+      cy.request(url).then((res) => {
+        expect(res.status).to.eq(200)
+        expect(res.headers['content-disposition']).to.contain('DC_JSLIB.ddl')
+        expect(res.body).to.contain('create table DC_JSLIB.MPE_CONFIG(')
+        expect(res.body).to.contain('VAR_NAME char(32)')
+        expect(res.body).to.contain('create table DC_JSLIB.MPE_X_TEST(')
+        // the SAS flavour has no schema statement and no unique index
+        expect(res.body).not.to.contain('CREATE SCHEMA')
+        expect(res.body).not.to.contain('UNIQUE INDEX')
+      })
+    })
+  })
+
+  it('5 | the PGSQL export honours the schema and quotes its identifiers', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().within(() => {
+      cy.get('select').select('PGSQL')
+      cy.get('input[type="text"]').clear().type('DC')
+    })
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').then((open: any) => {
+      const url = open.getCall(0).args[0]
+      expect(url).to.match(/&flavour=PGSQL&schema=DC$/)
+
+      cy.request(url).then((res) => {
+        expect(res.status).to.eq(200)
+        expect(res.body).to.contain('CREATE SCHEMA DC;')
+        expect(res.body).to.contain('CREATE TABLE DC.MPE_CONFIG (')
+        expect(res.body).to.contain('"VAR_NAME" VARCHAR(32)')
+        expect(res.body).to.contain('CREATE UNIQUE INDEX "MPE_CONFIG_pk"')
+      })
+    })
+  })
+})
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}

client/cypress/e2e/filter-panel.cy.ts

@@ -0,0 +1,257 @@
+export {}
+//
+// The applied-filter panel shows the clause that is in force. A long clause
+// does not fit the collapsed single line, so the panel carries a chevron that
+// expands it to show the clause in full - in the viewer and in the editor.
+//
+// The chevron is only rendered when the clause actually overflows, so a short
+// filter looks exactly as it did before.
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('applied filter panel: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  // the panel behaviour depends on how much room the clause has, so pin a
+  // laptop-sized window rather than the wider CI viewport
+  this.beforeEach(() => {
+    cy.viewport(1280, 800)
+    bootApp()
+  })
+
+  it('1 | without a filter the panel is not rendered', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    cy.get('.infoBar').should('not.exist')
+
+    cy.screenshot('filter-panel-1-no-filter', { capture: 'viewport' })
+  })
+
+  it('2 | a short filter is shown in full, with no chevron', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    openFilterPopup()
+    setFilterVariable('SOME_DROPDOWN')
+    setFilterValue('Option 1')
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', "SOME_DROPDOWN = 'Option 1'")
+    // the clause fits the collapsed line, so there is nothing to expand
+    assertClauseFitsCollapsedLine()
+    cy.get('.infoBar-toggle').should('not.exist')
+
+    cy.screenshot('filter-panel-2-filter', { capture: 'viewport' })
+  })
+
+  it('3 | a long filter is collapsed to a chevron and expands to the full clause', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    // SOME_CHAR holds a very long free-text value, so an IN over every value
+    // produces a clause of several hundred characters
+    openFilterPopup()
+    setFilterVariable('SOME_CHAR')
+    setFilterOperator('IN')
+    chooseAllFilterValues()
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', 'SOME_CHAR IN')
+    assertClauseOverflowsCollapsedLine()
+    cy.get('.infoBar-toggle')
+      .should('exist')
+      .and('have.attr', 'aria-expanded', 'false')
+
+    cy.screenshot('filter-panel-3-big-filter-collapsed', {
+      capture: 'viewport'
+    })
+
+    // expand: the whole clause is rendered, on several lines, unclipped
+    cy.get('.infoBar-toggle').click()
+    cy.get('.infoBar').should('have.class', 'expanded')
+    cy.get('.infoBar-toggle').should('have.attr', 'aria-expanded', 'true')
+    assertClauseFullyVisible()
+    assertClauseSpansSeveralLines()
+
+    cy.screenshot('filter-panel-3-big-filter-expanded', { capture: 'viewport' })
+
+    // collapse again
+    cy.get('.infoBar-toggle').click()
+    cy.get('.infoBar').should('not.have.class', 'expanded')
+    assertClauseOverflowsCollapsedLine()
+  })
+
+  it('4 | the editor panel collapses and expands in the same way', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    openFilterPopup()
+    setFilterVariable('SOME_CHAR')
+    setFilterOperator('IN')
+    chooseAllFilterValues()
+    submitFilter()
+
+    // the editor shows the filter the viewer just stored
+    cy.url().then((url) => {
+      const filterId = url.split('/').pop()
+      cy.visit(
+        `${hostUrl}${appLocation}/#/editor/DC_JSLIB.MPE_X_TEST/${filterId}`,
+        { timeout: longerCommandTimeout }
+      )
+    })
+
+    // wait for the editor itself - .editor-title/.btnCtrl are editor-only, so
+    // this cannot pass on the viewer the visit came from
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should('exist')
+    cy.get('.btnCtrl', { timeout: longerCommandTimeout }).should('exist')
+    cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'SOME_CHAR IN'
+    )
+    assertClauseOverflowsCollapsedLine()
+
+    cy.screenshot('filter-panel-4-editor-collapsed', { capture: 'viewport' })
+
+    cy.get('.infoBar-toggle').click()
+    cy.get('.infoBar').should('have.class', 'expanded')
+    assertClauseFullyVisible()
+    assertClauseSpansSeveralLines()
+
+    cy.screenshot('filter-panel-4-editor-expanded', { capture: 'viewport' })
+  })
+})
+
+/**
+ * Cypress clears cookies between tests, which drops the SASjs Server session
+ * and lands the app back on the evaluation agreement card. Accept it when it is
+ * shown - a no-op when the session survived.
+ */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: longerCommandTimeout })
+  cy.get('body').then(($body: any) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Opens a table in the viewer and waits for its grid to render. */
+const openTable = (libMem: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/view/data/${libMem}`, {
+    timeout: longerCommandTimeout
+  })
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).should('exist')
+  cy.wait(2500)
+}
+
+const openFilterPopup = () => {
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).click()
+  cy.get('clr-dropdown-menu', { timeout: longerCommandTimeout })
+    .contains('Filter')
+    .click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/**
+ * The soft-select inputs drop a transparent .overlay click-catcher over the
+ * modal while their suggestion list is open, so they need force.
+ */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterOperator = (operator: string) => {
+  cy.get('.filter-modal .operator-col select').first().select(operator)
+  cy.wait(500)
+}
+
+/** IN/NOT IN take their values from a modal of checkboxes - take them all. */
+const chooseAllFilterValues = () => {
+  cy.contains('.filter-modal button', 'Choose values').click()
+  cy.get('.in-values-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+  cy.get('.in-values-modal input[type=checkbox]').then(($checkboxes: any) => {
+    for (let i = 0; i < $checkboxes.length; i++) {
+      cy.get('.in-values-modal input[type=checkbox]')
+        .eq(i)
+        .click({ force: true })
+    }
+  })
+  cy.contains('.in-values-modal button', 'Apply').click()
+  cy.wait(500)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+/** The clause is wider than the collapsed line, so it is clipped. */
+const assertClauseOverflowsCollapsedLine = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    expect(
+      el.scrollWidth,
+      'the clause is wider than the collapsed line'
+    ).to.be.greaterThan(el.clientWidth)
+    expect(
+      el.ownerDocument.defaultView!.getComputedStyle(el).whiteSpace,
+      'collapsed panel does not wrap'
+    ).to.equal('nowrap')
+  })
+}
+
+/** The clause fits the collapsed line, so nothing is hidden. */
+const assertClauseFitsCollapsedLine = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    expect(el.scrollWidth, 'the clause fits the collapsed line').to.be.at.most(
+      el.clientWidth
+    )
+  })
+}
+
+/** Expanded, the whole clause is rendered and nothing is clipped. */
+const assertClauseFullyVisible = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    const style = el.ownerDocument.defaultView!.getComputedStyle(el)
+    expect(style.whiteSpace, 'expanded panel wraps').to.equal('normal')
+    expect(el.scrollWidth, 'expanded panel clips nothing').to.be.at.most(
+      el.clientWidth
+    )
+  })
+}
+
+/** Expanded, the clause occupies several lines rather than one. */
+const assertClauseSpansSeveralLines = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    const lineHeight =
+      parseFloat(
+        el.ownerDocument.defaultView!.getComputedStyle(el).lineHeight
+      ) || 20
+    expect(
+      el.clientHeight / lineHeight,
+      'the clause spans several lines'
+    ).to.be.greaterThan(3)
+  })
+}

client/cypress/e2e/filtering.cy.ts

@@ -11,6 +11,18 @@ context('filtering tests: ', function () {
   this.beforeAll(() => {
     cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
     cy.loginAndUpdateValidKey()
+
+    /**
+     * The evaluation agreement is per-origin localStorage, so a fresh browser
+     * profile lands on it and the nav tree never renders.  Guarded, so it is a
+     * no-op once an earlier spec in the run has accepted it.
+     */
+    cy.get('body').then(($body) => {
+      if ($body.find('#TCS input[type="checkbox"]').length) {
+        cy.get('#TCS input[type="checkbox"]').check({ force: true })
+        cy.wait(4000)
+      }
+    })
   })
 
   this.beforeEach(() => {
@@ -51,9 +63,13 @@ context('filtering tests: ', function () {
 
     openFilterPopup(() => {
       setFilterWithValue('SOME_TIME', '00:00:42', 'time', () => {
-        checkInfoBarIncludes(`(( SOME_TIME = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_TIME = '00:00:42't ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(3, () => done())
+          }
+        )
       })
     })
   })
@@ -63,9 +79,13 @@ context('filtering tests: ', function () {
 
     openFilterPopup(() => {
       setFilterWithValue('SOME_TIME', '42', 'value', () => {
-        checkInfoBarIncludes(`(( SOME_TIME = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_TIME = '00:00:42't ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(3, () => done())
+          }
+        )
       })
     }, false)
   })
@@ -75,9 +95,13 @@ context('filtering tests: ', function () {
 
     openFilterPopup(() => {
       setFilterWithValue('SOME_DATE', '12/02/1960', 'date', () => {
-        checkInfoBarIncludes(`(( SOME_DATE = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_DATE = '12FEB1960'd ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
       })
     })
   })
@@ -87,9 +111,13 @@ context('filtering tests: ', function () {
 
     openFilterPopup(() => {
       setFilterWithValue('SOME_DATE', '42', 'value', () => {
-        checkInfoBarIncludes(`(( SOME_DATE = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_DATE = '12FEB1960'd ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
       })
     }, false)
   })
@@ -104,9 +132,10 @@ context('filtering tests: ', function () {
         'datetime',
         () => {
           checkInfoBarIncludes(
-            `(( SOME_DATETIME = 42 ))`,
+            `(( SOME_DATETIME = '01JAN1960:00:00:42'dt ))`,
             (includes: boolean) => {
-              if (includes) done()
+              if (!includes) return
+              assertGridRowCount(5, () => done())
             }
           )
         }
@@ -120,9 +149,10 @@ context('filtering tests: ', function () {
     openFilterPopup(() => {
       setFilterWithValue('SOME_DATETIME', '42', 'value', () => {
         checkInfoBarIncludes(
-          `(( SOME_DATETIME = 42 ))`,
+          `(( SOME_DATETIME = '01JAN1960:00:00:42'dt ))`,
           (includes: boolean) => {
-            if (includes) done()
+            if (!includes) return
+            assertGridRowCount(5, () => done())
           }
         )
       })
@@ -134,9 +164,13 @@ context('filtering tests: ', function () {
 
     openFilterPopup(() => {
       setFilterWithValue('SOME_DATE', '', 'in', () => {
-        checkInfoBarIncludes(`(( SOME_DATE IN (42) ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_DATE IN ('12FEB1960'd) ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
       })
     })
   })
@@ -156,6 +190,34 @@ context('filtering tests: ', function () {
   //     })
   //   })
   // })
+
+  /**
+   * With the pickers off, a temporal BETWEEN takes both of its values from a
+   * soft-select over the column's values, so the value is chosen from that list
+   * rather than typed.  The range dropdown binds the formatted label, as the
+   * single-value dropdown's list does, so the field reads the date and the
+   * clause composition maps it back to the internal day / second count before
+   * the literal is built - a display string that reached sasTemporalLiteral as
+   * a non-number would come back null and be passed through to RAW_VALUE
+   * unquoted, which mp_filtercheck rejects on a numeric column and aborts the
+   * save.  The test asserts the applied clause and the row count, so it pins
+   * the mapping end to end.
+   */
+  it('7 | Non picker - filter date field BETWEEN from the value list', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    openFilterPopup(() => {
+      setTemporalRangeFromValueList('SOME_DATE', '12FEB1960', () => {
+        checkInfoBarIncludes(
+          `BETWEEN '12FEB1960'd AND '12FEB1960'd`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
+      })
+    }, false)
+  })
 })
 
 const checkInfoBarIncludes = (text: string, callback: any) => {
@@ -166,6 +228,21 @@ const checkInfoBarIncludes = (text: string, callback: any) => {
   })
 }
 
+/**
+ * The temporal filter tests assert the applied filter still selects the same
+ * rows as the internal value did - a readable clause that filtered differently
+ * would be a worse bug than the one being fixed.
+ */
+const assertGridRowCount = (expected: number, callback: any) => {
+  cy.get('#hotTable .ht_master tbody tr', {
+    timeout: longerCommandTimeout
+  })
+    .should('have.length', expected)
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
 const openFilterPopup = (
   callback?: any,
   usePickers: boolean = true,
@@ -331,6 +408,54 @@ const setFilterWithValue = (
   if (callback) callback()
 }
 
+/**
+ * Drives a temporal BETWEEN with the pickers off, choosing both values from the
+ * range soft-select's list rather than typing them - typing bypasses the
+ * binding this test exists to pin.
+ */
+const setTemporalRangeFromValueList = (
+  variableValue: string,
+  formatted: string,
+  callback?: any
+) => {
+  cy.wait(600)
+
+  cy.focused().type(variableValue)
+  cy.wait(100)
+  cy.get('.variable-col .autocomplete-wrapper', { withinSubject: null })
+    .first()
+    .trigger('keydown', { key: 'ArrowDown' })
+  cy.get('.variable-col .autocomplete-wrapper', {
+    withinSubject: null
+  }).trigger('keydown', { key: 'Enter' })
+  cy.focused().tab()
+  cy.wait(100)
+
+  cy.focused().select('BETWEEN').trigger('change')
+  cy.wait(400)
+
+  const chooseFromList = (range: 'start' | 'end') => {
+    const inputId = `vals-${range}_0_0`
+
+    cy.get(`#${inputId}`)
+      .clear({ force: true })
+      .type(formatted, { force: true })
+    cy.get(`#datalist_${inputId} option`)
+      .contains(formatted)
+      .click({ force: true })
+    cy.get(`#${inputId}`).trigger('keyup', { key: 'Escape', force: true })
+    cy.wait(300)
+  }
+
+  chooseFromList('start')
+  chooseFromList('end')
+
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+
+  if (callback) callback()
+}
+
 const openTableFromTree = (libNameIncludes: string, tablename: string) => {
   cy.get('.app-loading', { timeout: longerCommandTimeout })
     .should('not.exist')

client/cypress/e2e/hook-programs.cy.ts

@@ -0,0 +1,209 @@
+// Marks this file as an ES module (rather than a global script) so its
+// top-level consts don't collide, under the TS type-checker, with the same
+// names declared in other spec files.
+//
+// RUN ORDER MATTERS, and build.yaml reflects it: this spec must run after
+// csv-limited.cy.ts (which asserts the free tier and so needs the first slot,
+// before any spec applies a licence key) and before every spec that submits a
+// changeset. Test 2 reads the approval queue, which is paged oldest first, and
+// a changeset submitted by an earlier spec pushes this one off the first page.
+export {}
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('hook program tests: ', function () {
+  this.beforeAll(() => {
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    visitPage('home')
+  })
+
+  // TESTDATA.DEMO_MIRROR is an empty mirror of TESTDATA.DEMO_ORDERS, registered
+  // with a PRE_EDIT_HOOK and a POST_EDIT_HOOK (services/hooks/demo_mirror_*.js).
+  // The mock services run those hooks the way the SAS backend %includes the
+  // real hook programs - see dcMockUtils.mockHookSource.
+
+  it('1 | PRE_EDIT_HOOK: an empty mirror displays the live rows of its target table', () => {
+    openTableFromTree('testdata', 'demo_mirror')
+
+    // The mirror holds no rows of its own: everything on screen comes from the
+    // pre-edit hook, which loads TESTDATA.DEMO_ORDERS.
+    cy.get('#hotTable .ht_master tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 4)
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .first()
+      .should('contain.text', 'Acme Corp')
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .last()
+      .should('contain.text', 'Delta Systems')
+
+    // The grid is validated against the MIRROR's own rule set (AMOUNT <= 2000),
+    // not the real table's (AMOUNT <= 100000) - that is the point of routing
+    // edits through a mirror.
+    clickOnEdit(() => {
+      editCell(0, 'AMOUNT', '5000')
+    })
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .first()
+      .find('td.htInvalid')
+      .should('exist')
+  })
+
+  it('2 | POST_EDIT_HOOK: a change submitted against the mirror is raised against the target', () => {
+    openTableFromTree('testdata', 'demo_mirror')
+
+    clickOnEdit(() => {
+      editCell(0, 'AMOUNT', '750')
+    })
+
+    submitTable(() => {
+      cy.get('textarea.submit-reason', { timeout: longerCommandTimeout }).type(
+        'hook routing test'
+      )
+      submitTableMessage()
+    })
+
+    // A successful submit sends the app to the staged-data page; wait for that
+    // before navigating away, or the app's own redirect wins the race.
+    cy.url({ timeout: longerCommandTimeout }).should('include', '/stage/')
+
+    // The post-edit hook re-points the changeset at TESTDATA.DEMO_ORDERS, so the
+    // approval queue names the real table rather than the mirror. Search the
+    // whole grid rather than one row: the queue also holds changesets from
+    // earlier specs in the same run, so position is not a stable anchor.
+    visitPage('review/approve')
+
+    cy.get('clr-datagrid clr-dg-row', { timeout: longerCommandTimeout }).should(
+      ($rows) => {
+        const texts = Array.from($rows).map((row: any) =>
+          row.innerText.trim().replace(/\s+/g, ' ')
+        )
+
+        expect(
+          texts.some((text) => text.includes('TESTDATA.DEMO_ORDERS')),
+          `no approval row names TESTDATA.DEMO_ORDERS - rows: ${texts
+            .slice(0, 5)
+            .join(' | ')
+            .slice(0, 400)}`
+        ).to.be.true
+      }
+    )
+  })
+})
+
+// ─── helpers (declared at the bottom, per the other specs) ───────────────────
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let viyaLib
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            viyaLib = node
+            break
+          }
+        }
+
+        cy.get(viyaLib).within(() => {
+          cy.wait(300)
+
+          cy.get(
+            '.clr-tree-node-content-container .clr-treenode-content p'
+          ).click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+
+  cy.get('#hotTable .ht_clone_top .htCore thead button.changeType', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+}
+
+const clickOnEdit = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+// Edits a single cell, locating the column by its header label. Handsontable
+// renders the frozen header in a separate clone pane (.ht_clone_top) - the
+// header row inside .ht_master is kept visibility:hidden - so the labels are
+// read from the clone.
+const editCell = (rowIndex: number, colName: string, value: string) => {
+  cy.get('#hotTable .ht_clone_top .htCore thead th', {
+    timeout: longerCommandTimeout
+  }).then((headers: any) => {
+    let colIndex = -1
+    for (let i = 0; i < headers.length; i++) {
+      if (
+        String(headers[i].innerText || '')
+          .trim()
+          .toUpperCase() === colName.toUpperCase()
+      ) {
+        colIndex = i
+        break
+      }
+    }
+    expect(colIndex, `column ${colName} present`).to.be.greaterThan(-1)
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .eq(rowIndex)
+      .then((row: any) => {
+        cy.get(row[0].childNodes[colIndex])
+          .dblclick({ force: true })
+          .then(() => {
+            cy.focused()
+              .clear()
+              .type(value + '{enter}')
+          })
+      })
+  })
+}
+
+const submitTable = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+const submitTableMessage = (callback?: any) => {
+  cy.get('.modal-footer .btn.btn-sm.btn-success-outline', {
+    timeout: longerCommandTimeout
+  })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}

client/cypress/e2e/row-cell-limits.cy.ts

@@ -0,0 +1,415 @@
+// The EDIT screen refuses a selection that is too large - DC_MAXOBS_WEBEDIT
+// rows, or DC_MAXCELLS_WEBEDIT cells (rows x columns), whichever is reached
+// first - and the VIEW screen caps at DC_MAXOBS_WEBVIEW rows.
+//
+// This spec pins those limits against the mock, which mirrors the shipped
+// defaults (250 rows / 200000 cells for EDIT, 2000 rows for VIEW).  It uses
+// two existing fixtures rather than new ones:
+//   MPE_X_SEARCH - 1000 rows x 9 cols, so it trips the ROW limit
+//   MPE_X_WIDE   - 200 rows x 1001 cols (200,400 cells), under the row limit
+//                  but over the CELL limit
+export {}
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+const serverType = Cypress.env('serverType')
+const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
+
+context('EDIT row and cell limits, VIEW row limit: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`)
+    // forceLicenceKey=true: applies a valid licence rather than relying on an
+    // earlier spec in the run having done so (see commands.ts).
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+    visitPage('home')
+    visitPage('view/data')
+  })
+
+  it('1 | VIEW renders up to the raised row cap, past the old 500', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
+
+    // Neither the header nor the cell text can pin the cap: the header reports
+    // the table's true row count whether or not the grid is capped, and the
+    // fixture's own numeric columns carry values above 1500 (DEPTH_M reaches
+    // 4293, SAMPLE_COUNT 4210), so a text match can hit fixture data.
+    //
+    // The PK can pin it.  The first td of a .ht_master row is the PK - the row
+    // header, where present, is a th - and MPE_X_SEARCH holds PK 1001-2000, so
+    // the range check below is what stops a mis-selected column passing.  With
+    // the old 500 row cap the grid stopped at PK 1500, so a rendered PK above
+    // 1500 can only appear if the cap was raised.
+    scrollGridToBottom()
+    cy.get('#hotTable .ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should(($rows) => {
+      const pks = $rows
+        .toArray()
+        .map((row) => row.querySelectorAll('td')[0])
+        .filter((td) => !!td)
+        .map((td) => parseInt((td.textContent || '').trim(), 10))
+        .filter((pk) => !isNaN(pk))
+
+      expect(pks.length).to.be.greaterThan(0)
+      pks.forEach((pk) => expect(pk).to.be.within(1001, 2000))
+      expect(Math.max(...pks)).to.be.greaterThan(1500)
+    })
+  })
+
+  it('2 | EDIT refuses a table over the row limit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
+    openEdit()
+
+    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'Table is too big (1000 rows) - please filter and try again!'
+    )
+  })
+
+  it('3 | EDIT refuses a table over the cell limit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_wide')
+    openEdit()
+
+    // 200 rows x 1002 columns (the 1001 stored columns plus the housekeeping
+    // delete column) = 200,400 cells, over the 200,000 cell limit, while the
+    // row count is under the 250 row limit.
+    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'Selection is too wide (200 rows x 1002 cols) - please filter and try again!'
+    )
+  })
+
+  it('4 | the EDIT limits follow MPE_CONFIG, not the shipped defaults', () => {
+    // Two things at once.  The seeded config carries the shipped defaults (250
+    // rows / 200000 cells), which are the numbers the mock used to hard-code -
+    // so nothing above would notice if the mock stopped reading MPE_CONFIG at
+    // all.  And MPE_CONFIG lives in the control library, so reading it from the
+    // table's own library would fail for this table in particular.
+    //
+    // The cell limit is the one to move: 7 rows x 18 columns is 126 cells, so
+    // at 110 the TESTDATA.MPE_X_NEW edit must be refused, where a hard-coded
+    // 200000 - or a config read that fails and falls back to it - lets it
+    // through.  110 is still above MPE_CONFIG's own 17 x 6 = 102, so the option
+    // can be put back afterwards.
+    setConfigValue('DC_MAXCELLS_WEBEDIT', '110', () => {
+      visitPage('view/data/TESTDATA.MPE_X_NEW')
+      closeAbortModalIfPresent()
+      openEdit()
+
+      cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+        'contain.text',
+        'Selection is too wide (7 rows x 18 cols) - please filter and try again!'
+      )
+    })
+
+    // Put the option back, so the rest of the estate sees the shipped default.
+    setConfigValue('DC_MAXCELLS_WEBEDIT', '200000', () => {
+      visitPage('view/data/TESTDATA.MPE_X_NEW')
+      closeAbortModalIfPresent()
+      openEdit()
+
+      // 126 cells under a 200000 cell limit: the editor loads, no abort.
+      cy.get('#hotTable .ht_master .htCore tbody tr', {
+        timeout: longerCommandTimeout
+      }).should('exist')
+    })
+  })
+
+  it('5 | the filter picker still lists the variables after the abort', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
+    openEdit()
+
+    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'Table is too big (1000 rows) - please filter and try again!'
+    )
+
+    closeAbortModal()
+
+    // The abort returns no table, and the picker's variable list normally comes
+    // from the table's own columns - so without the public/getcols fallback the
+    // "please filter and try again" advice has nothing to pick from.  Every
+    // column of MPE_X_SEARCH must be on offer.
+    openFilterPopup()
+
+    filterVariables().should(($options) => {
+      const names = $options
+        .toArray()
+        .map((o) => (o.textContent || '').trim())
+        .filter((n) => n.length > 0)
+
+      expect(names, 'the picker offers the table columns').to.include.members([
+        'PRIMARY_KEY_FIELD',
+        'SITE_NAME',
+        'VESSEL',
+        'SAMPLE_COUNT',
+        'DEPTH_M',
+        'SPECIES',
+        'CRUISE_DATE',
+        'EXPEDITION_ID',
+        'NOTES'
+      ])
+    })
+
+    // and the suggestion list is usable, not just populated
+    setFilterVariable('SITE_NAME')
+
+    // Submitting is what exercises TYPE.  The picker keys its operator set and
+    // its value quoting on it, and the fallback list never passes through
+    // mergeColsRules, so without TYPE on this payload a character value would be
+    // submitted unquoted and the backend's %mp_filtercheck would reject it.
+    setFilterValue('Bristol')
+    submitFilter()
+
+    cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      "SITE_NAME = 'Bristol'"
+    )
+  })
+})
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}
+
+// The viewer's options dropdown holds the Edit item; it only renders for a
+// table that is registered in MPE_TABLES (tableEditExists()).
+const openEdit = () => {
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).click()
+  cy.contains('[clrDropdownItem]', 'Edit', {
+    timeout: longerCommandTimeout
+  }).click()
+}
+
+/** The abort modal offers Close, and the filter button sits behind it. */
+const closeAbortModal = () => {
+  cy.get('app-info-modal', { timeout: longerCommandTimeout })
+    .contains('button', 'Close')
+    .click()
+}
+
+/**
+ * The EDIT screen's Filter button opens the same .filter-modal the viewer uses.
+ * It only renders once the editor is in its read-only (non-editing) state,
+ * which is where a failed EDIT load leaves it.
+ */
+const openFilterPopup = () => {
+  cy.contains('button.btnView', 'Filter', {
+    timeout: longerCommandTimeout
+  }).click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/** The picker's variable suggestion list. */
+const filterVariables = () =>
+  cy.get('#datalist_vals_var_id0_0 option', { timeout: longerCommandTimeout })
+
+/** The soft-select drops a transparent overlay over the modal, so it needs force. */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+}
+
+/** The value input of the same soft-select row. */
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+/**
+ * Sets a DC option in MPE_CONFIG through the editor and approves it, then hands
+ * back to the callback.  The same route licensing.cy.ts uses for the licence
+ * keys: the config table needs an approval like any other.
+ */
+const setConfigValue = (
+  varName: string,
+  varValue: string,
+  callback: () => void
+) => {
+  reloadApp()
+  // straight to the config table's editor, so the flow does not depend on
+  // whichever view mode the previous step left behind
+  visitPage('editor/DC_JSLIB.MPE_CONFIG')
+  closeAbortModalIfPresent()
+  clickOnEdit()
+
+  editTableField([{ varName, varValue }], () => {
+    submitTable(() => {
+      cy.wait(2000)
+      approveTable(() => {
+        reloadApp()
+        callback()
+      })
+    })
+  })
+}
+
+/**
+ * A full page load.  A visit that only changes the hash does not reload the
+ * app, so an open modal - the abort message from the step just gone, say -
+ * would still be covering the page.
+ */
+const reloadApp = () => {
+  cy.visit(hostUrl + appLocation)
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+}
+
+/**
+ * DC puts abort messages up as modals over whatever page is showing, and they
+ * outlive a hash-only navigation, so clear one before driving the page.
+ */
+const closeAbortModalIfPresent = () => {
+  cy.get('body').then(($body) => {
+    if ($body.find('app-info-modal').length > 0) {
+      cy.get('app-info-modal')
+        .contains('button', 'Close')
+        .click({ force: true })
+    }
+  })
+}
+
+/**
+ * The viewer's Edit button.  The options dropdown (.filterSide) that openEdit()
+ * uses is not rendered for every table, so the config table is opened the way
+ * licensing.cy.ts opens it.
+ */
+const clickOnEdit = (callback?: () => void) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+/** Types a new value into the row whose cell carries varName. */
+const editTableField = (
+  edits: { varName: string; varValue: string }[],
+  callback: () => void
+) => {
+  cy.get('#hotTable .ht_master .htCore tbody td', {
+    timeout: longerCommandTimeout
+  }).then(($tds: any) => {
+    const nodes = $tds.toArray()
+
+    for (const edit of edits) {
+      for (let i = 0; i < nodes.length; i++) {
+        // exact match on the cell that holds the option name; the cell after it
+        // is the value
+        if ((nodes[i].textContent || '').trim() === edit.varName) {
+          cy.wrap(nodes[i + 1])
+            .dblclick()
+            .then(() => {
+              cy.focused()
+                .type('{selectall}')
+                .type(edit.varValue)
+                .type('{enter}')
+            })
+          break
+        }
+      }
+    }
+
+    callback()
+  })
+}
+
+const submitTable = (callback: () => void) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      cy.get(".modal.ng-star-inserted button[type='submit']").click()
+      callback()
+    })
+}
+
+const approveTable = (callback: () => void) => {
+  cy.get('button', { timeout: longerCommandTimeout })
+    .should('contain', 'Approve')
+    .then((allButtons: any) => {
+      for (const approvalButton of allButtons) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
+          approvalButton.click()
+          break
+        }
+      }
+
+      cy.get('button#acceptBtn', { timeout: longerCommandTimeout })
+        .should('exist')
+        .should('not.be.disabled')
+        .click()
+        .then(() => {
+          cy.get('app-history', { timeout: longerCommandTimeout }).should(
+            'exist'
+          )
+          callback()
+        })
+    })
+}
+
+// Handsontable virtualises rows, so the last row is only in the DOM once the
+// grid has been scrolled to the bottom.  The scroller is the .wtHolder inside
+// the .ht_master pane.
+const scrollGridToBottom = () => {
+  cy.get('#hotTable .ht_master .wtHolder', { timeout: longerCommandTimeout })
+    .scrollTo('bottom', { duration: 0 })
+    .trigger('scroll')
+}
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let targetLib
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            targetLib = node
+            break
+          }
+        }
+
+        cy.get(targetLib).within(() => {
+          // Small settle wait: right after a fresh visit the tree component can
+          // still be re-rendering, causing this node to be found then swapped
+          // out mid-click.
+          cy.wait(300)
+
+          cy.get(
+            '.clr-tree-node-content-container .clr-treenode-content p'
+          ).click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+
+  cy.get('#hotTable .ht_clone_top .htCore thead button.changeType', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+}

client/licenseChecker.js

@@ -10,7 +10,7 @@ const check = (cwd) => {
         onlyAllow:
           'AFLv2.1;Apache 2.0;Apache-2.0;Apache*;Artistic-2.0;BlueOak-1.0.0;0BSD;BSD*;BSD-2-Clause;BSD-3-Clause;CC0-1.0;CC-BY-3.0;CC-BY-4.0;ISC;MIT;MPL-2.0;ODC-By-1.0;Python-2.0;Unlicense;',
         excludePackages:
-          '@cds/city@1.1.0;@handsontable/angular-wrapper@16.0.1;@handsontable/angular-wrapper@17.1.0;@handsontable/angular-wrapper@18.0.0;handsontable@^16.0.1;handsontable@16.2.0;handsontable@17.1.0;handsontable@18.0.0;hyperformula@2.7.1;hyperformula@3.0.0;hyperformula@3.1.0;hyperformula@3.2.0;hyperformula@3.3.0;jackspeak@3.4.3;path-scurry@1.11.1;package-json-from-dist@1.0.1;buffers@0.1.1'
+          '@cds/city@1.1.0;@handsontable/angular-wrapper@16.0.1;@handsontable/angular-wrapper@17.1.0;@handsontable/angular-wrapper@18.0.0;@handsontable/angular-wrapper@18.1.1;handsontable@^16.0.1;handsontable@16.2.0;handsontable@17.1.0;handsontable@18.0.0;handsontable@18.1.1;hyperformula@2.7.1;hyperformula@3.0.0;hyperformula@3.1.0;hyperformula@3.2.0;hyperformula@3.3.0;jackspeak@3.4.3;path-scurry@1.11.1;package-json-from-dist@1.0.1;buffers@0.1.1'
       },
       (error, json) => {
         if (error) {

client/package.json

@@ -41,22 +41,22 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^20.3.27",
+    "@angular/animations": "20.3.33",
     "@angular/cdk": "^20.2.14",
-    "@angular/common": "^20.3.27",
-    "@angular/compiler": "^20.3.27",
-    "@angular/core": "^20.3.27",
-    "@angular/forms": "^20.3.27",
-    "@angular/platform-browser": "^20.3.27",
-    "@angular/platform-browser-dynamic": "^20.3.27",
-    "@angular/router": "^20.3.27",
+    "@angular/common": "20.3.33",
+    "@angular/compiler": "20.3.33",
+    "@angular/core": "20.3.33",
+    "@angular/forms": "20.3.33",
+    "@angular/platform-browser": "20.3.33",
+    "@angular/platform-browser-dynamic": "20.3.33",
+    "@angular/router": "20.3.33",
     "@cds/core": "^6.15.1",
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
     "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
-    "@handsontable/angular-wrapper": "18.0.0",
-    "@sasjs/adapter": "^4.18.0",
-    "@sasjs/utils": "^3.6.0",
+    "@handsontable/angular-wrapper": "18.1.1",
+    "@sasjs/adapter": "5.1.0",
+    "@sasjs/utils": "4.0.0",
     "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
     "@types/d3-graphviz": "^2.6.7",
     "@types/text-encoding": "0.0.35",
@@ -69,12 +69,11 @@
     "d3-graphviz": "^5.0.2",
     "exceljs": "^4.4.0",
     "fs-extra": "^7.0.1",
-    "handsontable": "18.0.0",
+    "handsontable": "18.1.1",
     "https-browserify": "1.0.0",
     "hyperformula": "^2.5.0",
     "iconv-lite": "^0.5.0",
     "jquery-datetimepicker": "^2.5.21",
-    "jsrsasign": "11.1.1",
     "marked": "^5.0.0",
     "moment": "^2.30.1",
     "ngx-clipboard": "^16.0.0",
@@ -99,8 +98,8 @@
     "@angular-eslint/eslint-plugin-template": "19.8.1",
     "@angular-eslint/schematics": "19.8.1",
     "@angular-eslint/template-parser": "19.8.1",
-    "@angular/cli": "^20.3.32",
-    "@angular/compiler-cli": "^20.3.27",
+    "@angular/cli": "20.3.37",
+    "@angular/compiler-cli": "20.3.33",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@compodoc/compodoc": "^2.0.0",
     "@cypress/webpack-preprocessor": "^5.17.1",
@@ -148,7 +147,7 @@
     "ajv": "8.18.0",
     "uuid": "11.1.1",
     "lighthouse": "13.4.0",
-    "fast-uri": "3.1.7",
+    "fast-uri": "3.1.8",
     "readdir-glob": {
       "brace-expansion": "^5.0.9"
     },

client/src/_hot-icons.scss

@@ -3,8 +3,7 @@
 
 [class*=ht-theme-classic] .htDropdownMenu table tbody tr td.htSubmenu .htItemWrapper::after,
 [class*=ht-theme-classic] .htContextMenu table tbody tr td.htSubmenu .htItemWrapper::after,
-[class*=ht-theme-classic] .htFiltersConditionsMenu table tbody tr td.htSubmenu .htItemWrapper::after,
-[class*=ht-theme-classic] .pika-single .pika-next {
+[class*=ht-theme-classic] .htFiltersConditionsMenu table tbody tr td.htSubmenu .htItemWrapper::after {
   width: var(--ht-icon-size);
   height: var(--ht-icon-size);
   -webkit-mask-size: contain;
@@ -12,14 +11,6 @@
   background-color: currentColor;
 }
 
-[class*=ht-theme-classic] .pika-single .pika-prev {
-  width: var(--ht-icon-size);
-  height: var(--ht-icon-size);
-  -webkit-mask-size: contain;
-  -webkit-mask-image: url("./assets/hot-icons/arrow-left.svg");
-  background-color: currentColor;
-}
-
 [class*=ht-theme-classic] .ht-page-size-section__select-wrapper::after {
   width: var(--ht-icon-size);
   height: var(--ht-icon-size);

client/src/app/app.component.html

@@ -318,6 +318,7 @@
             class="startup-check"
             [class.startup-check--error]="step.status === 'error'"
             [class.startup-check--done]="step.status === 'done'"
+            [class.startup-check--grid]="step.details?.length"
           >
             @switch (step.status) {
               @case ('pending') {
@@ -346,6 +347,18 @@
             @if (step.detail) {
               <span class="startup-check__detail">{{ step.detail }}</span>
             }
+            @if (step.details?.length) {
+              <div class="startup-check__grid">
+                @for (detail of step.details; track detail.name) {
+                  <span class="startup-check__grid-name">{{
+                    detail.name
+                  }}</span>
+                  <span class="startup-check__grid-value">{{
+                    detail.value
+                  }}</span>
+                }
+              </div>
+            }
           </div>
         }
       </div>

client/src/app/app.component.ts

@@ -20,6 +20,8 @@ import { DcAdapterSettings } from './models/DcAdapterSettings'
 import { AppStoreService } from './services/app-store.service'
 import { LicenceService } from './services/licence.service'
 import {
+  getSasjsTagDetails,
+  SASJS_TAG_STEP,
   StartupCheckService,
   StartupStep
 } from './services/startup-check.service'
@@ -190,40 +192,45 @@ export class AppComponent {
    * Parses adapter settings that are found in the <sasjs> tag inside index.html
    */
   private parseDcAdapterSettings() {
+    /* The <sasjs> tag is a mandatory part of index.html - it carries serverUrl,
+      serverType and the rest of the adapter config - so there is no config to
+      fall back on when it is absent. A missing one is reported on the boot
+      screen below rather than handled with an abort modal. */
     const sasjsElement = document.querySelector('sasjs')
 
-    if (!sasjsElement) {
-      this.licenceService.deactivateApp()
-      setTimeout(() => {
-        this.eventService.showAbortModal(
-          null,
-          "Please make sure 'SASJS' tag with config attributes is added to index.html",
-          null,
-          'SASjs Config not found'
-        )
-      })
-      return
-    }
-
     const getAppAttribute = (attribute: string) =>
-      sasjsElement.getAttribute(attribute) || undefined
+      sasjsElement?.getAttribute(attribute) || undefined
 
     const dcAdapterSettings = {
       serverUrl: getAppAttribute('serverUrl') || '',
       appLoc: getAppAttribute('appLoc') || '',
       serverType: getAppAttribute('serverType'),
       loginMechanism: getAppAttribute('loginMechanism') || '',
-      adminGroup: getAppAttribute('adminGroup') || '',
       dcPath: getAppAttribute('dcPath') || '',
       debug: getAppAttribute('debug') === 'true' || false,
       useComputeApi: this.parseComputeApi(getAppAttribute('useComputeApi')),
       runAsTask: getAppAttribute('runAsTask') === 'true' || false,
-      contextName: getAppAttribute('contextName') || '',
-      hotLicenceKey: getAppAttribute('hotLicenceKey') || ''
+      contextName: getAppAttribute('contextName') || ''
     }
 
     this.dcAdapterSettings = dcAdapterSettings as any
     this.appStoreService.setDcAdapterSettings(dcAdapterSettings as any)
+
+    /* Show the <sasjs> tag contents on the boot screen, so the effective
+      configuration (serverType, debug, useComputeApi, ...) is visible without
+      having to read the page source. The tag is mandatory - without it the
+      adapter has no serverUrl or serverType - so a missing one is called out
+      plainly rather than silently leaving the step pending. */
+    if (sasjsElement) {
+      this.startupCheckService.setTagDetails(getSasjsTagDetails(sasjsElement))
+    } else {
+      this.startupCheckService.updateStep(
+        SASJS_TAG_STEP,
+        'error',
+        'SASJS TAG MISSING!!'
+      )
+    }
+
     this.appService.sasServiceInit()
   }
 

client/src/app/deploy/deploy-routing.module.ts

@@ -2,10 +2,7 @@ import { NgModule } from '@angular/core'
 import { RouterModule, Routes } from '@angular/router'
 import { DeployComponent } from './deploy.component'
 
-const routes: Routes = [
-  { path: '', component: DeployComponent },
-  { path: 'manualdeploy', component: DeployComponent }
-]
+const routes: Routes = [{ path: '', component: DeployComponent }]
 
 @NgModule({
   imports: [RouterModule.forChild(routes)],

client/src/app/deploy/deploy.component.html

@@ -29,20 +29,6 @@
               ></label
             >
           </clr-checkbox-wrapper>
-          <!-- <hr />
-                <div class="clr-checkbox-wrapper">
-                  <input
-                    [(ngModel)]="autodeploy"
-                    type="checkbox"
-                    id="checkbox2"
-                    class="clr-checkbox"
-                    checked
-                    />
-                  <label for="checkbox2"
-                    >Autodeploy
-                    {{ !jsonFile ? '(json file is not available)' : '' }}</label
-                    >
-                  </div> -->
         </div>
       </div>
     }
@@ -52,29 +38,17 @@
   @if (step > 0) {
     @switch (true) {
       @case (sasJsConfig.serverType === ServerType.SasViya) {
-        @if (autodeploy) {
-          <div class="autodeploy-section card">
-            <app-automatic-deploy
-              [sasJs]="sasJs"
-              [sasJsConfig]="sasJsConfig"
-              [dcAdapterSettings]="dcAdapterSettings"
-              [appLoc]="appLoc"
-              [dcPath]="dcPath"
-              [selectedAdminGroup]="selectedAdminGroup"
-              (onNavigateToHome)="onNavigateToHome()"
-            ></app-automatic-deploy>
-          </div>
-        }
-        @if (!autodeploy) {
-          <div id="mainbody" class="card">
-            <app-manual-deploy
-              [sasJs]="sasJs"
-              [sasJsConfig]="sasJsConfig"
-              [dcAdapterSettings]="dcAdapterSettings"
-              (onNavigateToHome)="onNavigateToHome()"
-            ></app-manual-deploy>
-          </div>
-        }
+        <div class="autodeploy-section card">
+          <app-automatic-deploy
+            [sasJs]="sasJs"
+            [sasJsConfig]="sasJsConfig"
+            [dcAdapterSettings]="dcAdapterSettings"
+            [appLoc]="appLoc"
+            [dcPath]="dcPath"
+            [selectedAdminGroup]="selectedAdminGroup"
+            (onNavigateToHome)="onNavigateToHome()"
+          ></app-automatic-deploy>
+        </div>
       }
       @case (sasJsConfig.serverType === ServerType.Sasjs) {
         <div class="autodeploy-section card">

client/src/app/deploy/deploy.component.ts

@@ -1,8 +1,6 @@
 import { Component, OnInit, ViewEncapsulation } from '@angular/core'
 import { SasService } from '../services/sas.service'
 import { SASjsConfig } from '@sasjs/adapter'
-import { Router } from '@angular/router'
-import { LoggerService } from '../services/logger.service'
 import { ServerType } from '@sasjs/utils/types/serverType'
 import { AppStoreService } from '../services/app-store.service'
 import { DcAdapterSettings } from '../models/DcAdapterSettings'
@@ -19,15 +17,11 @@ import { DcAdapterSettings } from '../models/DcAdapterSettings'
 })
 export class DeployComponent implements OnInit {
   public step: number = 0
-  public adminGroups: any = []
 
   public appLoc: string = ''
   public dcPath: string = ''
   public selectedAdminGroup: string = ''
 
-  public autodeploy: boolean = true
-  public jsonFile: any = null
-
   public sasJs: any
   public sasJsConfig: SASjsConfig = new SASjsConfig()
 
@@ -37,16 +31,10 @@ export class DeployComponent implements OnInit {
 
   constructor(
     private appStoreService: AppStoreService,
-    private sasService: SasService,
-    private loggerService: LoggerService,
-    private router: Router
+    private sasService: SasService
   ) {
     this.dcAdapterSettings = this.appStoreService.getDcAdapterSettings()
 
-    if (this.router.url.includes('manualdeploy')) {
-      this.autodeploy = false
-    }
-
     this.sasJs = this.sasService.getSasjsInstance()
     this.sasJsConfig = this.sasService.getSasjsConfig()
     this.appLoc = this.dcAdapterSettings?.appLoc || ''
@@ -61,54 +49,16 @@ export class DeployComponent implements OnInit {
    */
   public setDeployDefaults() {
     this.dcPath = this.dcAdapterSettings?.dcPath || ''
-    this.selectedAdminGroup = this.dcAdapterSettings?.adminGroup || ''
-    if (!this.selectedAdminGroup) {
-      this.selectedAdminGroup = 'SASAdministrators'
-    }
+    this.selectedAdminGroup = 'SASAdministrators'
   }
 
   /**
    * Accepting terms of service shows next screen
    */
   public termsAgreeChange() {
-    if (!this.autodeploy) {
-      this.getAdminGroups()
-    }
-
     this.step++
   }
 
-  /**
-   * Fetches admin groups from VIYA to be selected for a backend deploy
-   */
-  public getAdminGroups() {
-    fetch(
-      this.sasJsConfig.serverUrl + '/identities/groups?sortBy=name&limit=5000',
-      {
-        headers: {
-          Accept: 'application/json'
-        }
-      }
-    )
-      .then((res: any) => {
-        return res.text()
-      })
-      .then((res: any) => {
-        let jsonRes
-
-        try {
-          jsonRes = JSON.parse(res)
-        } catch (err: any) {
-          console.error(err)
-        }
-
-        if (jsonRes) {
-          this.adminGroups = jsonRes.items
-          this.selectedAdminGroup = this.adminGroups[0].id
-        }
-      })
-  }
-
   public onNavigateToHome() {
     window.open(location.href.split('#')[0], '_blank')
   }

client/src/app/deploy/deploy.module.ts

@@ -2,7 +2,6 @@ import { NgModule } from '@angular/core'
 import { CommonModule } from '@angular/common'
 import { DeployComponent } from './deploy.component'
 import { AutomaticComponent } from './sections/automatic/automatic.component'
-import { ManualComponent } from './sections/manual/manual.component'
 import { SasjsConfiguratorComponent } from './sections/sasjs-configurator/sasjs-configurator.component'
 import { ClarityModule } from '@clr/angular'
 import { FormsModule } from '@angular/forms'
@@ -12,7 +11,6 @@ import { DeployRoutingModule } from './deploy-routing.module'
   declarations: [
     DeployComponent,
     AutomaticComponent,
-    ManualComponent,
     SasjsConfiguratorComponent
   ],
   imports: [CommonModule, FormsModule, ClarityModule, DeployRoutingModule]

client/src/app/deploy/sections/manual/manual.component.html

@@ -1,229 +0,0 @@
-<div class="card-header position-relative">
-  Configurator
-  <p class="d-inline-block ml-10 mb-10 mt-0">App Location: {{ appLoc }}</p>
-  <button
-    (click)="deleteKeys()"
-    class="btn btn-primary clear-memory-button position-absolute"
-  >
-    Clear memory
-  </button>
-</div>
-<div class="card-block">
-  <!-- <div class="card-title">
-  Client Details
-</div> -->
-
-  @if (needsLogin) {
-    <div id="loginForm" class="d-none">
-      <p class="mb-10">Please log in first</p>
-      <label for="username" class="clr-control-label">Username</label>
-      <div class="mb-10 clr-control-container">
-        <div class="clr-input-wrapper">
-          <input type="text" id="username" class="clr-input" />
-        </div>
-      </div>
-      <label for="password" class="clr-control-label">Password</label>
-      <div class="mb-10 clr-control-container">
-        <div class="clr-input-wrapper">
-          <input type="password" id="password" class="clr-input" />
-        </div>
-      </div>
-      <button class="btn btn-primary d-none" id="loginBtn">Log in</button>
-    </div>
-  }
-
-  @if (!needsLogin) {
-    <form>
-      <div class="clr-form-control">
-        <label for="select-full" class="clr-control-label">Admin group</label>
-        <div class="clr-control-container">
-          <div class="clr-select-wrapper">
-            <select
-              [(ngModel)]="selectedAdminGroup"
-              [ngModelOptions]="{ standalone: true }"
-              name="selectedAdminGroup"
-              id="adminGroupsSelect"
-              class="clr-select"
-            >
-              @for (adminGroup of adminGroups; track adminGroup) {
-                <option [value]="adminGroup.id">
-                  {{ adminGroup.name }}
-                </option>
-              }
-            </select>
-          </div>
-        </div>
-      </div>
-      <div class="clr-form-control">
-        <div [class.hidden]="contextsLoading">
-          <label for="select-full" class="clr-control-label">Context</label>
-          <div class="clr-control-container">
-            <div class="clr-select-wrapper">
-              <select
-                [(ngModel)]="selectedContext"
-                [ngModelOptions]="{ standalone: true }"
-                name="selectedContext"
-                class="clr-select"
-              >
-                @for (context of allContexts; track context) {
-                  <option [value]="context.name">
-                    {{ context.name }}
-                    @if ((context.attributes | json) != '{}') {
-                      <span>( {{ context.attributes.sysUserId }} )</span>
-                    }
-                  </option>
-                }
-              </select>
-            </div>
-          </div>
-          <button
-            (click)="executableContext()"
-            type="button"
-            class="btn btn-icon"
-            id="contexts-btn"
-          >
-            <clr-icon shape="play"></clr-icon>
-          </button>
-        </div>
-        <div [class.hidden]="!contextsLoading" class="d-flex">
-          <span class="spinner spinner-inline mr-10">
-            Loading contexts...
-          </span>
-          <span> Loading contexts... </span>
-        </div>
-        <label for="dcloc" class="mt-20 clr-control-label">DC Loc</label>
-        <div class="mb-10 clr-control-container">
-          <div class="clr-input-wrapper">
-            <input
-              [(ngModel)]="dcPath"
-              [ngModelOptions]="{ standalone: true }"
-              (focusout)="saveDcPath()"
-              name="dcPath"
-              type="text"
-              id="dcloc"
-              class="clr-input"
-            />
-          </div>
-        </div>
-        <p class="mt-10">
-          Select JSON file to upload (json build file preloaded):
-        </p>
-        <div class="d-flex flex-column">
-          <input
-            type="file"
-            (click)="clearUploadInput($event)"
-            (change)="onJsonFileChange($event)"
-          />
-          <!-- <button *ngIf="downloadFileBtn" (click)="downloadSasPrecodeFile()" style="width: 40px; min-width: 0;" class="btn btn-sm btn-icon">
-          <clr-icon shape="download"></clr-icon>
-        </button> -->
-        </div>
-        <div class="mt-20 d-flex align-items-center">
-          <button
-            (click)="executeJson()"
-            class="btn btn-primary"
-            [clrLoading]="isSubmittingJson"
-            id=""
-            class="mt-0"
-          >
-            SUBMIT JSON
-          </button>
-          @if (isJsonSubmitted) {
-            <span>JSON Submitted Successfully</span>
-          }
-          <!-- <span *ngIf="executingScript" class="spinner spinner-inline ml-3">
-        Loading...
-      </span> -->
-        </div>
-        <p class="mt-10">Select SAS file to upload:</p>
-        <div class="d-flex flex-column">
-          <input
-            type="file"
-            (click)="clearUploadInput($event)"
-            (change)="onSasFileChange($event); downloadFileBtn = true"
-          />
-          @if (downloadFileBtn) {
-            <button
-              (click)="downloadSasPrecodeFile()"
-              class="btn btn-sm btn-icon min-w-0 w-40"
-            >
-              <clr-icon shape="download"></clr-icon>
-            </button>
-          }
-        </div>
-        <div class="mt-20 d-flex align-items-center">
-          <button
-            (click)="executeSAS()"
-            class="btn btn-primary"
-            [clrLoading]="executingScript"
-            id="deploy"
-            class="mt-0"
-          >
-            SUBMIT
-          </button>
-          <!-- <span *ngIf="executingScript" class="spinner spinner-inline ml-3">
-      Loading...
-    </span> -->
-        </div>
-      </div>
-      @if (jobLog.length > 0) {
-        <p class="mb-0 mt-10">File execute completed</p>
-        <hr />
-        <div>
-          <button
-            (click)="downloadFile(jobLog, 'execute-script-log', 'txt')"
-            class="btn btn-primary mt-0 mr-20"
-          >
-            Download log
-          </button>
-        </div>
-      }
-      <button
-        [clrLoading]="createDatabaseLoading"
-        (click)="createDatabase()"
-        class="btn btn-primary mt-10"
-      >
-        Create Database
-      </button>
-      @if (makeDataResponse.length > 0) {
-        <p class="mb-0 mt-10">Create Database Completed</p>
-        <hr />
-        @if (makeDataResponse.length > 0) {
-          <div class="log-wrapper">
-            {{ makeDataResponse }}
-          </div>
-        }
-        <button (click)="navigateToHome()" class="btn btn-primary mt-15">
-          Let's get started
-        </button>
-        <button
-          (click)="downloadFile(makeDataResponse, 'create-database-log', 'txt')"
-          class="btn btn-primary mt-15"
-        >
-          Download log
-        </button>
-        <button (click)="validateDeploy()" class="btn btn-primary mt-15">
-          Validate
-        </button>
-        @if (validationState !== 'none' || isValidating) {
-          <div class="validation-bar">
-            @if (isValidating) {
-              <span class="spinner spinner-inline mr-10">
-                Validating deploy...
-              </span>
-              <span> Validating deploy... </span>
-            }
-            @if (!isValidating && validationState === 'error') {
-              <clr-icon shape="exclamation-circle" class="is-error"></clr-icon>
-              <span> Validation failed </span>
-            }
-            @if (!isValidating && validationState === 'success') {
-              <clr-icon shape="check-circle" class="is-success"></clr-icon>
-              <span> Validation succeeded </span>
-            }
-          </div>
-        }
-      }
-    </form>
-  }
-</div>

client/src/app/deploy/sections/manual/manual.component.spec.ts

@@ -1,90 +0,0 @@
-import { ManualComponent } from './manual.component'
-
-// validateDeploy() is fire-and-forget (doesn't return its own promise
-// chain), so tests need to wait for it to settle some other way. A
-// macrotask boundary guarantees every already-queued microtask (regardless
-// of how many .then() hops validateDeploy's chain has) has run first.
-const flushPromiseChain = () => new Promise((resolve) => setTimeout(resolve, 0))
-
-/**
- * validateDeploy() only touches sasService.request, eventService.showInfoModal
- * and loggerService.log - these stubs cover exactly that surface, not the
- * full real services (no TestBed/DI needed, same plain-instantiation-with-
- * stubs precedent as app.service.spec.ts).
- */
-const buildDeps = () => {
-  const sasService: any = {
-    request: jasmine.createSpy('request')
-  }
-  const eventService: any = {
-    showInfoModal: jasmine.createSpy('showInfoModal')
-  }
-  const loggerService: any = {
-    log: jasmine.createSpy('log')
-  }
-  const deployService: any = {}
-
-  return { sasService, eventService, loggerService, deployService }
-}
-
-const buildManualComponent = (deps: ReturnType<typeof buildDeps>) =>
-  new ManualComponent(
-    deps.sasService,
-    deps.eventService,
-    deps.loggerService,
-    deps.deployService
-  )
-
-describe('ManualComponent - validateDeploy', () => {
-  it('sets validationState to success when saslibs is present on a normal object response', async () => {
-    const deps = buildDeps()
-    deps.sasService.request.and.resolveTo({
-      adapterResponse: { saslibs: { SOME_LIB: ['SOME_TABLE'] } }
-    })
-
-    const component = buildManualComponent(deps)
-    component.validateDeploy()
-    await flushPromiseChain()
-
-    expect(component.validationState).toBe('success')
-    expect(deps.eventService.showInfoModal).not.toHaveBeenCalled()
-  })
-
-  it('sets validationState to error with no modal when saslibs is missing from an otherwise normal object response', async () => {
-    const deps = buildDeps()
-    deps.sasService.request.and.resolveTo({
-      adapterResponse: { SYSSITE: 'SITE1' }
-      // saslibs deliberately omitted
-    })
-
-    const component = buildManualComponent(deps)
-    component.validateDeploy()
-    await flushPromiseChain()
-
-    expect(component.validationState).toBe('error')
-    expect(deps.eventService.showInfoModal).not.toHaveBeenCalled()
-  })
-
-  // Same underlying bug as app.service.spec.ts's equivalent test: a
-  // misconfigured Viya computeTasks deployment returns a raw "Job error"
-  // text body, which the adapter resolves as adapterResponse verbatim.
-  it('sets validationState to error and shows the real response text when adapterResponse is a raw string, not an object', async () => {
-    const deps = buildDeps()
-    const rawJobError = [
-      'Job error',
-      'The Compute service could not execute the task c74a707a-8b88-46a5-8f28-9a7694ad5e13 because the context 340cd3eb-72ae is not reusable. Please provide a reusable context or supply the ID of an existing session in the task request.',
-      'path: /compute/tasks',
-      'correlator: dc1a9fda-8f65-4f32-b4d4-2a70e02179b1;1bcb6a4c-8a16-4053-a790-292544c25a03'
-    ].join('\n')
-    deps.sasService.request.and.resolveTo({ adapterResponse: rawJobError })
-
-    const component = buildManualComponent(deps)
-    component.validateDeploy()
-    await flushPromiseChain()
-
-    expect(component.validationState).toBe('error')
-    expect(deps.eventService.showInfoModal).toHaveBeenCalledTimes(1)
-    const [, message] = deps.eventService.showInfoModal.calls.mostRecent().args
-    expect(message).toContain(rawJobError)
-  })
-})

client/src/app/deploy/sections/manual/manual.component.ts

@@ -1,346 +0,0 @@
-import {
-  Component,
-  Input,
-  OnInit,
-  Output,
-  EventEmitter,
-  ViewEncapsulation
-} from '@angular/core'
-import SASjs, { SASjsConfig } from '@sasjs/adapter'
-import { DcAdapterSettings } from 'src/app/models/DcAdapterSettings'
-import { RequestWrapperResponse } from 'src/app/models/request-wrapper/RequestWrapperResponse'
-import { DeployService } from 'src/app/services/deploy.service'
-import { EventService } from 'src/app/services/event.service'
-import { LoggerService } from 'src/app/services/logger.service'
-import { SasService } from 'src/app/services/sas.service'
-import { getMalformedAdapterResponseMessage } from 'src/app/shared/utils/get-malformed-adapter-response-message'
-
-@Component({
-  selector: 'app-manual-deploy',
-  templateUrl: './manual.component.html',
-  styleUrls: ['./manual.component.scss'],
-  encapsulation: ViewEncapsulation.None,
-  standalone: false
-})
-export class ManualComponent implements OnInit {
-  @Input() sasJs!: SASjs
-  @Input() sasJsConfig: SASjsConfig = new SASjsConfig()
-  @Input() dcAdapterSettings: DcAdapterSettings | undefined
-
-  @Output() onNavigateToHome: EventEmitter<any> = new EventEmitter<any>()
-
-  public needsLogin: boolean = false
-  public adminGroups: any = []
-  public allContexts: any = []
-
-  public appLoc: string = ''
-  public dcPath: string = ''
-  public selectedAdminGroup: string = ''
-  public selectedContext: string = ''
-  public jobLog: string = ''
-  public makeDataResponse: string = ''
-  public linesOfCode: any = []
-  public fileName: string = ''
-  public preloadedFile: boolean = true
-  public executeSASEnabled: boolean = false
-
-  public contextsLoading: boolean = false
-  public createDatabaseLoading: boolean = false
-  public executingScript: boolean = false
-  public downloadFileBtn: boolean = false
-  public isValidating: boolean = false
-  public jsonFile: any = null
-  public isSubmittingJson: boolean = false
-  public isJsonSubmitted: boolean = false
-  public validationState: string = 'none'
-
-  constructor(
-    private sasService: SasService,
-    private eventService: EventService,
-    private loggerService: LoggerService,
-    private deployService: DeployService
-  ) {}
-
-  ngOnInit(): void {}
-
-  /**
-   * FIXME: Remove
-   */
-  public async executableContext() {
-    // getExecutableContexts now need AuthConfig parameter which we don't have on web
-    // this.contextsLoading = true
-    // let contexts = await this.sasJs.getExecutableContexts()
-    // this.allContexts = contexts
-    // if (contexts.length === 0) {
-    //   this.selectedContext = ''
-    // } else {
-    //   this.selectedContext = this.allContexts[0].name
-    // }
-    // this.contextsLoading = false
-  }
-
-  /**
-   * Removes sas.json file attached to the input
-   */
-  public clearUploadInput(event: Event) {
-    this.deployService.clearUploadInput(event)
-  }
-
-  /**
-   * Reads attached SAS file to be sent to sas for execution (backend deploy)
-   */
-  public onSasFileChange(event: any) {
-    this.preloadedFile = false
-
-    let file = event.target.files[0]
-    this.fileName = file.name
-
-    let fileReader = new FileReader()
-
-    fileReader.onload = () => {
-      if (fileReader.result) {
-        this.linesOfCode = (fileReader.result as string).split('\n')
-        this.linesOfCode = this.linesOfCode.filter((el: string) => {
-          return el !== '' && el !== null
-        })
-
-        this.executeSASEnabled = true
-
-        this.addPrecodeLines()
-      }
-    }
-
-    fileReader.readAsText(file)
-  }
-
-  /**
-   * Reads attached JSON file to be sent to sas for execution (backend deploy)
-   */
-  public async onJsonFileChange(event: any) {
-    let file = event.target.files[0]
-
-    this.jsonFile = await this.deployService.readFile(file)
-  }
-
-  /**
-   * Appending precode lines to the attached sas or json file for backend deploy
-   */
-  public addPrecodeLines() {
-    let headerLines = [
-      `%let context=${this.selectedContext};`,
-      `%let appLoc=${this.appLoc};`,
-      `%let admin=${this.selectedAdminGroup};`,
-      `%let dcpath=${this.dcPath};`
-    ]
-
-    this.linesOfCode.unshift(...headerLines)
-  }
-
-  /**
-   * Downloadng file with precode included
-   */
-  public downloadSasPrecodeFile() {
-    let linesAsText = this.linesOfCode.join('\n')
-    let filename = this.fileName.split('.')[0]
-
-    this.downloadFile(linesAsText, filename, 'sas')
-  }
-
-  /**
-   * Used for downloading log and repsonse as a file
-   */
-  public downloadFile(
-    content: any,
-    filename: string,
-    extension: string = 'txt'
-  ) {
-    this.deployService.downloadFile(content, filename, extension)
-  }
-
-  /**
-   * Saving dcpath to localstorage
-   * FIXME: maybe it'snot necessary
-   */
-  public saveDcPath() {
-    localStorage.setItem('deploy_dc_loc', this.dcPath)
-  }
-
-  /**
-   * Send sas.json to be executed (deploying backend)
-   */
-  public async executeJson() {
-    this.isSubmittingJson = true
-
-    try {
-      let uploadJsonFile = await this.sasJs.deployServicePack(
-        this.jsonFile,
-        this.dcAdapterSettings?.appLoc || '',
-        undefined,
-        undefined,
-        true
-      )
-
-      this.isJsonSubmitted = true
-    } catch (ex: any) {
-      let textEx = ''
-
-      if (typeof ex.message !== 'string') {
-        textEx = JSON.stringify(ex).replace(/\\/gm, '')
-      } else {
-        textEx = ex.message
-      }
-
-      this.eventService.showInfoModal(
-        'Deploy error',
-        `Exception: \n ${textEx !== '' ? textEx : ex}`
-      )
-
-      return
-    }
-
-    this.isSubmittingJson = false
-  }
-
-  /**
-   * Send sas file to be executed (deploying backend)
-   */
-  public async executeSAS() {
-    this.executingScript = true
-    this.jobLog = ''
-    this.makeDataResponse = ''
-
-    try {
-      let executeResponse = await this.sasJs.executeScript({
-        fileName: this.fileName,
-        linesOfCode: this.linesOfCode,
-        contextName: this.selectedContext
-      })
-
-      this.loggerService.log(executeResponse)
-
-      if (typeof executeResponse.log === 'string') {
-        executeResponse.log = JSON.parse(executeResponse.log)
-      }
-
-      if (executeResponse.jobStatus === 'error') {
-        alert('Error!')
-      } else {
-        this.jobLog = executeResponse.log.items
-          ? executeResponse.log.items.map((i: any) => i.line).join('\n')
-          : JSON.stringify(executeResponse.log)
-      }
-
-      this.executingScript = false
-    } catch (err) {
-      this.executingScript = false
-    }
-  }
-
-  /**
-   * Running makedata service
-   * @param newTab open and run in new tab
-   */
-  public createDatabase(newTab: boolean = true) {
-    if (newTab) {
-      // Debug is ALWAYS enabled for makedata — it runs only once during
-      // deployment and the full log is needed to diagnose any issues.
-      const _debug = '&_debug=131'
-      let url =
-        this.sasService.getSasjsConfig().serverUrl +
-          '/SASJobExecution/?_program=' +
-          this.dcAdapterSettings?.appLoc ||
-        '' +
-          '/admin/makedata' +
-          '&ADMIN=' +
-          this.selectedAdminGroup +
-          '&DCPATH=' +
-          this.dcPath +
-          _debug
-
-      window.open(url, '_blank')
-
-      return
-    }
-
-    this.createDatabaseLoading = true
-
-    let data = {
-      fromjs: [
-        {
-          ADMIN: this.selectedAdminGroup,
-          DCPATH: this.dcPath
-        }
-      ]
-    }
-
-    /**
-     * We are overriding default `sasjsConfig` object fields with this object fields.
-     * Here we want to run this request using original WEB method.
-     * contextName: null is the MUST field for it.
-     */
-    let overrideConfig = {
-      useComputeApi: null,
-      contextName: this.sasJsConfig.contextName,
-      debug: true
-    }
-
-    this.sasJs
-      .request(`services/admin/makedata`, data, overrideConfig, () => {
-        this.sasService.shouldLogin.next(true)
-      })
-      .then((res: any) => {
-        try {
-          this.makeDataResponse = JSON.stringify(res)
-        } catch {
-          this.makeDataResponse = res
-        }
-
-        this.createDatabaseLoading = false
-      })
-      .catch((err: any) => {
-        this.createDatabaseLoading = false
-
-        try {
-          this.makeDataResponse = JSON.stringify(err)
-        } catch {
-          this.makeDataResponse = err
-        }
-      })
-  }
-
-  public navigateToHome() {
-    this.onNavigateToHome.emit()
-  }
-
-  public validateDeploy() {
-    this.isValidating = true
-
-    this.sasService
-      .request('public/startupservice', null)
-      .then((res: RequestWrapperResponse) => {
-        this.loggerService.log(res.adapterResponse)
-
-        const malformedMessage = getMalformedAdapterResponseMessage(
-          res.adapterResponse
-        )
-        if (malformedMessage) {
-          this.validationState = 'error'
-          this.eventService.showInfoModal('Error', malformedMessage)
-        } else if (res.adapterResponse.saslibs) {
-          this.validationState = 'success'
-        } else {
-          this.validationState = 'error'
-        }
-
-        this.isValidating = false
-      })
-      .catch((err: any) => {
-        this.isValidating = false
-        this.validationState = 'error'
-      })
-  }
-
-  public deleteKeys() {
-    localStorage.removeItem('deploy_dc_loc')
-  }
-}

client/src/app/editor/editor.component.html

@@ -369,11 +369,30 @@
           </div>
         }
         @if (!['', ' '].includes(queryText)) {
-          <div class="clr-col-md-12 infoBar">
-            <span
+          <div
+            #infoBar
+            class="clr-col-md-12 infoBar"
+            [class.expanded]="filterExpanded"
+          >
+            <span class="infoBar-text"
               >FILTER :
               <b>{{ queryText }}</b>
             </span>
+            @if (filterOverflows) {
+              <button
+                type="button"
+                class="infoBar-toggle"
+                [attr.aria-expanded]="filterExpanded"
+                [attr.aria-label]="
+                  filterExpanded
+                    ? 'Collapse the filter clause'
+                    : 'Expand the filter clause'
+                "
+                (click)="toggleFilterPanel()"
+              >
+                <clr-icon aria-hidden="true" shape="caret down"></clr-icon>
+              </button>
+            }
           </div>
         }
       </div>

client/src/app/editor/editor.component.ts

@@ -1,4 +1,5 @@
 import {
+  AfterViewChecked,
   AfterViewInit,
   ChangeDetectorRef,
   Component,
@@ -88,6 +89,11 @@ import { withoutEditStatus } from './utils/withoutEditStatus'
 import { getEditStatusSymbol } from './utils/getEditStatusSymbol'
 import { normalizeSortConfig } from './utils/normalizeSortConfig'
 import { EDIT_STATUS_COLUMN_NAME } from '../shared/dc-validator/utils/editStatusColumnRule'
+import { DELETE_RECORD_COLUMN_NAME } from '../shared/dc-validator/utils/deleteRecordColumnRule'
+import {
+  prepareDeleteRecordColumn,
+  seedDeleteRecordColumn
+} from '../shared/dc-validator/utils/deleteRecordColumn'
 import {
   errorRenderer,
   noSpinnerRenderer,
@@ -112,7 +118,9 @@ import { ParseResult } from '../models/ParseResult.interface'
   encapsulation: ViewEncapsulation.None,
   standalone: false
 })
-export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
+export class EditorComponent
+  implements OnInit, AfterViewInit, AfterViewChecked, OnDestroy
+{
   @ViewChildren('uploadStater')
   uploadStaterCompList: QueryList<UploadStaterComponent> = new QueryList()
   @ViewChildren('queryFilter')
@@ -525,6 +533,19 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
   public tableData: Array<any> = []
   public queryText = ''
   public queryTextSaved = ''
+
+  /**
+   * The applied-filter panel is collapsed to a single line by default. The
+   * chevron that expands it is only useful when the clause does not fit that
+   * line, which depends on the rendered width - so it is measured from the DOM
+   * rather than guessed from the length of the text.
+   */
+  public filterExpanded = false
+  public filterOverflows = false
+
+  @ViewChild('infoBar') infoBar?: ElementRef<HTMLElement>
+
+  private filterMeasuredKey = ''
   public showApprovers = false
   public pkDups = false
   public validationDone = 0
@@ -1190,7 +1211,16 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
       this.cdf.detectChanges()
 
       this.submitLoading = false
-      this.sasStoreService.setQueryVariables(this.libds, this.cols)
+      // The columns that came with the table are the cheapest source for the
+      // picker. When the table was not returned - the row/cell limits abort the
+      // request before the data arrives - that list is empty, so the store
+      // falls back to public/getcols rather than leaving the picker with
+      // nothing to pick from.
+      this.sasStoreService
+        .setQueryVariablesFromTable(this.libds, this.cols)
+        .catch((err: any) =>
+          this.eventService.catchResponseError('public/getcols', err)
+        )
     }
   }
 
@@ -3494,14 +3524,24 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
           // A synchronous throw inside initSetup (called from the .then()
           // above) lands here too, not just a network failure - log it, or
           // the table just renders hidden with no clue why.
-          // eslint-disable-next-line no-console
-          console.error('editors/getdata failed:', err)
-          this.getdataError = true
-          this.tableTrue = true
+          this.handleGetdataFailure(err)
         })
     }
   }
 
+  /**
+   * Puts the page into its getdata error state. Shared by the callService
+   * .catch and initSetup's own retry branch: a throw from the retry runs on a
+   * bare setTimeout, outside that promise chain, so it would otherwise never
+   * reach the .catch and the user would be left on the loading spinner.
+   */
+  private handleGetdataFailure(err: any) {
+    // eslint-disable-next-line no-console
+    console.error('editors/getdata failed:', err)
+    this.getdataError = true
+    this.tableTrue = true
+  }
+
   /**
    * VA mode, initial (unfiltered) route only: briefly wait for VA's buffered
    * first message and, if it already carries a filter, save that filter and
@@ -3547,6 +3587,48 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
     }
   }
 
+  ngAfterViewChecked() {
+    this.scheduleFilterOverflowCheck()
+  }
+
+  /**
+   * Measures whether the applied-filter clause fits the collapsed single line,
+   * and shows or hides the chevron accordingly. The measurement is deferred
+   * because `filterOverflows` and `filterExpanded` are bound in this view -
+   * setting them inside the change-detection cycle would raise
+   * ExpressionChangedAfterItHasBeenChecked.
+   */
+  private scheduleFilterOverflowCheck() {
+    const el = this.infoBar?.nativeElement
+    if (!el) {
+      return
+    }
+
+    const text = el.querySelector('.infoBar-text') as HTMLElement | null
+    if (!text) {
+      return
+    }
+
+    // re-measure only when the clause, the available width, or the state changes
+    const key = `${this.queryText}|${text.clientWidth}|${this.filterExpanded}`
+    if (key === this.filterMeasuredKey) {
+      return
+    }
+    this.filterMeasuredKey = key
+
+    setTimeout(() => {
+      // while expanded the clause wraps, so there is nothing to measure
+      if (this.filterExpanded) {
+        return
+      }
+      this.filterOverflows = text.scrollWidth > text.clientWidth
+    })
+  }
+
+  public toggleFilterPanel() {
+    this.filterExpanded = !this.filterExpanded
+  }
+
   ngAfterViewInit() {
     // Fix ARIA accessibility issues after table initialization
     setTimeout(() => {
@@ -3808,13 +3890,28 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
 
     this.hotInstance = this.hotTableComponent?.hotInstance!
     if (!this.hotInstance) {
-      // Retry init, don't permanently abort
+      // Retry init, don't permanently abort. A throw from the retry runs on a
+      // bare setTimeout rather than in the caller's promise chain, so it would
+      // never reach that .catch - surface it through the same handler here.
       if (attempt < 60) {
-        setTimeout(() => this.initSetup(response, attempt + 1), 50)
+        setTimeout(() => {
+          try {
+            this.initSetup(response, attempt + 1)
+          } catch (err) {
+            this.handleGetdataFailure(err)
+          }
+        }, 50)
       }
       return
     }
 
+    // The delete column is client-owned (see deleteRecordColumnRule.ts):
+    // this asserts getdata did not send it, then prepends it to COLHEADERS so
+    // it is the grid's first column. A throw here lands in the caller's
+    // .catch - or, on a retry, the branch above - which sets getdataError and
+    // shows the error state.
+    prepareDeleteRecordColumn(response)
+
     this.cols = response.data.cols
     this.dsmeta = response.data.dsmeta
     this.versions = response.data.versions || []
@@ -3864,9 +3961,9 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
 
     this.checkEmptyRowWhenFilter()
 
-    if (this.headerColumns.indexOf('_____DELETE__THIS__RECORD_____') !== -1) {
+    if (this.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME) !== -1) {
       this.headerColumns[
-        this.headerColumns.indexOf('_____DELETE__THIS__RECORD_____')
+        this.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME)
       ] = 'Delete?'
     }
 
@@ -3931,6 +4028,10 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
       row[EDIT_STATUS_COLUMN_NAME] = 'U'
     }
 
+    // The delete flag is client-owned too (see deleteRecordColumnRule.ts) -
+    // getdata no longer sends the column, so every row starts unmarked.
+    seedDeleteRecordColumn(this.dataSource)
+
     // Seed HARDFORMULA/SOFTFORMULA columns with their computed formula
     // string per row. dataSourceUnchanged isn't populated yet this early in
     // a fresh load (only the excel-preview/add-row flows set it) - at this

client/src/app/editor/utils/characterColumnFormula.integration.spec.ts

@@ -417,9 +417,14 @@ describe('formulas plugin must not be disabled/re-enabled around a table load',
  * capping maxRows at editor_rows_allowed alone makes HyperFormula reject
  * any table bigger than that cap outright. See initSetup's own
  * Math.max(dataSource.length, editor_rows_allowed) fix.
+ *
+ * Handsontable 18.1 fixed the engine coupling upstream (#10672): the grid's
+ * maxRows/maxCols no longer limit the HyperFormula engine, so a capped
+ * maxRows no longer rejects a larger existing dataset. DC's own guard is
+ * still in place; the first test below now pins the fixed behaviour.
  */
 describe('maxRows must never be smaller than the actual loaded row count', () => {
-  it('throws when maxRows is capped below the real row count (proves the mechanism)', () => {
+  it('no longer throws when maxRows is capped below the real row count (HOT 18.1 #10672)', () => {
     const container = document.createElement('div')
     document.body.appendChild(container)
     const colNames = ['PK', 'A', 'B']
@@ -440,7 +445,7 @@ describe('maxRows must never be smaller than the actual loaded row count', () =>
       error = e
     }
 
-    expect(error?.message).toEqual('Sheet size limit exceeded')
+    expect(error).toBeNull()
   })
 
   it('does not throw when maxRows is never let below the actual row count (the fix)', () => {

client/src/app/editor/utils/sortedGridRowSync.integration.spec.ts

@@ -287,9 +287,13 @@ describe('syncOverwrittenCommentForCell resolves the correct visual cell on a so
  * trigger it, independent of what it actually changes. editTable(),
  * cancelEdit(), and every other place editor.component.ts calls
  * updateSettings() after the grid could already be sorted (e.g. the user
- * sorted while still read-only, then clicked Edit) hits this. The fix
- * (updateSettingsSortSafe) clears the sort before the call and restores it
- * immediately afterward.
+ * sorted while still read-only, then clicked Edit) hits this.
+ *
+ * Fixed upstream in Handsontable 18.1 (the formulas plugin now keeps its
+ * index translations across updateSettings), so the corruption no longer
+ * occurs. DC's own guard (updateSettingsSortSafe - clear the sort, call
+ * updateSettings, restore the sort) is still in place and still correct;
+ * this test now pins the fixed behaviour so a future regression shows up.
  */
 describe('updateSettings must not run while a sort is active - it corrupts formula cell references', () => {
   const buildSortedGrid = (): Handsontable => {
@@ -340,13 +344,13 @@ describe('updateSettings must not run while a sort is active - it corrupts formu
       hot.getDataAtRowProp(r, 'FORMULA_HARD_COL')
     )
 
-  it('demonstrates the failure mode: updateSettings while sorted corrupts formula cells', () => {
+  it('no longer corrupts formula cells when updateSettings runs while sorted (HOT 18.1)', () => {
     const hot = buildSortedGrid()
 
     hot.updateSettings({}, false)
     hot.render()
 
-    expect(formulaHardColValues(hot)).toContain('#REF!')
+    expect(formulaHardColValues(hot)).not.toContain('#REF!')
 
     hot.destroy()
   })

client/src/app/models/DcAdapterSettings.ts

@@ -1,7 +1,5 @@
 import { SASjsConfig } from '@sasjs/adapter'
 
 export interface DcAdapterSettings extends Partial<SASjsConfig> {
-  adminGroup: string
   dcPath: string
-  hotLicenceKey: string
 }

client/src/app/multi-dataset/multi-dataset.component.ts

@@ -18,6 +18,11 @@ import {
 import { globals } from '../_globals'
 import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
 import { DcValidator } from '../shared/dc-validator/dc-validator'
+import {
+  prepareDeleteRecordColumn,
+  seedDeleteRecordColumn
+} from '../shared/dc-validator/utils/deleteRecordColumn'
+import { DELETE_RECORD_COLUMN_NAME } from '../shared/dc-validator/utils/deleteRecordColumnRule'
 import { ExcelRule } from '../models/TableData'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { Col } from '../shared/dc-validator/models/col.model'
@@ -360,7 +365,21 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {
 
       this.uploadLoading = false
 
-      const datasetObjects = this.buildDatasetsObjects(datasets)
+      // buildDatasetsObjects throws when a getdata response carries the
+      // reserved delete-record column (see deleteRecordColumn.ts). This chain
+      // has no .catch, so an uncaught throw here would become an unhandled
+      // rejection that aborts the loop with nothing shown. Surface it on
+      // screen, the way the editor flow refuses the same response, rather
+      // than leaving the user on an empty table selection with the file
+      // still attached.
+      let datasetObjects: DatasetsObject[] = []
+      try {
+        datasetObjects = this.buildDatasetsObjects(datasets)
+      } catch (err) {
+        console.warn('Error building dataset objects', err)
+        this.eventService.showInfoModal('Upload error', (err as Error).message)
+        return
+      }
 
       datasetObjects.forEach((datasetObject) => {
         this.parsedDatasets.push({
@@ -1074,6 +1093,12 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {
 
     for (let response of responses) {
       if (response.data) {
+        // The delete column is client-owned (see deleteRecordColumnRule.ts):
+        // asserts getdata did not send it, then prepends it to COLHEADERS so
+        // it is index 0 for everything below.
+        prepareDeleteRecordColumn(response)
+        seedDeleteRecordColumn(response.data.sasdata)
+
         const datasetObject: DatasetsObject = {
           ...response,
           hotTable: {
@@ -1097,14 +1122,10 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {
         datasetObject.headerPks = response.data.sasparams[0].PK.split(' ')
 
         if (
-          datasetObject.headerColumns.indexOf(
-            '_____DELETE__THIS__RECORD_____'
-          ) !== -1
+          datasetObject.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME) !== -1
         ) {
           datasetObject.headerColumns[
-            datasetObject.headerColumns.indexOf(
-              '_____DELETE__THIS__RECORD_____'
-            )
+            datasetObject.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME)
           ] = 'Delete?'
         }
 

client/src/app/query/query.component.spec.ts

@@ -0,0 +1,110 @@
+import { QueryComponent } from './query.component'
+import { HelperService } from '../services/helper.service'
+
+/**
+ * dynamicWhereClauseCreator builds the prefilter that getcolvals sends when the
+ * dynamic where clause is on, and it composes a temporal value into the same
+ * SAS literal the applied filter carries.  The value it prefilters with comes
+ * from the *other* clauses in the query, so the clause under test sits at index
+ * 1 and index 0 is skipped.
+ */
+describe('QueryComponent - dynamic where clause', () => {
+  const buildComponent = (elements: any[]) => {
+    const component = new QueryComponent(
+      {} as any,
+      { catchResponseError: () => undefined } as any,
+      new HelperService(null as any),
+      { detectChanges: () => undefined } as any
+    )
+
+    component.clauses = {
+      queryObj: [{ clauseLogic: 'AND', elements }],
+      groupLogic: 'AND'
+    } as any
+
+    return component
+  }
+
+  const temporalClause = (over: any = {}) => ({
+    variable: 'SOME_DATE',
+    operator: '=',
+    value: '',
+    values: [],
+    type: 'num',
+    ddtype: 'DATE',
+    valueVariable: false,
+    ...over
+  })
+
+  it('sends an IN entry as the literal for its unformatted value', () => {
+    const component = buildComponent([
+      temporalClause(),
+      temporalClause({
+        operator: 'IN',
+        value: [{ checked: true, val: '12FEB1960' }],
+        values: [{ formatted: '12FEB1960', unformatted: 42 }]
+      })
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table.length).toBe(1)
+    expect(table[0].RAW_VALUE).toBe("('12FEB1960'd)")
+  })
+
+  it('maps both ends of a BETWEEN picked from the value list', () => {
+    const component = buildComponent([
+      temporalClause(),
+      temporalClause({
+        operator: 'BETWEEN',
+        value: ['12FEB1960', '13FEB1960'],
+        values: [
+          { formatted: '12FEB1960', unformatted: 42 },
+          { formatted: '13FEB1960', unformatted: 43 }
+        ]
+      })
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table.length).toBe(1)
+    expect(table[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  it('leaves an already-unformatted BETWEEN alone', () => {
+    const component = buildComponent([
+      temporalClause(),
+      temporalClause({
+        operator: 'BETWEEN',
+        value: [42, 43],
+        values: [
+          { formatted: '12FEB1960', unformatted: 42 },
+          { formatted: '13FEB1960', unformatted: 43 }
+        ]
+      })
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  it('emits the unformatted value in the non-literal IN fallback', () => {
+    const component = buildComponent([
+      temporalClause(),
+      {
+        variable: 'SOME_CHAR',
+        operator: 'IN',
+        value: [{ checked: true, val: 'ABC ' }],
+        values: [{ formatted: 'ABC ', unformatted: 'ABC' }],
+        type: 'char',
+        ddtype: 'CHAR',
+        valueVariable: false
+      }
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table[0].RAW_VALUE).toBe("('ABC')")
+  })
+})

client/src/app/query/query.component.ts

@@ -425,35 +425,57 @@ export class QueryComponent
             continue
 
           let rawValue = ''
-          const operator =
-            this.clauses.queryObj[groupInd].elements[clauseInd].operator
-          const variable =
-            this.clauses.queryObj[groupInd].elements[clauseInd].variable
+          const element = this.clauses.queryObj[groupInd].elements[clauseInd]
+          const operator = element.operator
+          const variable = element.variable
+          /* a temporal value is sent as a SAS literal (eg '20APR2042'd), the
+             same way the applied filter carries it */
+          const temporalValue = (value: any) =>
+            element.valueVariable
+              ? null
+              : this.helperService.sasTemporalLiteral(value, element.ddtype)
+          /* the dropdowns and the IN modal hold the *formatted* value (the
+             soft-select option value, and setINValues), so a picked value is
+             mapped back to the unformatted one the filter needs */
+          const unformattedValue = (value: any) =>
+            (element.values || []).find((v: any) => v.formatted === value)
+              ?.unformatted ?? value
 
           if (operator === 'BETWEEN') {
-            rawValue = `${this.clauses.queryObj[groupInd].elements[clauseInd].value[0]} AND ${this.clauses.queryObj[groupInd][clauseInd].value[1]}`
+            const from = unformattedValue(element.value[0])
+            const to = unformattedValue(element.value[1])
+            rawValue = `${temporalValue(from) ?? from} AND ${
+              temporalValue(to) ?? to
+            }`
           } else if (operator === 'IN' || operator === 'NOT IN') {
-            this.clauses.queryObj[groupInd][clauseInd].value.forEach(
-              (element: any) => {
-                if (element.checked) {
-                  let value = element.val
+            element.value.forEach((item: any) => {
+              if (item.checked) {
+                /* map once, and use the unformatted value in every branch - the
+                   literal is built from it, and so are the fallbacks, so a
+                   formatted label can never reach RAW_VALUE */
+                const value = unformattedValue(item.val)
+                const literal = temporalValue(value)
 
-                  if (typeof value === 'string' && !isSpecialMissing(value)) {
-                    if (rawValue.length > 0) {
-                      rawValue += `,'${value}'`
-                    } else {
-                      rawValue = `('${value}'`
-                    }
+                if (literal) {
+                  rawValue += (rawValue.length > 0 ? ',' : '(') + literal
+                } else if (
+                  typeof value === 'string' &&
+                  !isSpecialMissing(value)
+                ) {
+                  if (rawValue.length > 0) {
+                    rawValue += `,'${value}'`
                   } else {
-                    if (rawValue.length > 0) {
-                      rawValue += `,${value}`
-                    } else {
-                      rawValue = `(${value}`
-                    }
+                    rawValue = `('${value}'`
+                  }
+                } else {
+                  if (rawValue.length > 0) {
+                    rawValue += `,${value}`
+                  } else {
+                    rawValue = `(${value}`
                   }
                 }
               }
-            )
+            })
 
             if (rawValue.length > 0) {
               rawValue += ')'
@@ -470,11 +492,12 @@ export class QueryComponent
               rawValue = '.'
             }
           } else {
-            if (
-              this.clauses.queryObj[groupInd].elements[clauseInd].type ===
-              'char'
-            ) {
-              rawValue = `'${this.clauses.queryObj[groupInd].elements[clauseInd].value}'`
+            const literal = temporalValue(element.value)
+
+            if (literal) {
+              rawValue = literal
+            } else if (element.type === 'char') {
+              rawValue = `'${element.value}'`
             }
           }
 

client/src/app/services/helper.service.spec.ts

@@ -46,3 +46,72 @@ describe('HelperService - convertArrayValues', () => {
     expect(result.length).toBe(3)
   })
 })
+
+describe('HelperService - sasTemporalLiteral', () => {
+  let service: HelperService
+
+  beforeEach(() => {
+    service = new HelperService(null as any)
+  })
+
+  it('renders a SAS date value as a date literal', () => {
+    expect(service.sasTemporalLiteral(42, 'DATE')).toBe("'12FEB1960'd")
+    expect(service.sasTemporalLiteral(0, 'DATE')).toBe("'01JAN1960'd")
+    expect(service.sasTemporalLiteral(34322, 'DATE')).toBe("'20DEC2053'd")
+  })
+
+  it('renders a date before 1960', () => {
+    expect(service.sasTemporalLiteral(-1, 'DATE')).toBe("'31DEC1959'd")
+  })
+
+  it('renders a SAS datetime value as a datetime literal', () => {
+    expect(service.sasTemporalLiteral(42, 'DATETIME')).toBe(
+      "'01JAN1960:00:00:42'dt"
+    )
+    expect(service.sasTemporalLiteral(86400, 'DATETIME')).toBe(
+      "'02JAN1960:00:00:00'dt"
+    )
+    expect(service.sasTemporalLiteral(31536001, 'DATETIME')).toBe(
+      "'31DEC1960:00:00:01'dt"
+    )
+  })
+
+  it('rounds double noise onto the second', () => {
+    expect(service.sasTemporalLiteral(1.9999999999999, 'TIME')).toBe(
+      "'00:00:02't"
+    )
+    expect(service.sasTemporalLiteral(42.000000000000004, 'DATETIME')).toBe(
+      "'01JAN1960:00:00:42'dt"
+    )
+  })
+
+  it('returns null rather than round a genuinely fractional value', () => {
+    expect(service.sasTemporalLiteral(42.5, 'DATETIME')).toBeNull()
+    expect(service.sasTemporalLiteral(42.5, 'TIME')).toBeNull()
+    expect(service.sasTemporalLiteral(42.3, 'TIME')).toBeNull()
+  })
+
+  it('renders a SAS time value as a time literal', () => {
+    expect(service.sasTemporalLiteral(0, 'TIME')).toBe("'00:00:00't")
+    expect(service.sasTemporalLiteral(42, 'TIME')).toBe("'00:00:42't")
+    expect(service.sasTemporalLiteral(3661, 'TIME')).toBe("'01:01:01't")
+  })
+
+  it('accepts a value that arrives as a string', () => {
+    expect(service.sasTemporalLiteral('42', 'DATE')).toBe("'12FEB1960'd")
+  })
+
+  it('returns null when the value cannot be a literal', () => {
+    // not temporal
+    expect(service.sasTemporalLiteral(42, 'N')).toBeNull()
+    expect(service.sasTemporalLiteral(42, 'C')).toBeNull()
+    expect(service.sasTemporalLiteral(42, '')).toBeNull()
+    // not a number
+    expect(service.sasTemporalLiteral('12FEB1960', 'DATE')).toBeNull()
+    expect(service.sasTemporalLiteral('', 'DATE')).toBeNull()
+    expect(service.sasTemporalLiteral(null, 'DATE')).toBeNull()
+    expect(service.sasTemporalLiteral(undefined, 'DATE')).toBeNull()
+    // a fractional number of days is not a SAS date
+    expect(service.sasTemporalLiteral(42.5, 'DATE')).toBeNull()
+  })
+})

client/src/app/services/helper.service.ts

@@ -5,6 +5,22 @@ import { SasService } from './sas.service'
 
 const librariesToShow = 50
 
+/** Month names used by a SAS date literal, eg 20APR2042. */
+const sasMonths = [
+  'JAN',
+  'FEB',
+  'MAR',
+  'APR',
+  'MAY',
+  'JUN',
+  'JUL',
+  'AUG',
+  'SEP',
+  'OCT',
+  'NOV',
+  'DEC'
+]
+
 @Injectable({
   providedIn: 'root'
 })
@@ -226,6 +242,70 @@ export class HelperService {
     })
   }
 
+  /**
+   * Formats a SAS internal date / datetime / time value as the SAS literal a
+   * WHERE clause can carry, eg:
+   *   34322 (DATE)     -> '20DEC2053'd
+   *   42    (DATETIME) -> '01JAN1960:00:00:42'dt
+   *   42    (TIME)     -> '00:00:42't
+   *
+   * The value is days since 1960-01-01 for a date, and seconds since
+   * 1960-01-01 (datetime) or since midnight (time).  The date part is built
+   * arithmetically in UTC, so the day cannot shift with the browser timezone.
+   *
+   * SAS temporal values are whole days and whole seconds, so the value is
+   * rounded to seconds first - that kills the noise a double picks up on the
+   * way here, where 1.9999999999999 is simply the second 2.  A value that is
+   * genuinely fractional does not land on a second, and rounding it would
+   * silently widen the filter, so it returns null instead.
+   *
+   * Returns null when the value is not a finite number, is not a whole day or
+   * second, or the ddtype is not temporal - callers then send the raw value.
+   */
+  public sasTemporalLiteral(value: any, ddtype: string): string | null {
+    const kind = (ddtype ?? '').toString().toUpperCase()
+    if (kind !== 'DATE' && kind !== 'DATETIME' && kind !== 'TIME') return null
+    if (value === null || value === undefined || value === '') return null
+
+    const num = typeof value === 'number' ? value : Number(String(value).trim())
+    if (!isFinite(num)) return null
+
+    /* Within the precision of a double, is this a whole second? */
+    const seconds = Math.round(num * 1e8) / 1e8
+    if (!Number.isInteger(seconds)) return null
+
+    if (kind === 'TIME') return `'${this.sasTimeFromSeconds(seconds)}'t`
+
+    const days = kind === 'DATETIME' ? Math.floor(seconds / 86400) : seconds
+    const datePart = this.sasDateFromDays(days)
+
+    if (kind === 'DATE') return `'${datePart}'d`
+
+    return `'${datePart}:${this.sasTimeFromSeconds(seconds - days * 86400)}'dt`
+  }
+
+  /** `ddMMMyyyy` for a SAS date value (days since 1960-01-01). */
+  private sasDateFromDays(days: number): string {
+    const date = new Date(Date.UTC(1960, 0, 1) + days * 86400000)
+
+    return (
+      this.addLeadingZero(date.getUTCDate()) +
+      sasMonths[date.getUTCMonth()] +
+      date.getUTCFullYear()
+    )
+  }
+
+  /** `hh:mm:ss` for a SAS time value (whole seconds since midnight). */
+  private sasTimeFromSeconds(seconds: number): string {
+    return (
+      this.addLeadingZero(Math.floor(seconds / 3600)) +
+      ':' +
+      this.addLeadingZero(Math.floor((seconds % 3600) / 60)) +
+      ':' +
+      this.addLeadingZero(seconds % 60)
+    )
+  }
+
   public addLeadingZero(value: string | number): string {
     if (typeof value !== 'string') value = value.toString()
 

client/src/app/services/licence.service.ts

@@ -8,13 +8,20 @@ import * as encoding from 'text-encoding'
 import { Router } from '@angular/router'
 import { Globvar } from '../models/sas/public-startupservice.model'
 import { freeTierConfig } from '../free-tier.config'
-import { AppStoreService } from './app-store.service'
 import { HelperService } from './helper.service'
 import { LicenceState } from '../models/LicenceState'
 import { decodeLicenceFeatures as decodeFeatures } from './utils/decodeLicenceFeatures'
 import { LoggerService } from './logger.service'
 import { EventService } from './event.service'
 
+/**
+ * Handsontable licence key used until a licence is loaded from the backend.
+ * The decrypted licence carries the customer's own key (`hot_license_key`), so
+ * this is only the pre-licence / free-tier default - it used to be supplied by
+ * the `<sasjs>` tag.
+ */
+export const DEFAULT_HOT_LICENCE_KEY = 'non-commercial-and-evaluation'
+
 @Injectable({
   providedIn: 'root'
 })
@@ -43,7 +50,7 @@ export class LicenceService {
 
   private freeTierLicenceData: LicenseKeyData = {
     demo: true,
-    hot_license_key: this.appStoreService.getDcAdapterSettings()?.hotLicenceKey,
+    hot_license_key: DEFAULT_HOT_LICENCE_KEY,
     users_allowed: this._licenceState.users_allowed || 1,
     valid_until: moment().add(1, 'year').format('YYYY-MM-DD'),
     site_id: '',
@@ -63,7 +70,6 @@ export class LicenceService {
 
   constructor(
     private loggerService: LoggerService,
-    private appStoreService: AppStoreService,
     private eventService: EventService,
     private sasService: SasService,
     private helperService: HelperService,
@@ -79,8 +85,7 @@ export class LicenceService {
   }
 
   public async activation(startupservice: any) {
-    this.freeTierLicenceData.hot_license_key =
-      this.appStoreService.getDcAdapterSettings()?.hotLicenceKey
+    this.freeTierLicenceData.hot_license_key = DEFAULT_HOT_LICENCE_KEY
 
     await this.setStartupserviceRules(startupservice)
 
@@ -225,8 +230,7 @@ export class LicenceService {
         ? licenseData.hot_license_key
         : this.hot_license_key.value
 
-    if (!hotLicenceKey)
-      hotLicenceKey = this.appStoreService.getDcAdapterSettings()?.hotLicenceKey
+    if (!hotLicenceKey) hotLicenceKey = DEFAULT_HOT_LICENCE_KEY
 
     this.hot_license_key.next(hotLicenceKey)
 

client/src/app/services/sas-store.service.spec.ts

@@ -0,0 +1,313 @@
+import { SasStoreService } from './sas-store.service'
+import { HelperService } from './helper.service'
+
+/**
+ * The filter picker keys its operator set, its variable highlight and its value
+ * quoting on Col.TYPE being exactly 'num' or 'char'.  On the normal path
+ * mergeColsRules derives that from $dataFormats; on the public/getcols fallback
+ * (used when the EDIT/VIEW limits abort the load) the service speaks the SAS
+ * vocabulary (TYPE C/N, DDTYPE separating DATE/DATETIME/TIME), so the store has
+ * to map it.  These tests pin both the mapping and the empty-payload guard.
+ */
+describe('SasStoreService', () => {
+  const buildStore = (cols: any[]) =>
+    new SasStoreService(
+      {
+        request: () => Promise.resolve({ adapterResponse: { cols } })
+      } as any,
+      {} as any,
+      {} as any
+    )
+
+  const collectColumns = (store: SasStoreService) => {
+    const seen: any[] = []
+    store.columns.subscribe((value: any) => seen.push(value))
+    return seen
+  }
+
+  describe('getCols', () => {
+    it('maps the SAS C/N TYPE onto the picker vocabulary', async () => {
+      const store = buildStore([
+        { NAME: 'CHAR_COL', TYPE: 'C', DDTYPE: 'C' },
+        { NAME: 'NUM_COL', TYPE: 'N', DDTYPE: 'N' },
+        { NAME: 'DATE_COL', TYPE: 'N', DDTYPE: 'DATE' },
+        { NAME: 'DATETIME_COL', TYPE: 'N', DDTYPE: 'DATETIME' },
+        { NAME: 'TIME_COL', TYPE: 'N', DDTYPE: 'TIME' }
+      ])
+
+      const cols = await store.getCols('DC.MY_TABLE')
+
+      expect(cols.map((col: any) => col.TYPE)).toEqual([
+        'char',
+        'num',
+        'num',
+        'num',
+        'num'
+      ])
+    })
+
+    it('leaves the rest of the column payload untouched', async () => {
+      const store = buildStore([
+        {
+          NAME: 'CHAR_COL',
+          TYPE: 'C',
+          DDTYPE: 'C',
+          LENGTH: 8,
+          LABEL: 'A label'
+        }
+      ])
+
+      const cols = await store.getCols('DC.MY_TABLE')
+
+      expect(cols[0].NAME).toBe('CHAR_COL')
+      expect(cols[0].DDTYPE).toBe('C')
+      expect(cols[0].LENGTH).toBe(8)
+      expect(cols[0].LABEL).toBe('A label')
+    })
+
+    it('returns an empty list rather than throwing when the payload is empty', async () => {
+      const store = buildStore([])
+
+      const cols = await store.getCols('DC.MY_TABLE')
+
+      expect(cols).toEqual([])
+    })
+  })
+
+  describe('setQueryVariablesFromTable', () => {
+    it('publishes the table columns without calling the service', async () => {
+      let called = false
+      const store = new SasStoreService(
+        {
+          request: () => {
+            called = true
+            return Promise.resolve({})
+          }
+        } as any,
+        {} as any,
+        {} as any
+      )
+      const seen = collectColumns(store)
+
+      await store.setQueryVariablesFromTable('DC.MY_TABLE', [{ NAME: 'A' }])
+
+      expect(called).toBe(false)
+      expect(seen.length).toBe(1)
+      expect(seen[0].data.cols.length).toBe(1)
+    })
+
+    it('falls back to getcols and publishes the mapped columns', async () => {
+      const store = buildStore([{ NAME: 'A', TYPE: 'C', DDTYPE: 'C' }])
+      const seen = collectColumns(store)
+
+      await store.setQueryVariablesFromTable('DC.MY_TABLE', [])
+
+      expect(seen.length).toBe(1)
+      expect(seen[0].libds).toBe('DC.MY_TABLE')
+      expect(seen[0].data.cols[0].TYPE).toBe('char')
+    })
+
+    it('does not publish an empty fallback, which the picker cannot render', async () => {
+      const store = buildStore([])
+      const seen = collectColumns(store)
+
+      await store.setQueryVariablesFromTable('DC.MY_TABLE', [])
+
+      expect(seen.length).toBe(0)
+    })
+  })
+})
+
+/**
+ * The clause shown in the filter panel is composed by whereClauseCreator, and
+ * its temporal rewrite turns the internal day / second count into a SAS
+ * literal.  It may only do that when the composed value really is nothing but
+ * those numbers - a value typed by hand is not, and rewriting the digit runs
+ * inside it produced clauses like '13JAN1960'd/'03JAN1960'd/'14MAY1965'd for
+ * a date typed as 12/02/1960.
+ */
+describe('SasStoreService - temporal clause preview', () => {
+  const buildStore = () =>
+    new SasStoreService({} as any, new HelperService(null as any), {} as any)
+
+  const clauseFor = (
+    value: any,
+    operator: string,
+    ddtype: string,
+    values: any[] = []
+  ) => ({
+    queryObj: [
+      {
+        clauseLogic: '',
+        elements: [
+          {
+            variable: 'SOME_DATE',
+            operator,
+            value,
+            type: 'num',
+            ddtype,
+            valueVariable: false,
+            values
+          }
+        ]
+      }
+    ],
+    groupLogic: ''
+  })
+
+  it('leaves a typed temporal value alone when it is not a number', () => {
+    const store = buildStore()
+
+    const result = store.whereClauseCreator(
+      clauseFor('12/02/1960', '=', 'DATE'),
+      '',
+      'lib.ds'
+    )
+
+    expect(result.whereClause).toContain('12/02/1960')
+    expect(result.whereClause).not.toContain("'d")
+  })
+
+  it('still shows an internal day count as a date literal', () => {
+    const store = buildStore()
+
+    const result = store.whereClauseCreator(
+      clauseFor('42', '=', 'DATE'),
+      '',
+      'lib.ds'
+    )
+
+    expect(result.whereClause).toContain("'12FEB1960'd")
+  })
+
+  /**
+   * A BETWEEN pair is composed as `<from> AND <to>`, so the guard that decides
+   * whether the rewrite may run has to see past the separator - its letters are
+   * not part of the numbers the value is built from.
+   */
+  it('shows a BETWEEN pair picked from the value list as literals', () => {
+    const store = buildStore()
+
+    const result = store.whereClauseCreator(
+      clauseFor(['12FEB1960', '13FEB1960'], 'BETWEEN', 'DATE', [
+        { formatted: '12FEB1960', unformatted: '42' },
+        { formatted: '13FEB1960', unformatted: '43' }
+      ]),
+      '',
+      'lib.ds'
+    )
+
+    expect(result.whereClause).toContain(
+      "BETWEEN '12FEB1960'd AND '13FEB1960'd"
+    )
+  })
+})
+
+/**
+ * createFilterQueryTable builds the RAW_VALUE the backend validates and stores.
+ * The range dropdown binds the formatted label, so a BETWEEN has to map each
+ * end back to its unformatted value before the literal is built - otherwise
+ * sasTemporalLiteral returns null and an unquoted date reaches the backend.
+ */
+describe('SasStoreService - temporal RAW_VALUE', () => {
+  const buildStore = () =>
+    new SasStoreService({} as any, new HelperService(null as any), {} as any)
+
+  const betweenQuery = (value: any[], values: any[]) => ({
+    groupLogic: 'AND',
+    filterGroups: [
+      {
+        clauseLogic: 'AND',
+        filterClauses: [
+          {
+            ddtype: 'DATE',
+            invalidClause: false,
+            logic: '',
+            operator: 'BETWEEN',
+            operators: [],
+            type: 'num',
+            value,
+            valueVariable: false,
+            values,
+            variable: 'SOME_DATE'
+          }
+        ]
+      }
+    ]
+  })
+
+  it('maps a BETWEEN picked from the value list back to the internal value', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      betweenQuery(
+        ['12FEB1960', '13FEB1960'],
+        [
+          { formatted: '12FEB1960', unformatted: 42 },
+          { formatted: '13FEB1960', unformatted: 43 }
+        ]
+      ) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  it('still builds literals when the values are already internal counts', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      betweenQuery(['42', '43'], []) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  const inQuery = (value: any[], values: any[]) => ({
+    groupLogic: 'AND',
+    filterGroups: [
+      {
+        clauseLogic: 'AND',
+        filterClauses: [
+          {
+            ddtype: 'DATE',
+            invalidClause: false,
+            logic: '',
+            operator: 'IN',
+            operators: [],
+            type: 'num',
+            value,
+            valueVariable: false,
+            values,
+            variable: 'SOME_DATE'
+          }
+        ]
+      }
+    ]
+  })
+
+  it('maps an IN entry picked from the value list back to the internal value', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      inQuery(
+        [{ checked: true, val: '12FEB1960' }],
+        [{ formatted: '12FEB1960', unformatted: 42 }]
+      ) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("('12FEB1960'd)")
+  })
+
+  it('passes an already-unformatted IN entry through unchanged', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      inQuery(
+        [{ checked: true, val: 42 }],
+        [{ formatted: '12FEB1960', unformatted: 42 }]
+      ) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("('12FEB1960'd)")
+  })
+})

client/src/app/services/sas-store.service.ts

@@ -295,6 +295,68 @@ export class SasStoreService {
     this.columns.next(columnsData)
   }
 
+  /**
+   * Column metadata for the filter picker, read from the service.
+   *
+   * The picker's variable list normally comes from the columns that arrived
+   * with the table (see setQueryVariablesFromTable). This is the fallback for
+   * when there is no table - public/getcols reads the metadata only.
+   *
+   * public/getcols speaks the SAS vocabulary: TYPE is C/N and DDTYPE separates
+   * DATE, DATETIME and TIME. The picker keys its operator set (query.component),
+   * its variable highlight and its value quoting (sas-store) on the client
+   * vocabulary instead - TYPE is exactly 'num' or 'char' - which
+   * mergeColsRules derives from $dataFormats on the normal path. That pass
+   * never runs for this fallback, so map TYPE here, with the same rule: the
+   * date and time types are numeric.
+   */
+  public async getCols(libds: string) {
+    const tables = { iwant: [{ libds: libds }] }
+
+    const res = await this.sasService.request('public/getcols', tables)
+
+    const cols = res.adapterResponse.cols || []
+
+    return cols.map((col: any) => ({
+      ...col,
+      TYPE:
+        ['DATE', 'DATETIME', 'TIME'].includes(col.DDTYPE) || col.TYPE === 'N'
+          ? 'num'
+          : 'char'
+    }))
+  }
+
+  /**
+   * Feeds the filter picker its variable list.
+   *
+   * `cols` is the column metadata that came back with the table, which costs
+   * nothing extra and is the normal path. When the table was not returned - the
+   * EDIT/VIEW row and cell limits abort the request before the data arrives -
+   * that list is empty, which is what makes "please filter and try again"
+   * unactionable: the picker has nothing to offer. Fall back to
+   * public/getcols, which reads the column metadata without the rows.
+   */
+  public async setQueryVariablesFromTable(libds: string, cols: any[]) {
+    if (cols && cols.length > 0) {
+      this.setQueryVariables(libds, cols)
+      return
+    }
+
+    const fallback = await this.getCols(libds)
+
+    /*
+      public/getcols emits an empty cols table when the dataset has no
+      metadata.  Publishing that would push an empty array into the filter
+      picker, whose subscriber reads cols[0].NAME - the throw is caught, but
+      the picker is left half-updated (this.cols assigned, the filter.cols
+      assignment below the throwing line never run).  There is nothing to
+      offer, so leave the picker as it was.
+    */
+    if (!fallback || fallback.length === 0) return
+
+    this.setQueryVariables(libds, fallback)
+  }
+
   public async getChangeInfo(tableId: any) {
     let obj = { TABLE: tableId }
     let table = { SASControlTable: [obj] }
@@ -534,6 +596,27 @@ export class SasStoreService {
         }
         oper = this.checkOperator(oper, value, type).operator
 
+        /* A temporal value is shown as a SAS literal (eg '20DEC2053'd) rather
+           than the internal day / second count.  Only rewrite when the composed
+           value is nothing but the numbers the clause is built from - a single
+           value, a BETWEEN pair or an IN list.  The BETWEEN separator is
+           removed before the test because ' AND ' is not part of that character
+           set; a typed value that is not a number, such as a date entered by
+           hand, would otherwise have every digit run inside it rewritten into
+           an unrelated literal. */
+        if (
+          !isValueVariable &&
+          ['DATE', 'DATETIME', 'TIME'].includes(query.ddtype) &&
+          /^[-0-9.,()\s]+$/.test(String(value).replace(/\s+AND\s+/gi, ' '))
+        ) {
+          value = String(value).replace(
+            /-?\d+(?:\.\d+)?/g,
+            (match) =>
+              this.helperService.sasTemporalLiteral(match, query.ddtype) ??
+              match
+          )
+        }
+
         if (type === 'char' && oper !== 'IN' && oper !== 'NOT IN') {
           if (typeof value === 'undefined') {
             value = ''
@@ -600,17 +683,53 @@ export class SasStoreService {
     filterQuery.filterGroups.forEach(
       (filterGroup: FilterGroup, groupIndex: number) => {
         filterGroup.filterClauses.forEach((filterClause: FilterClause) => {
+          /* A temporal value is sent as a SAS literal (eg '20APR2042'd) rather
+             than the internal day / second count, so the resolved WHERE clause
+             - and the FILTER_TEXT shown above the grid - carries a date. */
+          const temporalValue = (value: any) =>
+            filterClause.valueVariable
+              ? null
+              : this.helperService.sasTemporalLiteral(
+                  value,
+                  filterClause.ddtype
+                )
+
+          /* The range dropdown binds the formatted label, as the value list
+             and the IN modal do, so each end of a BETWEEN is mapped back to
+             its unformatted value before the literal is built.  This is the
+             same lookup whereClauseCreator applies for the preview - without
+             it the formatted string reaches sasTemporalLiteral, comes back
+             null, and RAW_VALUE carries an unquoted date that the backend
+             rejects on a numeric column. */
+          const unformattedValue = (value: any) => {
+            const match = (filterClause.values || []).find(
+              (entry: any) => entry.formatted === value
+            )
+            return match ? match.unformatted : value
+          }
+
           let rawValue = ''
           if (filterClause.operator === 'BETWEEN') {
-            rawValue = `${filterClause.value[0]} AND ${filterClause.value[1]}`
+            const from = unformattedValue(filterClause.value[0])
+            const to = unformattedValue(filterClause.value[1])
+            rawValue = `${temporalValue(from) ?? from} AND ${
+              temporalValue(to) ?? to
+            }`
           } else if (
             filterClause.operator === 'IN' ||
             filterClause.operator === 'NOT IN'
           ) {
             filterClause.value.forEach((element: any) => {
               if (element.checked) {
-                const value = element.val
-                if (
+                /* Mapped like the BETWEEN ends - whereClauseCreator normally
+                   rewrites element.val to the unformatted count before submit,
+                   so this is a pass-through for the usual path and a guard for
+                   any path that reaches here without the preview having run. */
+                const value = unformattedValue(element.val)
+                const literal = temporalValue(value)
+                if (literal) {
+                  rawValue += (rawValue.length > 0 ? ',' : '(') + literal
+                } else if (
                   typeof value === 'string' &&
                   (!isSpecialMissing(value) || filterClause.type === 'char')
                 ) {
@@ -638,7 +757,13 @@ export class SasStoreService {
               rawValue = '.'
             }
           } else {
-            if (filterClause.type === 'char' && !filterClause.valueVariable) {
+            const literal = temporalValue(filterClause.value)
+            if (literal) {
+              rawValue = literal
+            } else if (
+              filterClause.type === 'char' &&
+              !filterClause.valueVariable
+            ) {
               rawValue = `'${filterClause.value.replace(/'/g, "''")}'`
             }
           }

client/src/app/services/sas.service.spec.ts

@@ -0,0 +1,165 @@
+import { DEBUG_STATE_KEY, SasService } from './sas.service'
+
+/**
+ * SasService's constructor is a plain 8-argument constructor with an empty
+ * body, and `request()` touches only a handful of collaborators, so this
+ * stubs exactly that surface - no TestBed/DI needed, same precedent as
+ * app.service.spec.ts and va-filter.service.spec.ts.
+ *
+ * The SASjs adapter is not built by the constructor either (sasServiceInit
+ * does that), so it is assigned directly - and its `request` spy is where the
+ * outgoing body is observable, which is the whole point of these tests.
+ */
+const buildService = () => {
+  const sasjsAdapter: any = {
+    request: jasmine.createSpy('request').and.resolveTo({}),
+    getSasRequests: () => [],
+    setDebugState: jasmine.createSpy('setDebugState')
+  }
+  const loggerService: any = {
+    logRequestData: jasmine.createSpy('logRequestData')
+  }
+  const userService: any = { user: undefined }
+  const eventService: any = {
+    showAbortModal: jasmine.createSpy('showAbortModal'),
+    startupDataLoaded: jasmine.createSpy('startupDataLoaded')
+  }
+  const router: any = { navigateByUrl: jasmine.createSpy('navigateByUrl') }
+
+  const service = new SasService(
+    {} as any, // appStoreService
+    userService,
+    eventService,
+    {} as any, // sasjsService
+    {} as any, // sasViyaService
+    loggerService,
+    {} as any, // startupCheckService
+    router
+  )
+  ;(service as any).sasjsAdapter = sasjsAdapter
+
+  return { service, sasjsAdapter }
+}
+
+/** The body the adapter was asked to send for the most recent request. */
+const sentBody = (sasjsAdapter: any) =>
+  sasjsAdapter.request.calls.mostRecent().args[1]
+
+describe('SasService diagnostics payload', () => {
+  it('gives the startup service a body, so browser_info can ride on it', async () => {
+    const { service, sasjsAdapter } = buildService()
+
+    // Both startup service call sites pass null - there is nothing to send -
+    // and the service is the one whose log a support ticket is read from.
+    await service.request('public/startupservice', null)
+
+    const body = sentBody(sasjsAdapter)
+    expect(body).toBeTruthy()
+    expect(body.browser_info.length).toBe(1)
+    expect(body.browser_info[0].user_agent).toBe(navigator.userAgent)
+    expect('timezone' in body.browser_info[0]).toBe(true)
+    expect('tz_offset' in body.browser_info[0]).toBe(true)
+  })
+
+  it('leaves a non-diagnostics service payload alone', async () => {
+    const { service, sasjsAdapter } = buildService()
+
+    await service.request('public/viewlibs', null)
+
+    expect(sentBody(sasjsAdapter)).toBeNull()
+  })
+
+  it('adds browser_info to a body that already carries content', async () => {
+    const { service, sasjsAdapter } = buildService()
+
+    await service.request('editors/getdata', { table: 'MPE_X_TEST' })
+
+    const body = sentBody(sasjsAdapter)
+    expect(body.table).toBe('MPE_X_TEST')
+    expect(body.browser_info.length).toBe(1)
+  })
+
+  it('does not overwrite a browser_info the caller supplied', async () => {
+    const { service, sasjsAdapter } = buildService()
+    const supplied = [{ url: 'supplied by the caller' }]
+
+    await service.request('editors/getdata', { browser_info: supplied })
+
+    expect(sentBody(sasjsAdapter).browser_info).toBe(supplied)
+  })
+})
+
+/**
+ * The `<sasjs>` tag is re-read on every load, so the debug toggle has to survive
+ * a refresh on its own.
+ */
+describe('SasService debug state', () => {
+  /** Builds a service whose init can actually run (real adapter, stub deps). */
+  const initService = (tagDebug = false) => {
+    const appStoreService: any = {
+      getDcAdapterSettings: () => ({
+        serverUrl: '',
+        appLoc: '/Public/app/dc',
+        serverType: 'SAS9',
+        loginMechanism: '',
+        dcPath: '',
+        debug: tagDebug,
+        useComputeApi: null,
+        runAsTask: true,
+        contextName: ''
+      })
+    }
+
+    const service = new SasService(
+      appStoreService,
+      { user: undefined } as any,
+      { showInfoModal: jasmine.createSpy('showInfoModal') } as any,
+      { setup: jasmine.createSpy('setup') } as any,
+      { setup: jasmine.createSpy('setup') } as any,
+      { logRequestData: jasmine.createSpy('logRequestData') } as any,
+      { updateStep: jasmine.createSpy('updateStep') } as any,
+      { navigateByUrl: jasmine.createSpy('navigateByUrl') } as any
+    )
+
+    service.sasServiceInit()
+
+    return service
+  }
+
+  beforeEach(() => localStorage.removeItem(DEBUG_STATE_KEY))
+  afterEach(() => localStorage.removeItem(DEBUG_STATE_KEY))
+
+  it('stores the state when the toggle changes', () => {
+    const { service, sasjsAdapter } = buildService()
+
+    service.setDebugState(true)
+
+    expect(localStorage.getItem(DEBUG_STATE_KEY)).toBe('true')
+    expect(sasjsAdapter.setDebugState).toHaveBeenCalledWith(true)
+  })
+
+  it('stores a switch back off, rather than clearing it', () => {
+    const { service } = buildService()
+
+    service.setDebugState(true)
+    service.setDebugState(false)
+
+    expect(localStorage.getItem(DEBUG_STATE_KEY)).toBe('false')
+  })
+
+  it('reinstates the stored state on the next load', () => {
+    localStorage.setItem(DEBUG_STATE_KEY, 'true')
+
+    expect(initService().getSasjsConfig().debug).toBe(true)
+  })
+
+  it('leaves the tag value alone when nothing is stored', () => {
+    expect(initService(true).getSasjsConfig().debug).toBe(true)
+  })
+
+  it('reinstates an off state over a tag that turns debug on', () => {
+    localStorage.setItem(DEBUG_STATE_KEY, 'false')
+
+    expect(initService(true).getSasjsConfig().debug).toBe(false)
+  })
+})

client/src/app/services/sas.service.ts

@@ -20,6 +20,33 @@ import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapper
 import { SasViyaService } from './sas-viya.service'
 import { ViyaApiFolder } from '../viya-api-explorer/models/viya-api-folder.model'
 import { ViyaApiFolderMembers } from '../viya-api-explorer/models/viya-api-folder-content.model'
+import { parseUserAgent } from '../shared/utils/parse-user-agent'
+import { VERSION } from '../../environments/version'
+
+/**
+ * Services that receive the `browser_info` / `browser_url_vars` diagnostics
+ * tables: the startup service (so every session logs its context once) and
+ * every service that executes customer-provided code - hook scripts via
+ * %mpe_runhook (getdata, stagedata, loadfile, restore, postdata) and the
+ * dynamic cell dropdown programs (getdynamiccolvals). Other services never
+ * see the tables, so the payload stays off the high-frequency calls.
+ */
+const DIAGNOSTICS_SERVICES = [
+  'services/public/startupservice',
+  'services/editors/getdata',
+  'services/editors/getdynamiccolvals',
+  'services/editors/stagedata',
+  'services/editors/loadfile',
+  'services/editors/restore',
+  'services/auditors/postdata'
+]
+
+/**
+ * Where the debug toggle state is kept.  The `<sasjs>` tag in index.html is
+ * re-read on every load, so without this a refresh would drop the state the user
+ * set from the UI.
+ */
+export const DEBUG_STATE_KEY = 'dc-debug-state'
 
 @Injectable({
   providedIn: 'root'
@@ -69,6 +96,13 @@ export class SasService {
 
     this.sasjsAdapter = new SASjs(this.dcAdapterSettings)
 
+    /* The tag resets debug on every load, so reinstate the state the user last
+      chose from the UI. */
+    const storedDebugState = localStorage.getItem(DEBUG_STATE_KEY)
+    if (storedDebugState !== null) {
+      this.sasjsAdapter.setDebugState(storedDebugState === 'true')
+    }
+
     switch (this.dcAdapterSettings.serverType) {
       case ServerType.SasViya: {
         this.checkViyaDeploy(this.dcAdapterSettings.appLoc || '')
@@ -118,6 +152,50 @@ export class SasService {
 
     if (!wrapperOptions) wrapperOptions = {}
 
+    // Support diagnostics: tell the backend where the request came from.
+    // Sent only to the services that can act on it - the startup service
+    // (so every session logs its context once) and the services that
+    // %include customer-provided code (hook scripts, dynamic cell dropdown
+    // programs). This is the URL of this page - the Data Controller iframe -
+    // not the document that embeds it, so an embedded report can be
+    // distinguished. The browser_url_vars table carries the URL parameters
+    // as name/value pairs, which is easier for a SAS developer to read than
+    // parsing the url string.
+    // The startup service is called with a null payload - there is nothing to
+    // send - so give it one to carry the diagnostics. It is the service whose
+    // log a support ticket is read from, and without this it never receives
+    // them.
+    if (
+      DIAGNOSTICS_SERVICES.includes(url) &&
+      (data === null || data === undefined)
+    ) {
+      data = {}
+    }
+
+    if (data && typeof data === 'object' && !data.browser_info) {
+      if (DIAGNOSTICS_SERVICES.includes(url)) {
+        const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
+        const ua = parseUserAgent(navigator.userAgent)
+        data.browser_info = [
+          {
+            url: window.location.href,
+            referrer: document.referrer,
+            timezone: tz || '',
+            tz_offset: new Date().getTimezoneOffset(),
+            locale: navigator.language || '',
+            dc_version: VERSION.semverString,
+            adapter_version: VERSION.adapterVersion,
+            browser: ua.browser,
+            browser_version: ua.browserVersion,
+            platform: ua.platform,
+            user_agent: navigator.userAgent || ''
+          }
+        ]
+        const urlVars = this.collectBrowserUrlVars()
+        if (urlVars.length) data.browser_url_vars = urlVars
+      }
+    }
+
     // If debug is on it will print what is going inside the adapter
     this.loggerService.logRequestData(url, data)
 
@@ -263,6 +341,34 @@ export class SasService {
     })
   }
 
+  /**
+   * Collects the page URL parameters as name/value pairs for the
+   * `browser_url_vars` table. Parameters appear both in the search string
+   * (before the hash) and in the hash query string (Angular routes carry
+   * them after the `#`), so both are read - search first, and the first
+   * occurrence of a name wins, so the search string value beats the hash
+   * value on a collision.
+   */
+  private collectBrowserUrlVars(): Array<{ name: string; value: string }> {
+    const vars: { name: string; value: string }[] = []
+    const seen = new Set<string>()
+
+    const collect = (search: string) => {
+      new URLSearchParams(search).forEach((value, name) => {
+        if (!seen.has(name)) {
+          seen.add(name)
+          vars.push({ name, value })
+        }
+      })
+    }
+
+    if (window.location.search) collect(window.location.search.slice(1))
+    const hashQuery = window.location.hash.split('?')[1]
+    if (hashQuery) collect(hashQuery)
+
+    return vars
+  }
+
   /**
    * Uploads a file to the backend, using the adapter upload function.
    *
@@ -711,6 +817,7 @@ export class SasService {
   }
 
   public setDebugState(state: boolean) {
+    localStorage.setItem(DEBUG_STATE_KEY, String(state))
     this.sasjsAdapter.setDebugState(state)
   }
 

client/src/app/services/startup-check.service.spec.ts

@@ -0,0 +1,93 @@
+import {
+  getSasjsTagDetails,
+  SASJS_TAG_STEP,
+  StartupCheckService
+} from './startup-check.service'
+
+describe('StartupCheckService', () => {
+  let service: StartupCheckService
+
+  beforeEach(() => {
+    service = new StartupCheckService()
+  })
+
+  it('includes a SASjs tag step after the startup checks', () => {
+    const steps = service.steps$.value
+
+    expect(steps.length).toBe(SASJS_TAG_STEP + 1)
+    expect(steps[SASJS_TAG_STEP].label).toBe('SASjs tag')
+    expect(steps[SASJS_TAG_STEP].status).toBe('pending')
+  })
+
+  it('shows the tag contents against the SASjs tag step', () => {
+    service.setTagDetails([{ name: 'serverType', value: 'SASVIYA' }])
+
+    expect(service.steps$.value[SASJS_TAG_STEP]).toEqual({
+      label: 'SASjs tag',
+      status: 'done',
+      details: [{ name: 'serverType', value: 'SASVIYA' }]
+    })
+  })
+
+  it('flags a missing tag on the SASjs tag step', () => {
+    service.updateStep(SASJS_TAG_STEP, 'error', 'SASJS TAG MISSING!!')
+
+    expect(service.steps$.value[SASJS_TAG_STEP]).toEqual({
+      label: 'SASjs tag',
+      status: 'error',
+      detail: 'SASJS TAG MISSING!!'
+    })
+  })
+})
+
+describe('getSasjsTagDetails', () => {
+  const tagWith = (attributes: Record<string, string>) => {
+    const element = document.createElement('sasjs')
+
+    Object.entries(attributes).forEach(([name, value]) =>
+      element.setAttribute(name, value)
+    )
+
+    return element
+  }
+
+  it('reads the tag attributes as name/value pairs', () => {
+    const details = getSasjsTagDetails(
+      tagWith({ serverType: 'SASVIYA', debug: 'false', useComputeApi: 'true' })
+    )
+
+    expect(details).toEqual([
+      { name: 'serverType', value: 'SASVIYA' },
+      { name: 'debug', value: 'false' },
+      { name: 'useComputeApi', value: 'true' }
+    ])
+  })
+
+  it('keeps attributes the parser does not know about', () => {
+    expect(getSasjsTagDetails(tagWith({ serverTyp: 'SASJS' }))).toEqual([
+      { name: 'servertyp', value: 'SASJS' }
+    ])
+  })
+
+  it('shows every attribute, including the admin group and the licence key', () => {
+    const details = getSasjsTagDetails(
+      tagWith({
+        serverType: 'SASJS',
+        adminGroup: 'SASAdministrators',
+        hotLicenceKey: 'non-commercial-and-evaluation'
+      })
+    )
+
+    expect(details).toEqual([
+      { name: 'serverType', value: 'SASJS' },
+      { name: 'adminGroup', value: 'SASAdministrators' },
+      { name: 'hotLicenceKey', value: 'non-commercial-and-evaluation' }
+    ])
+  })
+
+  it('flags attributes that carry no value', () => {
+    expect(getSasjsTagDetails(tagWith({ runAsTask: '' }))).toEqual([
+      { name: 'runAsTask', value: '(empty)' }
+    ])
+  })
+})

client/src/app/services/startup-check.service.ts

@@ -3,18 +3,64 @@ import { BehaviorSubject } from 'rxjs'
 
 export type StartupStepStatus = 'pending' | 'in_progress' | 'done' | 'error'
 
+/** A name/value pair shown under a startup step. */
+export interface StartupStepDetail {
+  name: string
+  value: string
+}
+
 export interface StartupStep {
   label: string
   status: StartupStepStatus
   detail?: string
+  /** Rendered as a grid under the label. */
+  details?: StartupStepDetail[]
 }
 
+/**
+ * Index of the `<sasjs>` tag dump within the startup steps.  It is appended
+ * after the three startup checks so the indices the startup flow already drives
+ * (sas.service / app.service) are unchanged.
+ */
+export const SASJS_TAG_STEP = 3
+
+/**
+ * Canonical spelling of the `<sasjs>` tag attributes.  HTML lower-cases attribute
+ * names in the DOM, so they are mapped back for display.
+ */
+const TAG_ATTRIBUTE_NAMES: Record<string, string> = {
+  serverurl: 'serverUrl',
+  apploc: 'appLoc',
+  servertype: 'serverType',
+  loginmechanism: 'loginMechanism',
+  admingroup: 'adminGroup',
+  dcpath: 'dcPath',
+  debug: 'debug',
+  usecomputeapi: 'useComputeApi',
+  runastask: 'runAsTask',
+  contextname: 'contextName',
+  hotlicencekey: 'hotLicenceKey'
+}
+
+/**
+ * Reads the attributes of the `<sasjs>` tag for display in the startup checks.
+ * Every attribute is shown - including ones the parser does not know about, so a
+ * misspelt one is visible and a wrong value is exactly the misconfiguration this
+ * dump exists to expose.
+ */
+export const getSasjsTagDetails = (element: Element): StartupStepDetail[] =>
+  Array.from(element.attributes).map((attribute) => ({
+    name: TAG_ATTRIBUTE_NAMES[attribute.name.toLowerCase()] || attribute.name,
+    value: attribute.value || '(empty)'
+  }))
+
 @Injectable({ providedIn: 'root' })
 export class StartupCheckService {
   private steps: StartupStep[] = [
     { label: 'Checking app location', status: 'pending' },
     { label: 'Checking Viya deploy', status: 'pending' },
-    { label: 'Loading startup service', status: 'pending' }
+    { label: 'Loading startup service', status: 'pending' },
+    { label: 'SASjs tag', status: 'pending' }
   ]
 
   public steps$ = new BehaviorSubject<StartupStep[]>([...this.steps])
@@ -30,6 +76,16 @@ export class StartupCheckService {
     }
   }
 
+  /** Shows the `<sasjs>` tag contents on the boot screen. */
+  public setTagDetails(details: StartupStepDetail[]) {
+    this.steps[SASJS_TAG_STEP] = {
+      ...this.steps[SASJS_TAG_STEP],
+      status: 'done',
+      details
+    }
+    this.steps$.next([...this.steps])
+  }
+
   public setSteps(steps: StartupStep[]) {
     this.steps = steps
     this.steps$.next([...this.steps])

client/src/app/services/va-filter.service.spec.ts

@@ -1,27 +1,6 @@
 import { VaFilterService } from './va-filter.service'
 import { VaMessage } from './va-messaging.service'
-
-/**
- * Minimal stand-in for HelperService.convertJsDateToSasDate using the real
- * epoch math, so date/datetime assertions use the actual SAS internal values
- * without pulling in HelperService's dependency graph (SasService etc.).
- */
-class HelperServiceStub {
-  convertJsDateToSasDate(jsDate: Date, unit = 'days'): number {
-    const ms = Date.UTC(
-      jsDate.getFullYear(),
-      jsDate.getMonth(),
-      jsDate.getDate(),
-      jsDate.getHours(),
-      jsDate.getMinutes(),
-      jsDate.getSeconds()
-    )
-    const sasMs = ms + 315619200000
-    return unit === 'days'
-      ? Math.floor(Math.abs(sasMs / 86400000))
-      : sasMs / 1000
-  }
-}
+import { HelperService } from './helper.service'
 
 function msg(parameters: any[]): VaMessage {
   return {
@@ -63,7 +42,7 @@ describe('VaFilterService', () => {
   ]
 
   beforeEach(() => {
-    service = new VaFilterService(new HelperServiceStub() as any)
+    service = new VaFilterService(new HelperService({} as any))
   })
 
   const labelToVar = () =>
@@ -109,28 +88,30 @@ describe('VaFilterService', () => {
       expect(build([{ label: 'SOME_NUM', value: 'abc' }])).toEqual([])
     })
 
-    it('converts temporal values to SAS internal numbers', () => {
+    it('renders temporal values as SAS literals', () => {
       expect(
         build([{ label: 'SOME_TIME', value: '0:00:02' }])[0].RAW_VALUE
-      ).toBe('2')
+      ).toBe("'00:00:02't")
       expect(
         build([{ label: 'SOME_DATE', value: '01JUL1960' }])[0].RAW_VALUE
-      ).toBe('182')
+      ).toBe("'01JUL1960'd")
       expect(
         build([{ label: 'SOME_DATETIME', value: '01JAN1960:00:06:05' }])[0]
           .RAW_VALUE
-      ).toBe('365')
+      ).toBe("'01JAN1960:00:06:05'dt")
     })
 
-    it('passes an already-internal (bare number) temporal value through raw', () => {
+    it('renders an already-internal (bare number) temporal value as a literal', () => {
       // VA sends the raw SAS internal value when the data item has no format.
       expect(build([{ label: 'SOME_DATE', value: '182' }])[0].RAW_VALUE).toBe(
-        '182'
+        "'01JUL1960'd"
       )
       expect(
         build([{ label: 'SOME_DATETIME', value: '365' }])[0].RAW_VALUE
-      ).toBe('365')
-      expect(build([{ label: 'SOME_TIME', value: '2' }])[0].RAW_VALUE).toBe('2')
+      ).toBe("'01JAN1960:00:06:05'dt")
+      expect(build([{ label: 'SOME_TIME', value: '2' }])[0].RAW_VALUE).toBe(
+        "'00:00:02't"
+      )
     })
 
     it('parses ISO date shapes identically to DATE9 (VA sends no format)', () => {
@@ -232,9 +213,9 @@ describe('VaFilterService', () => {
       expect(clauses.map((c) => [c.VARIABLE_NM, c.RAW_VALUE])).toEqual([
         ['SOME_CHAR', "'x'"],
         ['SOME_NUM', '10'],
-        ['SOME_TIME', '2'],
-        ['SOME_DATE', '182'],
-        ['SOME_DATETIME', '365']
+        ['SOME_TIME', "'00:00:02't"],
+        ['SOME_DATE', "'01JUL1960'd"],
+        ['SOME_DATETIME', "'01JAN1960:00:06:05'dt"]
       ])
     })
 
@@ -247,9 +228,9 @@ describe('VaFilterService', () => {
         ])
       )
       expect(clauses.map((c) => [c.VARIABLE_NM, c.RAW_VALUE])).toEqual([
-        ['SOME_DATE', '182'],
-        ['SOME_DATE2', '182'],
-        ['SOME_DTTM', '365']
+        ['SOME_DATE', "'01JUL1960'd"],
+        ['SOME_DATE2', "'01JUL1960'd"],
+        ['SOME_DTTM', "'01JAN1960:00:06:05'dt"]
       ])
     })
   })

client/src/app/services/va-filter.service.ts

@@ -223,15 +223,18 @@ export class VaFilterService {
 
   /**
    * Formats one VA filter value into a DC RAW_VALUE for the column kind, or null
-   * when invalid (dropped). Matches DC's native filters, which send the NUMERIC
-   * INTERNAL value (not SAS literals — those error out in the backend):
-   *  - numeric  -> finite number, raw (else null — injection guard)
-   *  - time     -> seconds since midnight   (VA: "0:00:02" -> 2)
-   *  - date     -> days since 1960-01-01    (VA: "01JUL1960" -> 182)
-   *  - datetime -> seconds since 1960-01-01 (VA: "01JAN1960:00:06:05" -> 365)
+   * when invalid (dropped). Matches DC's native filters, which send a SAS
+   * literal for a temporal column and the raw value otherwise:
+   *  - numeric  -> finite number, raw (else null - injection guard)
+   *  - time     -> seconds since midnight, as a SAS time literal
+   *                (VA: "0:00:02" -> '00:00:02't)
+   *  - date     -> days since 1960-01-01, as a SAS date literal
+   *                (VA: "01JUL1960" -> '01JUL1960'd)
+   *  - datetime -> seconds since 1960-01-01, as a SAS datetime literal
+   *                (VA: "01JAN1960:00:06:05" -> '01JAN1960:00:06:05'dt)
    *  - char     -> single-quote escaped
    * Temporal strings are regex-parsed (digits/letters/colons only), so a value
-   * either yields a finite number or is dropped — also the injection guard.
+   * either yields a finite number or is dropped - also the injection guard.
    * Date/datetime reuse HelperService.convertJsDateToSasDate so the value is
    * byte-identical to what DC's own pickers produce.
    *
@@ -239,7 +242,7 @@ export class VaFilterService {
    * itself (VA sends it unformatted when the data item carries no format / is
    * used as a measure), so it is passed through raw. This is unambiguous because
    * the only date formats in use (DATE9, ISO) always contain a letter or a
-   * separator — a plain integer is never a formatted date.
+   * separator - a plain integer is never a formatted date.
    */
   private formatValueByKind(
     value: any,
@@ -254,23 +257,32 @@ export class VaFilterService {
         return Number.isFinite(num) ? `${num}` : null
       }
       case 'time': {
-        if (this.isBareNumber(s)) return `${Number(s)}`
-        const secs = this.timeToSeconds(s)
-        return secs === null ? null : `${secs}`
+        const secs = this.isBareNumber(s) ? Number(s) : this.timeToSeconds(s)
+        return secs === null
+          ? null
+          : this.helperService.sasTemporalLiteral(secs, 'TIME')
       }
       case 'date': {
-        if (this.isBareNumber(s)) return `${Number(s)}`
-        const d = this.parseDate(s, order)
-        return d
-          ? `${this.helperService.convertJsDateToSasDate(d, 'days')}`
-          : null
+        let days: number | null = null
+        if (this.isBareNumber(s)) days = Number(s)
+        else {
+          const d = this.parseDate(s, order)
+          if (d) days = this.helperService.convertJsDateToSasDate(d, 'days')
+        }
+        return days === null
+          ? null
+          : this.helperService.sasTemporalLiteral(days, 'DATE')
       }
       case 'datetime': {
-        if (this.isBareNumber(s)) return `${Number(s)}`
-        const d = this.parseDatetime(s, order)
-        return d
-          ? `${this.helperService.convertJsDateToSasDate(d, 'seconds')}`
-          : null
+        let secs: number | null = null
+        if (this.isBareNumber(s)) secs = Number(s)
+        else {
+          const d = this.parseDatetime(s, order)
+          if (d) secs = this.helperService.convertJsDateToSasDate(d, 'seconds')
+        }
+        return secs === null
+          ? null
+          : this.helperService.sasTemporalLiteral(secs, 'DATETIME')
       }
       default:
         return `'${s.replace(/'/g, "''")}'`

client/src/app/shared/dc-validator/dc-validator.ts

@@ -61,13 +61,13 @@ export class DcValidator {
     // injected separately — see its own doc comment for why.
     //
     // COLHEADERS is the authoritative grid order and the only safe key to
-    // build against: it is the variable list of the *output* dataset, where
-    // getdata.sas hoists the delete column and the PK to the front ("order
-    // delete var and pk fields at start of table"), whereas cols[].VARNUM
-    // describes the *source* table and cols[] itself can arrive sorted by
-    // NAME. Those orders diverge whenever the PK isn't already the source's
-    // first column, so rules are keyed by name and emitted in COLHEADERS
-    // order — never paired positionally against cols[] or VARNUM.
+    // build against: it is the variable list of the *output* dataset, and the
+    // editor prepends the client-owned delete column to it (see
+    // editor.component.ts's initSetup) before this constructor runs, whereas
+    // cols[].VARNUM describes the *source* table and cols[] itself can arrive
+    // sorted by NAME. Those orders diverge whenever the PK isn't already the
+    // source's first column, so rules are keyed by name and emitted in
+    // COLHEADERS order — never paired positionally against cols[] or VARNUM.
     //
     // A column with no parseable COLTYPE falls back to a plain text rule
     // rather than being dropped: getRules() is consumed positionally against
@@ -92,14 +92,49 @@ export class DcValidator {
     this.rules.push({ ...EDIT_STATUS_COLUMN_RULE })
     this.hiddenColumns.push(this.rules.length - 1)
 
-    this.dqrules = dqRules
+    this.dqrules = [...dqRules]
     this.dqdata = dqData
     this.primaryKeys = sasparams.PK.split(' ')
 
+    // A primary key is NOT NULL by definition, so it must reject a blank and
+    // a special missing (".A"-".Z", "._") even when the target table carries
+    // no physical NOT NULL constraint and MPE_VALIDATIONS has no NOTNULL rule
+    // for it.  Synthesised here so every keyed table gets it, and so the grid,
+    // the edit-record modal and Excel upload validation all see the same rule.
+    this.addPrimaryKeyNotNullRules()
+
     this.updateDqData()
     this.setupValidations()
   }
 
+  /**
+   * Adds a NOTNULL rule for each primary key column that does not already
+   * have one.  A primary key identifies the row, so a blank or a special
+   * missing there is never valid.
+   */
+  private addPrimaryKeyNotNullRules(): void {
+    for (const pk of this.primaryKeys) {
+      if (!pk) continue
+
+      // A buskey can name a column the table no longer has - do not
+      // synthesise a rule for a column that is not in the grid.
+      if (!this.rules.some((rule) => rule.data === pk)) continue
+
+      const hasNotNull = this.dqrules.some(
+        (rule) => rule.BASE_COL === pk && rule.RULE_TYPE === 'NOTNULL'
+      )
+
+      if (!hasNotNull) {
+        this.dqrules.push({
+          BASE_COL: pk,
+          RULE_TYPE: 'NOTNULL',
+          RULE_VALUE: '',
+          X: 1
+        })
+      }
+    }
+  }
+
   registerCustomEditors() {
     Handsontable.editors.registerEditor(
       'autocomplete.custom',
@@ -283,7 +318,14 @@ export class DcValidator {
              * So we will convert it before pushing to array.
              */
             if (rule.type && rule.type === 'numeric') {
-              details.push(Number(data['RULE_DATA']))
+              // A special missing reaches us as a bare letter ("A", "_"), and
+              // the regular missing as "." - Number() would turn either into
+              // NaN, so a strict (HARDSELECT) dropdown could never accept a
+              // value the column actually holds.  Keep them as they are.
+              const rawValue = data['RULE_DATA']
+              details.push(
+                isSpecialMissing(rawValue) ? rawValue : Number(rawValue)
+              )
             } else {
               details.push(data['RULE_DATA'])
             }

client/src/app/shared/dc-validator/tests/dc-validator.spec.ts

@@ -118,7 +118,12 @@ describe('DC Validator', () => {
     expect(dcValidator.getRule('SOME_TIME')).toBeUndefined()
 
     // Test data quality functions
-    expect(dcValidator.getDqDetails()).toHaveSize(dqRules.length)
+    // dqRules + 2 synthesised rules: the SOFTSELECT rule updateDqData()
+    // derives for SOME_DROPDOWN out of dqdata, and the NOTNULL rule
+    // addPrimaryKeyNotNullRules() gives the primary key column (example_dqRules
+    // has none for it).  Note the constructor copies dqRules rather than
+    // aliasing it, so this array is no longer mutated by construction.
+    expect(dcValidator.getDqDetails()).toHaveSize(dqRules.length + 2)
     expect(dcValidator.getDqDetails('non_existant')).toHaveSize(0)
     expect(dcValidator.getDqDetails('SOME_NUM')).toHaveSize(2)
     expect(dcValidator.isDqCol('SOME_NUM')).toBeTrue()
@@ -133,6 +138,91 @@ describe('DC Validator', () => {
     ])
   })
 
+  it('treats the primary key column as NOT NULL, even with no NOTNULL rule configured', () => {
+    const sasparams: SASParam = example_sasparams
+    const cols: Col[] = example_cols
+    const dqRules: DQRule[] = example_dqRules // no NOTNULL for PRIMARY_KEY_FIELD
+    const dqData: DQData[] = example_dqData
+    const $dataFormats: $DataFormats = example_dataformats
+
+    const dcValidator: DcValidator = new DcValidator(
+      sasparams,
+      $dataFormats,
+      cols,
+      dqRules,
+      dqData
+    )
+
+    const pkRules = dcValidator.getDqDetails('PRIMARY_KEY_FIELD')
+    expect(pkRules.some((rule) => rule.RULE_TYPE === 'NOTNULL')).toBeTrue()
+
+    const pkRule = dcValidator.getRule('PRIMARY_KEY_FIELD')
+
+    // A primary key identifies the row, so neither a blank nor a special
+    // missing can satisfy it.
+    dcValidator.executeHotValidator(pkRule!, null, (valid: boolean) => {
+      expect(valid).toBeFalse()
+    })
+    dcValidator.executeHotValidator(pkRule!, 'A', (valid: boolean) => {
+      expect(valid).toBeFalse()
+    })
+    dcValidator.executeHotValidator(pkRule!, 5, (valid: boolean) => {
+      expect(valid).toBeTrue()
+    })
+  })
+
+  it('keeps a special missing in a numeric dropdown source, so a strict rule can match it', () => {
+    const dqData: DQData[] = [
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: 1,
+        SELECTBOX_ORDER: 1
+      },
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: 'A',
+        SELECTBOX_ORDER: 2
+      },
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: '_',
+        SELECTBOX_ORDER: 3
+      },
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: '.',
+        SELECTBOX_ORDER: 4
+      }
+    ] as DQData[]
+
+    const dcValidator: DcValidator = new DcValidator(
+      example_sasparams,
+      example_dataformats,
+      example_cols,
+      example_dqRules,
+      dqData
+    )
+
+    // SOME_NUM is numeric and carries a HARDSELECT_HOOK, so its dropdown
+    // source comes from dqdata.  Number() would have NaN'd the special
+    // missings, leaving the strict membership test unable to match a value
+    // the column actually holds.
+    const source = dcValidator.getDqDropdownSource(
+      dcValidator.getRule('SOME_NUM')!
+    )
+
+    expect(source[0]).toEqual(1)
+    expect(source[1]).toEqual('A')
+    expect(source[2]).toEqual('_')
+    // the regular missing is a dropdown option too, not a NaN
+    expect(source[3]).toEqual('.')
+    expect(source.some((value) => Number.isNaN(value as number))).toBeFalse()
+  })
+
   it('should test hot validator', () => {
     const sasparams: SASParam = example_sasparams
     const cols: Col[] = example_cols
@@ -163,21 +253,22 @@ describe('DC Validator', () => {
     dcValidator.executeHotValidator(someNumRule!, 'ss', (valid: boolean) => {
       expect(valid).toBeFalse()
     })
-    //Special missings
+    // Special missings - a SAS NOT NULL (or primary key) constraint rejects
+    // these, so the rule must reject them too: they are NULL, not values.
     dcValidator.executeHotValidator(someNumRule!, 's', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '.s', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '.', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '..', (valid: boolean) => {
       expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '._', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
 
     // MINVAL, MAXVAL Validation
@@ -193,7 +284,7 @@ describe('DC Validator', () => {
       expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(shortNumRule!, 's', (valid: boolean) => {
-      expect(valid).toBeFalse() // Special missings are lowest numbers, if any MINVAL is set, special missing is always lower
+      expect(valid).toBeFalse() // Special missings are the lowest numbers, so any MINVAL is above them
     })
 
     // CASE validation
@@ -462,11 +553,27 @@ describe('DC Validator', () => {
         example_dqData
       )
 
+    // HOT 18.1 tightened `ColumnSettings['validator']` to the real
+    // `string | RegExp | function` union (18.0 erased it behind an index
+    // signature), so narrow it the way the production code does before
+    // invoking it.
+    const validatorOf = (rule: ReturnType<DcValidator['getRule']>) => {
+      const validator = rule?.validator
+      if (typeof validator !== 'function') {
+        throw new Error('expected the rule to carry a function validator')
+      }
+      return validator as (
+        this: any,
+        value: any,
+        callback: (valid: boolean) => void
+      ) => void
+    }
+
     it('exempts an invalid non-PK cell on a delete-marked row', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(true),
         'not a number',
         (valid: boolean) => {
@@ -479,7 +586,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(true),
         null,
         (valid: boolean) => {
@@ -492,7 +599,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(false),
         'not a number',
         (valid: boolean) => {
@@ -505,7 +612,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(false),
         2,
         (valid: boolean) => {
@@ -518,7 +625,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const pkRule = dcValidator.getRule('PRIMARY_KEY_FIELD')
 
-      pkRule!.validator!.call(
+      validatorOf(pkRule).call(
         fakeCellProps(true),
         'not a number',
         (valid: boolean) => {

client/src/app/shared/dc-validator/tests/dqValidation.spec.ts

@@ -24,7 +24,44 @@ describe('DC Validator - dq validation', () => {
     expect(dqValidate(dqRules, invalidValue)).toBeFalse()
     expect(dqValidate(dqRules, invalidStringValue)).toBeFalse()
     expect(dqValidate(dqRules, numericStringValue)).toBeTrue()
+    // A missing sorts below every number, so it is below the floor.
     expect(dqValidate(dqRules, numericSpecialMissingValue)).toBeFalse()
+    expect(dqValidate(dqRules, null)).toBeFalse()
+    expect(dqValidate(dqRules, undefined)).toBeFalse()
+  })
+
+  it('should order the missing values in a range rule as SAS does', () => {
+    const missingRange: DQRule[] = [
+      { BASE_COL: 'test', RULE_TYPE: 'MINVAL', RULE_VALUE: '.A', X: 0 },
+      { BASE_COL: 'test', RULE_TYPE: 'MAXVAL', RULE_VALUE: '.C', X: 0 }
+    ]
+
+    // Inside the range of missings
+    expect(dqValidate(missingRange, '.A')).toBeTrue()
+    expect(dqValidate(missingRange, '.B')).toBeTrue()
+    expect(dqValidate(missingRange, '.C')).toBeTrue()
+    expect(dqValidate(missingRange, 'b')).toBeTrue()
+
+    // Outside it - above the ceiling
+    expect(dqValidate(missingRange, '.D')).toBeFalse()
+    expect(dqValidate(missingRange, 'z')).toBeFalse()
+
+    // Outside it - below the floor.  The regular missing sits between ._ and .A
+    expect(dqValidate(missingRange, '._')).toBeFalse()
+    expect(dqValidate(missingRange, null)).toBeFalse()
+
+    // Every number sorts above every missing, so it is above the ceiling
+    expect(dqValidate(missingRange, 0)).toBeFalse()
+    expect(dqValidate(missingRange, 5)).toBeFalse()
+
+    // A floor of .A alone still lets the numbers through: they are above it
+    const missingFloor: DQRule[] = [
+      { BASE_COL: 'test', RULE_TYPE: 'MINVAL', RULE_VALUE: '.A', X: 0 }
+    ]
+    expect(dqValidate(missingFloor, '.A')).toBeTrue()
+    expect(dqValidate(missingFloor, '.Z')).toBeTrue()
+    expect(dqValidate(missingFloor, 5)).toBeTrue()
+    expect(dqValidate(missingFloor, '._')).toBeFalse()
   })
 
   it('should validate MAXVAL value', () => {
@@ -49,7 +86,10 @@ describe('DC Validator - dq validation', () => {
     expect(dqValidate(dqRules, invalidValue)).toBeFalse()
     expect(dqValidate(dqRules, invalidStringValue)).toBeFalse()
     expect(dqValidate(dqRules, numericStringValue)).toBeTrue()
+    // A missing sorts below every number, so it is below the ceiling
     expect(dqValidate(dqRules, numericSpecialMissingValue)).toBeTrue()
+    expect(dqValidate(dqRules, null)).toBeTrue()
+    expect(dqValidate(dqRules, undefined)).toBeTrue()
   })
 
   it('should validate UPCASE value', () => {
@@ -111,6 +151,43 @@ describe('DC Validator - dq validation', () => {
     expect(dqValidate(dqRules, invalidValue2)).toBeFalse()
   })
 
+  it('should reject a special missing on a numeric column (a SAS NOT NULL constraint does)', () => {
+    const dqRules: DQRule[] = [
+      {
+        BASE_COL: 'test',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: ' ',
+        X: 0
+      }
+    ]
+
+    // The values a real SAS service delivers for a numeric column.
+    expect(dqValidate(dqRules, 'A', true)).toBeFalse()
+    expect(dqValidate(dqRules, '_', true)).toBeFalse()
+    expect(dqValidate(dqRules, '.', true)).toBeFalse()
+    // Ordinary numbers and numeric strings still pass.
+    expect(dqValidate(dqRules, 5, true)).toBeTrue()
+    expect(dqValidate(dqRules, '5', true)).toBeTrue()
+  })
+
+  it('should not reject a single letter on a character column', () => {
+    const dqRules: DQRule[] = [
+      {
+        BASE_COL: 'test',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: ' ',
+        X: 0
+      }
+    ]
+
+    // There is no special-missing concept on a character column - a lone
+    // letter is ordinary data.
+    expect(dqValidate(dqRules, 'A')).toBeTrue()
+    expect(dqValidate(dqRules, '_')).toBeTrue()
+    expect(dqValidate(dqRules, '.')).toBeTrue()
+    expect(dqValidate(dqRules, '', false)).toBeFalse()
+  })
+
   it('should return true if rule not found', () => {
     const validValue = 5
 

client/src/app/shared/dc-validator/utils/deleteRecordColumn.spec.ts

@@ -0,0 +1,75 @@
+import {
+  prepareDeleteRecordColumn,
+  seedDeleteRecordColumn
+} from './deleteRecordColumn'
+import { DELETE_RECORD_COLUMN_NAME } from './deleteRecordColumnRule'
+
+const response = (overrides: any = {}) => ({
+  data: {
+    sasparams: [{ COLHEADERS: 'PRIMARY_KEY_FIELD,SOME_CHAR' }],
+    cols: [],
+    $sasdata: { vars: {} },
+    ...overrides
+  }
+})
+
+describe('prepareDeleteRecordColumn', () => {
+  it('prepends the delete column to COLHEADERS', () => {
+    const res = response()
+
+    prepareDeleteRecordColumn(res)
+
+    expect(res.data.sasparams[0].COLHEADERS).toEqual(
+      `${DELETE_RECORD_COLUMN_NAME},PRIMARY_KEY_FIELD,SOME_CHAR`
+    )
+  })
+
+  it('throws when COLHEADERS already carries the delete column', () => {
+    const res = response({
+      sasparams: [{ COLHEADERS: `${DELETE_RECORD_COLUMN_NAME},SOME_CHAR` }]
+    })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError(/reserved column/)
+  })
+
+  it('throws when cols carries the delete column', () => {
+    const res = response({ cols: [{ NAME: DELETE_RECORD_COLUMN_NAME }] })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError(/reserved column/)
+  })
+
+  it('throws when $sasdata vars carries the delete column', () => {
+    const res = response({
+      $sasdata: { vars: { [DELETE_RECORD_COLUMN_NAME]: {} } }
+    })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError(/reserved column/)
+  })
+
+  it('leaves COLHEADERS untouched when it throws', () => {
+    const colHeaders = `${DELETE_RECORD_COLUMN_NAME},SOME_CHAR`
+    const res = response({ sasparams: [{ COLHEADERS: colHeaders }] })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError()
+    expect(res.data.sasparams[0].COLHEADERS).toEqual(colHeaders)
+  })
+})
+
+describe('seedDeleteRecordColumn', () => {
+  it("sets every row's delete flag to 'No'", () => {
+    const rows: any[] = [{ PRIMARY_KEY_FIELD: 1 }, { PRIMARY_KEY_FIELD: 2 }]
+
+    seedDeleteRecordColumn(rows)
+
+    expect(rows[0][DELETE_RECORD_COLUMN_NAME]).toEqual('No')
+    expect(rows[1][DELETE_RECORD_COLUMN_NAME]).toEqual('No')
+  })
+
+  it('overwrites a flag a row already carried', () => {
+    const rows: any[] = [{ [DELETE_RECORD_COLUMN_NAME]: 'Yes' }]
+
+    seedDeleteRecordColumn(rows)
+
+    expect(rows[0][DELETE_RECORD_COLUMN_NAME]).toEqual('No')
+  })
+})

client/src/app/shared/dc-validator/utils/deleteRecordColumn.ts

@@ -0,0 +1,64 @@
+import { Col } from '../models/col.model'
+import { DELETE_RECORD_COLUMN_NAME } from './deleteRecordColumnRule'
+
+/**
+ * The parts of a getdata response this module touches. Declared structurally
+ * so the editor and the multi-dataset loader can both use it without either
+ * owning the other's response type.
+ */
+interface DeleteColumnResponse {
+  data: {
+    sasparams: { COLHEADERS: string }[]
+    cols?: Col[]
+    $sasdata?: { vars?: Record<string, unknown> }
+  }
+}
+
+/**
+ * Asserts that getdata did NOT send the client-owned delete column, then
+ * prepends it to COLHEADERS so the grid's first column is the delete flag.
+ *
+ * `_____DELETE__THIS__RECORD_____` is created by the editor itself (see
+ * deleteRecordColumnRule.ts) and getdata must never emit it. Its presence in a
+ * response can only mean a real table column of that name, or a hook that
+ * re-added it - either way it would shadow the delete flag, so refuse the load
+ * rather than render a table whose delete column cannot be trusted. Callers
+ * run this inside a promise chain whose `.catch` surfaces the error (the
+ * editor sets `getdataError`).
+ *
+ * Call it BEFORE constructing a DcValidator or deriving headerArray: rules,
+ * `columnHeader` and `headerArray` are all keyed off COLHEADERS order, and the
+ * delete column is always index 0.
+ */
+export const prepareDeleteRecordColumn = (response: DeleteColumnResponse) => {
+  const sasparams = response.data.sasparams[0]
+
+  const sentByBackend =
+    sasparams.COLHEADERS.split(',').includes(DELETE_RECORD_COLUMN_NAME) ||
+    (response.data.cols || []).some(
+      (col) => col.NAME === DELETE_RECORD_COLUMN_NAME
+    ) ||
+    Object.prototype.hasOwnProperty.call(
+      response.data.$sasdata?.vars || {},
+      DELETE_RECORD_COLUMN_NAME
+    )
+
+  if (sentByBackend) {
+    throw new Error(
+      `getdata returned the reserved column ${DELETE_RECORD_COLUMN_NAME}. ` +
+        'That name is created by the editor itself and must not exist in the target table.'
+    )
+  }
+
+  sasparams.COLHEADERS = `${DELETE_RECORD_COLUMN_NAME},${sasparams.COLHEADERS}`
+}
+
+/**
+ * Seeds every row's delete flag to 'No' - getdata no longer sends the column,
+ * so the editor owns the value too (see deleteRecordColumnRule.ts).
+ */
+export const seedDeleteRecordColumn = (rows: any[]) => {
+  for (const row of rows) {
+    row[DELETE_RECORD_COLUMN_NAME] = 'No'
+  }
+}

client/src/app/shared/dc-validator/utils/deleteRecordColumnRule.ts

@@ -1,16 +1,21 @@
 import { DcValidation } from '../models/dc-validation.model'
 
 /**
- * `_____DELETE__THIS__RECORD_____` is a client-side UI concept (the delete
- * checkbox column) that getdata.sas synthesizes into vars3/vars4 for display
- * purposes only — it's never a real column of the underlying dataset, so
- * `%mp_getcols` (which builds `cols1`, the left side of the join into
- * `cols`) never has a row for it, and its rule can no longer travel via
- * `cols[].COLTYPE` (see issue #253's vertical-array fix). The rule is fixed
- * and data-independent, so it's injected directly here instead.
+ * `_____DELETE__THIS__RECORD_____` is a purely client-owned column - the
+ * delete checkbox column. getdata.sas does NOT emit it: the editor adds the
+ * name to COLHEADERS itself (see editor.component.ts's initSetup) and seeds
+ * every row to 'No'. A response that carries it is refused, because the name
+ * can then only come from a real table column of that name (or a hook that
+ * re-added it), which would silently shadow the delete flag.
+ *
+ * Its rule can never travel via `cols[].COLTYPE` - `%mp_getcols` (which
+ * builds `cols1`, the left side of the join into `cols`) has no row for it -
+ * so it is injected directly, here and in DcValidator's constructor.
  */
+export const DELETE_RECORD_COLUMN_NAME = '_____DELETE__THIS__RECORD_____'
+
 export const DELETE_RECORD_COLUMN_RULE: DcValidation = {
-  data: '_____DELETE__THIS__RECORD_____',
+  data: DELETE_RECORD_COLUMN_NAME,
   type: 'dropdown',
   source: ['No', 'Yes']
 }

client/src/app/shared/dc-validator/validations/dq-validation.ts

@@ -1,8 +1,49 @@
 import { DQRule } from '../models/dq-rules.model'
-import { specialMissingNumericValidator } from './hot-custom-validators'
+import { isSpecialMissing } from '@sasjs/utils/input/validators'
 import { isRegexRuleExempt } from '../utils/isRegexRuleExempt'
 import { parseRegexRule } from '../utils/parseRegexRule'
 
+/**
+ * A SAS numeric variable's values have a total order, and its missing values sit
+ * below every non-missing value.  The missing values are themselves ordered:
+ * `._` is the lowest, then the regular missing, then `.A` through `.Z`.
+ *
+ * A range rule compares in that order.  That is what makes a range of missings
+ * meaningful - with `MINVAL .A` and `MAXVAL .C`, `.B` is inside the range and `.D`
+ * is outside it - and it is also why a special missing fails a numeric floor: it
+ * sorts below every number.
+ *
+ * The key is a pair: the class (0 for a missing, 1 for a number, so that every
+ * number sorts above every missing) and the position within that class.  A value
+ * that is neither a number nor a missing has no place in the order and gets null,
+ * which no rule accepts.
+ */
+const sasNumericOrderKey = (value: any): [number, number] | null => {
+  if (value === undefined || value === null || value === '') return [0, 1] // regular missing
+
+  if (typeof value === 'string') {
+    const upper = value.trim().toUpperCase()
+
+    if (upper === '.' || upper === '') return [0, 1] // regular missing
+    if (upper === '._' || upper === '_') return [0, 0] // the lowest missing
+    if (/^\.?[A-Z]$/.test(upper))
+      return [0, 2 + (upper.charCodeAt(upper.length - 1) - 65)] // .A .. .Z
+
+    const numValue = parseFloat(upper)
+
+    return isNaN(numValue) ? null : [1, numValue]
+  }
+
+  const numValue = Number(value)
+
+  return isNaN(numValue) ? null : [1, numValue]
+}
+
+const compareSasNumericOrder = (
+  a: [number, number],
+  b: [number, number]
+): number => (a[0] !== b[0] ? a[0] - b[0] : a[1] - b[1])
+
 const dqValidation: {
   [key: string]: (
     value: any,
@@ -33,25 +74,39 @@ const dqValidation: {
     return true
   },
   MINVAL: (value: any, ruleValue: string | number): boolean => {
-    const isValidNumeric = specialMissingNumericValidator(value)
-    const numValue = parseFloat(value)
+    const valueKey = sasNumericOrderKey(value)
+    const ruleKey = sasNumericOrderKey(ruleValue)
 
-    // If it's validNumeric and it is NaN it means it is special numeric, and those are always less then any
-    // min value set
-    if (isValidNumeric && isNaN(numValue)) return false
+    // A value that is neither a number nor a missing has no place in the order,
+    // so nothing satisfies the rule.
+    if (!valueKey || !ruleKey) return false
 
-    return numValue >= Number(ruleValue.toString())
+    return compareSasNumericOrder(valueKey, ruleKey) >= 0
   },
   MAXVAL: (value: any, ruleValue: string | number): boolean => {
-    const isValidNumeric = specialMissingNumericValidator(value)
-    const numValue = parseFloat(value)
+    const valueKey = sasNumericOrderKey(value)
+    const ruleKey = sasNumericOrderKey(ruleValue)
 
-    if (isValidNumeric && isNaN(numValue)) return true
+    if (!valueKey || !ruleKey) return false
 
-    return numValue <= Number(ruleValue.toString())
+    return compareSasNumericOrder(valueKey, ruleKey) <= 0
   },
-  NOTNULL: (value: any, ruleValue: string | number): boolean => {
-    return value !== undefined && value !== null && value.toString().length > 0
+  NOTNULL: (
+    value: any,
+    ruleValue: string | number,
+    isNumeric: boolean = false
+  ): boolean => {
+    if (value === undefined || value === null) return false
+
+    // A special missing (.A-.Z, ._) is NULL as far as a SAS NOT NULL (or
+    // primary key) constraint is concerned - an insert carrying one is
+    // rejected with an integrity constraint error - so the rule must reject
+    // it too, or the editor would accept a value the target table refuses.
+    // Numeric columns only: a lone letter is ordinary data on a character
+    // column.
+    if (isNumeric && isSpecialMissing(value)) return false
+
+    return value.toString().length > 0
   },
   // Pattern is used as authored, not auto-anchored — a rule author who
   // wants a full-value match must write ^...$ themselves.

client/src/app/shared/sidebar/sidebar.component.spec.ts

@@ -0,0 +1,34 @@
+import { SidebarComponent } from './sidebar.component'
+
+describe('SidebarComponent', () => {
+  // getSubPage is a pure read of the router's current URL, and the only other
+  // dependencies are the two SasService calls in the constructor - so the
+  // class can be constructed directly, without a TestBed.
+  const buildComponent = (routerUrl: string) =>
+    new SidebarComponent(
+      { url: routerUrl } as any,
+      {} as any,
+      {
+        getSasjsConfig: () => ({}),
+        getServerType: () => 'SASVIYA'
+      } as any
+    )
+
+  describe('getSubPage', () => {
+    it('returns the sub-page segment of a plain route', () => {
+      expect(buildComponent('/home/tables').getSubPage()).toEqual('tables')
+    })
+
+    it('excludes the query string - a VA report embed always adds one', () => {
+      expect(buildComponent('/home/tables?embed=va').getSubPage()).toEqual(
+        'tables'
+      )
+    })
+
+    it('excludes the query string on the viewer route', () => {
+      expect(buildComponent('/view/data?labels=true').getSubPage()).toEqual(
+        'data'
+      )
+    })
+  })
+})

client/src/app/shared/sidebar/sidebar.component.ts

@@ -67,7 +67,10 @@ export class SidebarComponent implements OnInit {
   }
 
   public getSubPage() {
-    let url = this._router.url.split('/')
+    // `Router.url` carries the query string, so take the path segment only -
+    // otherwise any route with a parameter renders it as part of the label
+    // (a VA report embed always adds one, e.g. `?embed=va`).
+    let url = this._router.url.split('?')[0].split('/')
 
     return url[2]
   }

client/src/app/shared/utils/parse-user-agent.spec.ts

@@ -0,0 +1,119 @@
+import { parseUserAgent } from './parse-user-agent'
+
+/**
+ * Table-driven: the point of the parser is the token ORDER, and the only way
+ * to pin an order is to feed it real user agent strings that carry several
+ * tokens at once. Every case below is a verbatim UA from the product named
+ * (the iOS ones are the ones that used to fall through to Safari).
+ */
+describe('parseUserAgent', () => {
+  const cases: Array<{
+    label: string
+    ua: string
+    browser: string
+    version: string
+    platform: string
+  }> = [
+    {
+      label: 'desktop Edge (Edg)',
+      ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.2592.87',
+      browser: 'Edge',
+      version: '126.0.2592.87',
+      platform: 'Windows'
+    },
+    {
+      label: 'desktop Edge (legacy EdgeHTML)',
+      ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763',
+      browser: 'Edge',
+      version: '18.17763',
+      platform: 'Windows'
+    },
+    {
+      label: 'desktop Opera (OPR)',
+      ua: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 OPR/111.0.0.0',
+      browser: 'Opera',
+      version: '111.0.0.0',
+      platform: 'Linux'
+    },
+    {
+      label: 'desktop Firefox',
+      ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0',
+      browser: 'Firefox',
+      version: '140.0',
+      platform: 'Linux'
+    },
+    {
+      label: 'desktop Chrome',
+      ua: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+      browser: 'Chrome',
+      version: '126.0.0.0',
+      platform: 'Linux'
+    },
+    {
+      label: 'desktop Safari',
+      ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15',
+      browser: 'Safari',
+      version: '17.5',
+      platform: 'macOS'
+    },
+    {
+      label: 'Android Chrome',
+      ua: 'Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36',
+      browser: 'Chrome',
+      version: '126.0.0.0',
+      platform: 'Android'
+    },
+    {
+      label: 'iOS Chrome (CriOS) - not Safari',
+      ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/126.0.6478.153 Mobile/15E148 Safari/604.1',
+      browser: 'Chrome',
+      version: '126.0.6478.153',
+      platform: 'iOS'
+    },
+    {
+      label: 'iOS Firefox (FxiOS) - not Safari',
+      ua: 'Mozilla/5.0 (iPad; CPU OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/127.0 Mobile/15E148 Safari/605.1.15',
+      browser: 'Firefox',
+      version: '127.0',
+      platform: 'iOS'
+    },
+    {
+      label: 'iOS Edge (EdgiOS) - not Safari',
+      ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) EdgiOS/126.0.2592.86 Version/17.0 Mobile/15E148 Safari/604.1',
+      browser: 'Edge',
+      version: '126.0.2592.86',
+      platform: 'iOS'
+    },
+    {
+      label: 'iOS Safari',
+      ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
+      browser: 'Safari',
+      version: '17.5',
+      platform: 'iOS'
+    },
+    {
+      label: 'a non-browser client',
+      ua: 'curl/8.5.0',
+      browser: 'Unknown',
+      version: '',
+      platform: 'Unknown'
+    },
+    {
+      label: 'an empty user agent',
+      ua: '',
+      browser: 'Unknown',
+      version: '',
+      platform: 'Unknown'
+    }
+  ]
+
+  cases.forEach((c) => {
+    it(`reports ${c.label} as ${c.browser} ${c.version || '(no version)'} on ${c.platform}`, () => {
+      expect(parseUserAgent(c.ua)).toEqual({
+        browser: c.browser,
+        browserVersion: c.version,
+        platform: c.platform
+      })
+    })
+  })
+})

client/src/app/shared/utils/parse-user-agent.ts

@@ -0,0 +1,77 @@
+/**
+ * Summarises `navigator.userAgent` into the three things worth logging about a
+ * browser: its family, its version, and the platform it is running on.
+ *
+ * The raw user agent is kept alongside these - it is what a support ticket
+ * actually needs - but a hook is far more likely to want to branch on a name
+ * than to write its own user agent parsing.
+ *
+ * Order matters: Edge and Opera both claim to be Chrome, and Chrome claims to
+ * be Safari, so the most specific token has to be tested first. The mobile
+ * tokens are the sharpest case - an iOS browser carries its own token
+ * (`CriOS`, `FxiOS`, `EdgiOS`) *and* `Safari/`, so without the mobile token
+ * being tested first every iPhone and iPad client was reported as Safari with
+ * an empty version.
+ */
+export interface UserAgentSummary {
+  browser: string
+  browserVersion: string
+  platform: string
+}
+
+/**
+ * Most specific token first. Each entry carries the version pattern for the
+ * same token, so the family and its version can never be read from different
+ * products.
+ */
+const BROWSER_TOKENS: Array<{ name: string; token: RegExp; version: RegExp }> =
+  [
+    {
+      name: 'Edge',
+      token: /(?:EdgiOS|Edg[A-Z]?|Edge)\//,
+      version: /(?:EdgiOS|Edg[A-Z]?|Edge)\/([\d.]+)/
+    },
+    {
+      name: 'Opera',
+      token: /(?:OPiOS|OPR)\//,
+      version: /(?:OPiOS|OPR)\/([\d.]+)/
+    },
+    {
+      name: 'Firefox',
+      token: /(?:FxiOS|Firefox)\//,
+      version: /(?:FxiOS|Firefox)\/([\d.]+)/
+    },
+    {
+      name: 'Chrome',
+      token: /(?:CriOS|Chrome)\//,
+      version: /(?:CriOS|Chrome)\/([\d.]+)/
+    },
+    // Safari is the fallback token: everything else that reaches here is
+    // WebKit wearing another product's name.
+    { name: 'Safari', token: /Safari\//, version: /Version\/([\d.]+)/ }
+  ]
+
+export function parseUserAgent(userAgent: string): UserAgentSummary {
+  const ua = userAgent || ''
+  const match = (re: RegExp) => (ua.match(re) || [])[1] || ''
+
+  let browser = 'Unknown'
+  let browserVersion = ''
+
+  for (const entry of BROWSER_TOKENS) {
+    if (entry.token.test(ua)) {
+      browser = entry.name
+      browserVersion = match(entry.version)
+      break
+    }
+  }
+
+  let platform = 'Unknown'
+  if (/Android/.test(ua)) platform = 'Android'
+  else if (/iPhone|iPad|iPod/.test(ua)) platform = 'iOS'
+  else if (/Windows/.test(ua)) platform = 'Windows'
+  else if (/Mac OS X/.test(ua)) platform = 'macOS'
+  else if (/Linux/.test(ua)) platform = 'Linux'
+
+  return { browser, browserVersion, platform }
+}

client/src/app/shared/viewboxes/viewboxes.component.ts

@@ -1404,10 +1404,17 @@ export class ViewboxesComponent implements OnInit, AfterViewInit, OnDestroy {
     this.filter = true
     this.cdf.detectChanges()
 
-    this.sasStoreService.setQueryVariables(
-      this.filterLibds,
-      viewboxTable.hotTable.cols
-    )
+    // Table columns when we have them (the normal, cheapest source); when the
+    // view was aborted by the row limit there is no loaded table to read them
+    // from, so the store falls back to public/getcols.
+    const cols =
+      viewboxTable && viewboxTable.hotTable ? viewboxTable.hotTable.cols : []
+
+    this.sasStoreService
+      .setQueryVariablesFromTable(this.filterLibds, cols)
+      .catch((err: any) =>
+        this.eventService.catchResponseError('public/getcols', err)
+      )
   }
 
   /**

client/src/app/system/system.component.html

@@ -286,6 +286,12 @@
             UPDATE
           </button>
         </div>
+        <div class="admin-action">
+          Export DC Library DDL
+          <button (click)="openDdlExport()" class="btn btn-info btn-sm">
+            EXPORT
+          </button>
+        </div>
       }
     </div>
   </div>
@@ -306,3 +312,40 @@
     </button>
   </div>
 </clr-modal>
+
+<clr-modal [(clrModalOpen)]="ddlModal">
+  <h3 class="modal-title">Export DC Library DDL</h3>
+  <div class="modal-body">
+    <clr-select-container>
+      <label>Flavour</label>
+      <select [(ngModel)]="ddlFlavour" clrSelect>
+        <option value="SAS">SAS</option>
+        <option value="PGSQL">PGSQL</option>
+        <option value="TSQL">TSQL</option>
+      </select>
+    </clr-select-container>
+    @if (ddlFlavour !== 'SAS') {
+      <clr-input-container>
+        <label>Schema (optional)</label>
+        <input
+          clrInput
+          type="text"
+          [(ngModel)]="ddlSchema"
+          placeholder="e.g. DC"
+        />
+      </clr-input-container>
+    }
+  </div>
+  <div class="modal-footer">
+    <button
+      type="button"
+      class="btn btn-sm btn-outline"
+      (click)="ddlModal = false"
+    >
+      Cancel
+    </button>
+    <button type="button" class="btn btn-sm btn-primary" (click)="exportDdl()">
+      Export
+    </button>
+  </div>
+</clr-modal>

client/src/app/system/system.component.ts

@@ -42,6 +42,9 @@ export class SystemComponent implements OnInit {
   Infinity = Infinity
   dcLib: string = globals.dcLib
   targetLibref: string = globals.dcLib
+  ddlModal: boolean = false
+  ddlFlavour: string = 'SAS'
+  ddlSchema: string = ''
 
   licenceState = this.licenceService.licenceState
   settings: AppSettings
@@ -92,6 +95,32 @@ export class SystemComponent implements OnInit {
     window.open(downUrl)
   }
 
+  openDdlExport() {
+    this.ddlFlavour = 'SAS'
+    this.ddlSchema = ''
+    this.ddlModal = true
+  }
+
+  exportDdl() {
+    let sasjsConfig = this.sasService.getSasjsConfig()
+    let storage = sasjsConfig.serverUrl
+    let metaData = sasjsConfig.appLoc
+    let path = this.sasService.getExecutionPath()
+    let downUrl =
+      storage +
+      path +
+      '/?_program=' +
+      metaData +
+      '/services/admin/exportdb&flavour=' +
+      encodeURIComponent(this.ddlFlavour)
+    let schema = this.ddlSchema.trim()
+    if (this.ddlFlavour !== 'SAS' && schema) {
+      downUrl += '&schema=' + encodeURIComponent(schema)
+    }
+    window.open(downUrl)
+    this.ddlModal = false
+  }
+
   refreshDataCatalog() {
     this.refreshingDataCatalog = true
 

client/src/app/viewer/viewer.component.html

@@ -499,10 +499,29 @@
             !['', ' '].includes(queryText) &&
             !abortActive
           ) {
-            <div class="clr-col-md-12 infoBar">
-              <span
+            <div
+              #infoBar
+              class="clr-col-md-12 infoBar"
+              [class.expanded]="filterExpanded"
+            >
+              <span class="infoBar-text"
                 >FILTER : <b>{{ queryText }}</b></span
               >
+              @if (filterOverflows) {
+                <button
+                  type="button"
+                  class="infoBar-toggle"
+                  [attr.aria-expanded]="filterExpanded"
+                  [attr.aria-label]="
+                    filterExpanded
+                      ? 'Collapse the filter clause'
+                      : 'Expand the filter clause'
+                  "
+                  (click)="toggleFilterPanel()"
+                >
+                  <clr-icon aria-hidden="true" shape="caret down"></clr-icon>
+                </button>
+              }
             </div>
           }
         </div>

client/src/app/viewer/viewer.component.ts

@@ -3,6 +3,8 @@ import {
   AfterContentInit,
   ChangeDetectorRef,
   AfterViewInit,
+  AfterViewChecked,
+  ElementRef,
   OnDestroy,
   ViewChildren,
   QueryList,
@@ -58,7 +60,7 @@ import { buildColInfoHtml } from '../shared/utils/col-info-html'
   standalone: false
 })
 export class ViewerComponent
-  implements AfterContentInit, AfterViewInit, OnDestroy
+  implements AfterContentInit, AfterViewInit, AfterViewChecked, OnDestroy
 {
   @ViewChildren('queryFilter')
   queryFilterCompList: QueryList<QueryComponent> = new QueryList()
@@ -96,6 +98,19 @@ export class ViewerComponent
   public libTab!: string
   public queryText: string = ''
   public webQueryText: string = ''
+
+  /**
+   * The applied-filter panel is collapsed to a single line by default. The
+   * chevron that expands it is only useful when the clause does not fit that
+   * line, which depends on the rendered width - so it is measured from the DOM
+   * rather than guessed from the length of the text.
+   */
+  public filterExpanded = false
+  public filterOverflows = false
+
+  @ViewChild('infoBar') infoBar?: ElementRef<HTMLElement>
+
+  private filterMeasuredKey = ''
   public submitLoading!: boolean
   public queryErr: boolean = false
   public queryErrMessage!: string
@@ -1445,6 +1460,48 @@ export class ViewerComponent
     }
   }
 
+  ngAfterViewChecked() {
+    this.scheduleFilterOverflowCheck()
+  }
+
+  /**
+   * Measures whether the applied-filter clause fits the collapsed single line,
+   * and shows or hides the chevron accordingly. The measurement is deferred
+   * because `filterOverflows` and `filterExpanded` are bound in this view -
+   * setting them inside the change-detection cycle would raise
+   * ExpressionChangedAfterItHasBeenChecked.
+   */
+  private scheduleFilterOverflowCheck() {
+    const el = this.infoBar?.nativeElement
+    if (!el) {
+      return
+    }
+
+    const text = el.querySelector('.infoBar-text') as HTMLElement | null
+    if (!text) {
+      return
+    }
+
+    // re-measure only when the clause, the available width, or the state changes
+    const key = `${this.queryText}|${text.clientWidth}|${this.filterExpanded}`
+    if (key === this.filterMeasuredKey) {
+      return
+    }
+    this.filterMeasuredKey = key
+
+    setTimeout(() => {
+      // while expanded the clause wraps, so there is nothing to measure
+      if (this.filterExpanded) {
+        return
+      }
+      this.filterOverflows = text.scrollWidth > text.clientWidth
+    })
+  }
+
+  public toggleFilterPanel() {
+    this.filterExpanded = !this.filterExpanded
+  }
+
   ngAfterViewInit() {
     // Fix ARIA accessibility issues after table initialization
     setTimeout(() => {

client/src/index.html

@@ -42,11 +42,13 @@
     * debug: Whether to enable debug on startup
 
     ## Viya only settings ##
-    * adminGroup: This group will have unrestricted access (Viya Only)
     * dcPath: The physical location in which to deploy the staging area (Viya only)
     * useComputeApi: Connection method (Viya only)
     * contextName: Compute Context (Viya only)
 
+    The admin group is not configured here - it is a DC setting (DC_ADMIN_GROUP),
+    shown on the System screen.
+
   -->
 
   <sasjs
@@ -58,9 +60,6 @@
     useComputeApi="null"
     runAsTask="true"
     contextName="SAS Job Execution compute context"
-    adminGroup="SASAdministrators"
-    dcPath="/tmp/dc"
-    hotLicenceKey="non-commercial-and-evaluation"
   >
   </sasjs>
 

client/src/styles.scss

@@ -120,30 +120,6 @@ app-editor {
 
   .infoBar {
     margin-top: 14px;
-    background: #495967;
-    color: white;
-    text-align: center;
-    padding: 3px;
-    font-size: 16px;
-
-    height: 30px;
-
-    text-overflow: ellipsis;
-    overflow: hidden;
-    white-space: nowrap;
-
-    span {
-      width: 80%;
-    }
-
-    &:hover {
-      height: unset;
-      white-space: normal;
-
-      span {
-        width: unset;
-      }
-    }
   }
 
   .pkHeader {
@@ -1228,11 +1204,6 @@ app-viewer {
 
   .infoBar {
     margin-top: 10px;
-    background: #495967;
-    color: white;
-    text-align: center;
-    padding: 3px;
-    font-size: 16px;
   }
 
   .filterSide {
@@ -1693,6 +1664,13 @@ app-requests-modal {
   padding: 20px 10px 5px 10px;
 }
 
+/* The SAS log pane is `white-space: pre`, so it never wraps.  The stock `xl`
+  width is a fixed 57.6rem, which forced left/right scrolling to read a log
+  line. */
+.requests-modal .modal-dialog {
+  width: 90vw;
+}
+
 .work-tables-dropdown button {
   color: var(--clr-nav-link-color, #8c8c8c) !important;
 }
@@ -2318,15 +2296,6 @@ app-alerts {
 }
 // ALERTS.COMPONENT END
 
-// MANUAL-DEPLOY.COMPONENT
-app-manual-deploy {
-  .clear-memory-button {
-    right: 10px;
-    top: 2px;
-  }
-}
-// MANUAL-DEPLOY.COMPONENT END
-
 // LINEAGE.COMPONENT
 app-lineage {
   .toggle-switch input[type='checkbox']:checked + label:before {
@@ -4284,6 +4253,30 @@ body[cds-theme='light'] {
     color: #c21d00;
   }
 
+  /* The SASjs tag dump drops onto its own line and reads as a name/value grid. */
+  &--grid {
+    flex-wrap: wrap;
+    align-items: baseline;
+  }
+
+  &__grid {
+    flex-basis: 100%;
+    display: grid;
+    grid-template-columns: auto 1fr;
+    gap: 1px 12px;
+    margin-top: 4px;
+    font-size: 12px;
+  }
+
+  &__grid-name {
+    color: #999;
+  }
+
+  &__grid-value {
+    color: #ccc;
+    word-break: break-word;
+  }
+
   &__icon--pending {
     color: #999;
   }
@@ -5448,3 +5441,58 @@ body[cds-theme='dark'] {
   cursor: pointer;
   text-decoration: underline;
 }
+
+/* The applied-filter panel. Collapsed to a single line by default so that a
+   long filter does not take over the header; the chevron - rendered only when
+   the clause does not fit that line - expands the panel to show it in full. */
+.infoBar {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 8px;
+  background: #495967;
+  color: white;
+  text-align: center;
+  padding: 3px 8px;
+  font-size: 16px;
+  /* the panel shows a query, so set it in the monospace face the app already
+     uses for code-like content (SAS logs, cell text) */
+  font-family: 'Lucida Console', Monaco, monospace;
+
+  .infoBar-text {
+    flex: 1;
+    min-width: 0;
+    overflow: hidden;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+  }
+
+  &.expanded {
+    align-items: flex-start;
+
+    .infoBar-text {
+      overflow: visible;
+      white-space: normal;
+      text-overflow: clip;
+      overflow-wrap: anywhere;
+    }
+  }
+
+  .infoBar-toggle {
+    flex: none;
+    background: transparent;
+    border: 0;
+    color: inherit;
+    cursor: pointer;
+    padding: 0 2px;
+    line-height: 1;
+
+    clr-icon {
+      transition: transform 0.15s ease-in;
+    }
+  }
+
+  &.expanded .infoBar-toggle clr-icon {
+    transform: rotate(180deg);
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "7.14.2",
+  "version": "7.18.0",
   "description": "Data Controller",
   "devDependencies": {
     "@nogoo9/gitleaks": "8.30.1-post.2",

sas/mocks/sas9/sasjs/services/editors/getdata.js

@@ -14,7 +14,6 @@ function makeRows(n) {
     const rows = []
     for (let i = 0; i < n; i++) {
         rows.push({
-            _____DELETE__THIS__RECORD_____: "No",
             PRIMARY_KEY_FIELD: i,
             SOME_CHAR: "dummy data row " + i,
             SOME_DROPDOWN: dropdowns[i % dropdowns.length],
@@ -241,7 +240,6 @@ const data = {
         { ODS_TABLE: "ENGINEHOST", NAME: "File Size (bytes)", VALUE: "1048576" }
     ],
     maxvarlengths: [
-        { NAME: "_____DELETE__THIS__RECORD_____", MAXLEN: 3 },
         { NAME: "PRIMARY_KEY_FIELD", MAXLEN: 4 },
         { NAME: "some_char", MAXLEN: 591 },
         { NAME: "some_dropdown", MAXLEN: 8 },
@@ -261,7 +259,6 @@ const data = {
     sasdata: makeRows(100),
     $sasdata: {
         vars: {
-            _____DELETE__THIS__RECORD_____: { format: "$3.", label: "_____DELETE__THIS__RECORD_____", length: "3", type: "char" },
             PRIMARY_KEY_FIELD: { format: "best.", label: "PRIMARY_KEY_FIELD", length: "8", type: "num" },
             SOME_CHAR: { format: "$32767.", label: "SOME_CHAR", length: "32767", type: "char" },
             SOME_DROPDOWN: { format: "$128.", label: "SOME_DROPDOWN", length: "128", type: "char" },
@@ -280,14 +277,14 @@ const data = {
     },
     sasparams: [
         {
-            COLHEADERS: "_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,SOME_HARDSELECT,SOME_NUM,SOME_DATE,SOME_DATETIME,SOME_TIME,SOME_SHORTNUM,SOME_BESTNUM,READONLY_COL,HIDDEN_COL,ROUND_COL,NUMFMT_COL",
+            COLHEADERS: "PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,SOME_HARDSELECT,SOME_NUM,SOME_DATE,SOME_DATETIME,SOME_TIME,SOME_SHORTNUM,SOME_BESTNUM,READONLY_COL,HIDDEN_COL,ROUND_COL,NUMFMT_COL",
             FILTER_TEXT: "",
             PKCNT: 1,
             PK: "PRIMARY_KEY_FIELD",
             DTVARS: " SOME_DATE",
             DTTMVARS: " SOME_DATETIME",
             TMVARS: " SOME_TIME",
-            COLTYPE: "{\"data\":\"_____DELETE__THIS__RECORD_____\",\"type\":\"dropdown\",\"source\":[\"No\",\"Yes\"]},{\"data\":\"PRIMARY_KEY_FIELD\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_CHAR\"},{\"data\":\"SOME_DROPDOWN\"},{\"data\":\"SOME_HARDSELECT\"},{\"data\":\"SOME_NUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_DATE\",\"type\":\"date\"},{\"data\":\"SOME_DATETIME\",\"type\":\"datetime\"},{\"data\":\"SOME_TIME\",\"type\":\"time\"},{\"data\":\"SOME_SHORTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_BESTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"READONLY_COL\"},{\"data\":\"HIDDEN_COL\"},{\"data\":\"ROUND_COL\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"NUMFMT_COL\",\"type\":\"numeric\",\"format\":\"0\"}",
+            COLTYPE: "{\"data\":\"PRIMARY_KEY_FIELD\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_CHAR\"},{\"data\":\"SOME_DROPDOWN\"},{\"data\":\"SOME_HARDSELECT\"},{\"data\":\"SOME_NUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_DATE\",\"type\":\"date\"},{\"data\":\"SOME_DATETIME\",\"type\":\"datetime\"},{\"data\":\"SOME_TIME\",\"type\":\"time\"},{\"data\":\"SOME_SHORTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_BESTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"READONLY_COL\"},{\"data\":\"HIDDEN_COL\"},{\"data\":\"ROUND_COL\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"NUMFMT_COL\",\"type\":\"numeric\",\"format\":\"0\"}",
             LOADTYPE: "UPDATE",
             RK_FLAG: 0,
             CLS_FLAG: 0

sas/mocks/sasjs/sasjsconfig.json

@@ -14,7 +14,7 @@
       "streamConfig": {
         "streamWeb": true,
         "streamWebFolder": "web9",
-        "webSourcePath": "`../../../../client/dist",
+        "webSourcePath": "../../client/dist",
         "assetPaths": [],
         "streamServiceName": "clickme"
       }

sas/mocks/sasjs/services/admin/exportdb.js

@@ -0,0 +1,160 @@
+const nodePath = require('path')
+
+let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
+const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
+const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
+
+eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
+
+/**
+ * Mock of services/admin/exportdb.sas - exports the DC library as DDL.
+ *
+ * The real service streams mp_ds2ddl output for every table in &DC_LIBREF, in
+ * the requested flavour, and mp_lib2inserts appends the rows for the SAS and
+ * PGSQL flavours.  This mock emits the DDL only - a basic, readable rendering
+ * per flavour, enough to exercise the System screen's Export DC Library DDL
+ * action end to end (the download headers, the filename, the flavour and the
+ * optional schema).  It deliberately does not append INSERTs: the real service
+ * does, but for a mock that would mean serialising the whole library's rows
+ * (MPE_X_WIDE alone is 200,000 cells) for no extra test value.
+ *
+ * Params arrive from the URL the client opens:
+ *   <serverUrl>/SASjsApi/stp/execute/?_program=<appLoc>/services/admin/exportdb
+ *     &flavour=SAS|PGSQL|TSQL[&schema=<name>]
+ * The schema box only applies to the DB flavours, and the client omits the
+ * param entirely when it is blank, so both must be read defensively - an
+ * unset URL param is an undeclared const here, not an empty string.
+ */
+
+const ddlFlavour = (typeof flavour === 'undefined' || !flavour)
+    ? 'SAS'
+    : String(flavour).toUpperCase()
+
+const dcConf = loadDcSettings(driveRoot, appLoc)
+const libref = dcConf.dcLibref
+const dataDir = dcConf.dataDir
+
+// if no schema param, the DC libref is used (as in exportdb.sas)
+const ddlSchema = (typeof schema === 'undefined' || !schema)
+    ? libref
+    : String(schema)
+
+// ─── Column types, per flavour ───────────────────────────────────────────────
+
+function colType(col, flavour) {
+    const isChar = String(col.type).toUpperCase() === 'C'
+    const fmt = String(col.format || '').toUpperCase()
+    const len = Number(col.length) || (isChar ? 1 : 8)
+
+    if (flavour === 'SAS') {
+        return isChar ? `char(${len})` : 'num'
+    }
+
+    // PGSQL and TSQL share the mapping; only the quoting differs.  The one type
+    // that must not be shared is the datetime: T-SQL's TIMESTAMP is a deprecated
+    // synonym for rowversion - a binary row version, not a datetime - so it maps
+    // to DATETIME2 there.  DOUBLE PRECISION is a T-SQL synonym for float, so it
+    // stays valid on both.
+    if (isChar) return `VARCHAR(${len})`
+    if (fmt.startsWith('DATETIME'))
+        return flavour === 'TSQL' ? 'DATETIME2' : 'TIMESTAMP'
+    if (fmt.startsWith('DATE')) return 'DATE'
+    if (fmt.startsWith('TIME')) return 'TIME'
+    return 'DOUBLE PRECISION'
+}
+
+function colRef(name, flavour) {
+    if (flavour === 'PGSQL') return `"${name}"`
+    if (flavour === 'TSQL') return `[${name}]`
+    return name
+}
+
+function tableRef(table, flavour) {
+    if (flavour === 'TSQL') return `[${ddlSchema}].[${table}]`
+    return `${ddlSchema}.${table}`
+}
+
+// ─── Buskeys (for the unique index), from MPE_TABLES ─────────────────────────
+
+const buskeys = {}
+try {
+    const reg = makeTableLoader(dataDir)('MPE_TABLES')
+    for (const row of ((reg && reg.rows) || [])) {
+        const dsn = String(colVal(row, 'DSN') || '').trim().toUpperCase()
+        const buskey = String(colVal(row, 'BUSKEY') || '').trim()
+        if (dsn && buskey) buskeys[dsn] = buskey
+    }
+} catch(err) { /* no MPE_TABLES - the DDL is still emitted, without indexes */ }
+
+// ─── Build the DDL ───────────────────────────────────────────────────────────
+
+const lines = []
+const stamp = new Date().toISOString().replace('T', ':').substring(0, 19)
+lines.push(`/* DDL generated by the Data Controller mock on ${stamp} */`)
+
+if (ddlFlavour === 'PGSQL') {
+    lines.push(`CREATE SCHEMA ${ddlSchema};`)
+}
+
+let tables = []
+try {
+    tables = fs.readdirSync(dataDir)
+        .filter(f => f.endsWith('.json'))
+        .map(f => {
+            try {
+                return JSON.parse(fs.readFileSync(nodePath.resolve(dataDir, f), {encoding:'utf8'}).toString())
+            } catch(e) { return null }
+        })
+        .filter(t => t && t.name && Array.isArray(t.columns))
+        .sort((a, b) => String(a.name).localeCompare(String(b.name)))
+} catch(err) { /* nothing to export */ }
+
+for (const table of tables) {
+    const tableName = String(table.name).toUpperCase()
+    const cols = table.columns.filter(c => String(c.name || '').trim() !== '')
+    if (cols.length === 0) continue
+
+    if (ddlFlavour === 'SAS') {
+        lines.push(`/* SAS flavour DDL for ${libref}.${tableName} */`)
+        lines.push(`create table ${tableRef(tableName, ddlFlavour)}(`)
+    } else if (ddlFlavour === 'PGSQL') {
+        lines.push(`/* Postgres Flavour DDL for ${libref}.${tableName} */`)
+        lines.push(`CREATE TABLE ${tableRef(tableName, ddlFlavour)} (`)
+    } else {
+        lines.push(`/* TSQL Flavour DDL for ${libref}.${tableName} */`)
+        lines.push(`create table ${tableRef(tableName, ddlFlavour)}(`)
+    }
+
+    cols.forEach((col, idx) => {
+        const name = colRef(String(col.name).toUpperCase(), ddlFlavour)
+        const prefix = idx === 0 ? '   ' : '  ,'
+        lines.push(`${prefix}${name} ${colType(col, ddlFlavour)}`)
+    })
+    lines.push(');')
+
+    // a unique index on the buskey, where MPE_TABLES declares one - SAS has no
+    // equivalent in mp_ds2ddl output, so it is skipped for that flavour
+    const buskey = buskeys[tableName]
+    if (buskey && ddlFlavour !== 'SAS') {
+        const keys = buskey.split(/[\s,]+/).filter(Boolean)
+            .map(k => colRef(k.toUpperCase(), ddlFlavour))
+        lines.push(
+            `CREATE UNIQUE INDEX ${colRef(tableName + '_pk', ddlFlavour)} ON ` +
+            `${tableRef(tableName, ddlFlavour)}(\n  ${keys.join('\n ,')}\n);`
+        )
+    }
+    lines.push('')
+}
+
+// ─── Stream it back as a download, as mp_streamfile does ─────────────────────
+
+const filename = `${libref}.ddl`
+fs.writeFile(
+    _SASJS_WEBOUT_HEADERS,
+    `Content-type: text/plain \n Content-Disposition: attachment;filename="${filename}"`,
+    function (err) {
+        if (err) throw err
+    }
+)
+
+_webout = lines.join('\n') + '\n'

sas/mocks/sasjs/services/auditors/postdata.js

@@ -152,10 +152,19 @@ for (const col of diffCols) colDdtypes[col.name] = getDdType(col)
 // Staged values arrive from CSV as strings ("42"), base values are native JSON
 // (42). Compare numerics by value and strings case-sensitively.
 
+// A SAS special missing (._ , .A-.Z) reaches us as a string. Special missings
+// are numeric-only, and real DC writes the DIFF with `missing=STRING` - whose
+// format maps ._ and .a-.z to a string but leaves a bare `.` as null (see the
+// `bart` format in mp_jsonout.sas). Coercing a special missing with Number()
+// would yield NaN and blank the cell out of the DIFF.
+const SPECIAL_MISSING_RE = /^\.(_|[a-z])$/i
+
 function normVal(value, colName) {
   const col = diffCols.find((c) => c.name === colName)
   if (col && col.type === 'N') {
     if (value === null || value === undefined || value === '') return null
+    const str = String(value).trim()
+    if (SPECIAL_MISSING_RE.test(str)) return str
     const num = Number(value)
     return isNaN(num) ? null : num
   }
@@ -543,7 +552,7 @@ if (action === 'SHOW_DIFFS') {
   const stageFolder = getStageFolder(loadRef)
 
   // check: has this user already approved? (mirrors prev_upload_check in postdata.sas)
-  const reviewData = loadTableData('MPE_REVIEW') || { rows: [] }
+  const reviewData = mpeLoadTableData('MPE_REVIEW') || { rows: [] }
   const alreadyApproved = reviewData.rows.some(
     (r) =>
       r.TABLE_ID === loadRef &&

sas/mocks/sasjs/services/dcMockUtils.js

@@ -5,6 +5,7 @@
  * Provides:
  *   - fetchTable / fetchRaw / parseCsv : emulate SAS %webout(FETCH)
  *   - webOutOpen / webOutObj / webOutClose : emulate SAS %webout(OPEN/OBJ/CLOSE)
+ *   - mpeinit                         : emulate the %mpeinit debug dump
  *   - formatSasValue                  : apply a SAS format to a numeric value
  *   - parseFormattedToSas             : convert an ISO string back to a SAS numeric
  *   - getDdType                       : derive DATE/DATETIME/TIME/N/C from a column format
@@ -147,14 +148,63 @@ function _parseCsvLine(line) {
 // ─── SAS format <-> ISO conversions ──────────────────────────────────────────
 
 /**
- * Derives a DDTYPE (DATE, DATETIME, TIME, N, C) from a SAS column format string.
+ * Column metadata, mirroring %mp_getcols.sas.
+ *
+ * The SAS macro runs proc contents and derives, per column: NAME (upcased),
+ * TYPE (C/N), LENGTH, LABEL (defaulting to the name), VARNUM, FMTNAME, FORMAT
+ * and DDTYPE - CHARACTER, NUMERIC, DATETIME, DATE or TIME.  The date list is
+ * the macro's (YYMMDD, MMDDYY, DDMMYY, MONYY, E8601DA, B8601DA, E8601DT,
+ * NLDATM, NLDATE), so a column types the same way here as it does in the
+ * service that shares this code.
+ *
+ * @param {Array} schemaCols the table's columns as the mock data stores them
+ *   ({name, type, length, format, label}), the equivalent of the proc contents
+ *   output the macro reads
+ * @returns {Array} one entry per column, in the macro's shape and order
+ */
+function mpGetcols(schemaCols) {
+    return (schemaCols || []).map((col, index) => {
+        const name = String(col.name || '').toUpperCase()
+        const isChar = String(col.type || '').toUpperCase() === 'C'
+        const fmtname = String(col.format || '')
+            .replace(/[\d.]+$/, '')
+            .toUpperCase()
+
+        let ddtype
+        if (isChar) ddtype = 'CHARACTER'
+        else if (fmtname.startsWith('DATETIME') || fmtname.startsWith('E8601DT')
+            || fmtname.startsWith('NLDATM')) ddtype = 'DATETIME'
+        else if (fmtname.startsWith('DATE') || fmtname.startsWith('DDMMYY')
+            || fmtname.startsWith('MMDDYY') || fmtname.startsWith('YYMMDD')
+            || fmtname.startsWith('E8601DA') || fmtname.startsWith('B8601DA')
+            || fmtname.startsWith('MONYY') || fmtname.startsWith('NLDATE')) ddtype = 'DATE'
+        else if (fmtname.startsWith('TIME')) ddtype = 'TIME'
+        else ddtype = 'NUMERIC'
+
+        return {
+            NAME: name,
+            TYPE: isChar ? 'C' : 'N',
+            LENGTH: Number(col.length) || 0,
+            LABEL: col.label || col.name,
+            VARNUM: index + 1,
+            FORMAT: String(col.format || ''),
+            FMTNAME: fmtname,
+            DDTYPE: ddtype
+        }
+    })
+}
+
+/**
+ * Derives a DDTYPE (DATE, DATETIME, TIME, N, C) from a SAS column format
+ * string, in the shape the client expects.  Delegates to mpGetcols so the
+ * inference is the macro's, rather than a second format list that can drift
+ * away from the service's.
  */
 function getDdType(col) {
-    const fmt = (col.format || '').toLowerCase()
-    if (fmt.startsWith('datetime') || fmt.startsWith('e8601dt')) return 'DATETIME'
-    if (fmt.startsWith('date') || fmt.startsWith('yymmdd') || fmt.startsWith('ddmmyy') || fmt.startsWith('mmddyy')) return 'DATE'
-    if (fmt.startsWith('time')) return 'TIME'
-    return col.type === 'N' ? 'N' : 'C'
+    const ddtype = mpGetcols([col])[0].DDTYPE
+    if (ddtype === 'CHARACTER') return 'C'
+    if (ddtype === 'NUMERIC') return 'N'
+    return ddtype
 }
 
 /**
@@ -336,6 +386,44 @@ function loadTableSchema(dataDir, tableName) {
     return (data && data.columns) ? data.columns : []
 }
 
+/**
+ * Resolves a hook program name to the source of its mock implementation.
+ *
+ * MPE_TABLES.pre_edit_hook / post_edit_hook (and the SOFTSELECT_HOOK /
+ * HARDSELECT_HOOK rule_value in MPE_VALIDATIONS) hold a hook program: either a
+ * physical .sas path, or a path relative to the appLoc (e.g.
+ * 'services/hooks/mytable_postedit').  The real backend %includes the program
+ * into the running service, so the hook shares the service's macro variables
+ * and work datasets and can modify them.
+ *
+ * The mock mirrors that: this returns the source of the hook's .js counterpart,
+ * which the calling service eval()s in its OWN scope so the hook can read and
+ * modify the service's variables.  Returns null when the hook has no mock
+ * implementation, so callers can fall back to an inline emulation or ignore it.
+ *
+ * Requires nodePath, driveRoot and appLoc in the caller's scope.
+ */
+function mockHookSource(hookValue) {
+    if (!hookValue) return null
+    let hook = String(hookValue).trim()
+    if (!hook) return null
+    // Reduce to the services-relative form, dropping any drive path or appLoc
+    // prefix the config may carry.
+    const idx = hook.indexOf('services/')
+    if (idx > -1) hook = hook.slice(idx)
+    // A physical hook is a .sas program on the server; its mock is a .js file
+    // sitting in the same place on the Drive.
+    hook = hook.replace(/\.sas$/i, '')
+    const hookFile = nodePath.resolve(driveRoot, 'files', appLoc, hook + '.js')
+    // The hook value is hand-edited mock config, and a value carrying '..'
+    // segments would otherwise resolve outside the Drive and be eval()'d
+    // in-service. Keep it inside the Drive's files tree.
+    const filesRoot = nodePath.resolve(driveRoot, 'files')
+    if (!hookFile.startsWith(filesRoot + nodePath.sep)) return null
+    if (!fs.existsSync(hookFile)) return null
+    return fs.readFileSync(hookFile, 'utf8')
+}
+
 /**
  * Synthesises NOTNULL dqrules from the table schema, mirroring the
  * dictionary.columns union in getdata.sas:
@@ -492,6 +580,7 @@ function makeTableId() {
  *   ('A','B')   -> ["A","B"]        (IN list)
  *   (1,2,3)     -> [1,2,3]          (IN list, numeric)
  *   1 AND 5     -> [1, 5]           (BETWEEN)
+ *   '20DEC2053'd-> 34322            (date / datetime / time literal)
  */
 function parseFilterRawValue(raw, operator) {
     if (raw === undefined || raw === null || raw === '') return null
@@ -510,8 +599,40 @@ function parseFilterRawValue(raw, operator) {
     return parseFilterScalar(s)
 }
 
+/**
+ * SAS date / datetime / time literal -> the SAS internal value, or null.
+ *   34322 <- '20DEC2053'd
+ *   42    <- '01JAN1960:00:00:42'dt
+ *   42    <- '00:00:42't
+ * The client builds these in canonical ddMMMyyyy form (HelperService
+ * .sasTemporalLiteral), so only that shape needs parsing here.
+ */
+const TEMPORAL_LITERAL_RE = /^'(\d{2})([A-Z]{3})(\d{4})(?::(\d{2}):(\d{2}):(\d{2})(\.\d+)?)?'(dt|d|t)$/i
+/* A time literal carries no date part, so it is matched separately */
+const TIME_LITERAL_RE = /^'(\d{2}):(\d{2}):(\d{2})(\.\d+)?'t$/i
+
+function parseTemporalLiteral(s) {
+    const t = TIME_LITERAL_RE.exec(s)
+    if (t) {
+        return Number(t[1]) * 3600 + Number(t[2]) * 60 + Number(t[3]) +
+            (t[4] ? Number(t[4]) : 0)
+    }
+    const m = TEMPORAL_LITERAL_RE.exec(s)
+    if (!m) return null
+    const month = MONTHS_ABBR.indexOf(m[2].toUpperCase())
+    if (month < 0) return null
+    const days = (Date.UTC(Number(m[3]), month, Number(m[1])) - SAS_EPOCH_MS) / 86400000
+    const suffix = m[8].toLowerCase()
+    if (suffix === 'd') return days
+    const secs = Number(m[4]) * 3600 + Number(m[5]) * 60 + Number(m[6]) +
+        (m[7] ? Number(m[7]) : 0)
+    return suffix === 'dt' ? days * 86400 + secs : secs
+}
+
 function parseFilterScalar(s) {
     s = s.trim()
+    const temporal = parseTemporalLiteral(s)
+    if (temporal !== null) return temporal
     if (s.startsWith("'") && s.endsWith("'")) {
         return s.slice(1, -1).replace(/''/g, "'")
     }
@@ -1063,6 +1184,61 @@ function loadDcSettings(driveRoot, appLoc) {
     return result
 }
 
+/**
+ * Reads the DC-scope options out of MPE_CONFIG, mirroring the loop in the real
+ * backend's generated settings.sas: only rows that are active (var_active=1)
+ * and current (now < tx_to) count, and the latest tx_from wins.  Returns a map
+ * of var_name -> trimmed var_value.
+ *
+ * An option that is absent, or switched off, is simply missing from the map so
+ * the caller's own default applies - which is what the real service's
+ * %symexist fallback does (a switched-off option means "use the code default",
+ * not "no limit").
+ *
+ * Requires nodePath, fs and the caller's driveRoot/appLoc/dataDir.
+ */
+function loadDcConfig(driveRoot, appLoc, dataDir) {
+    const out = {}
+    try {
+        const configFile = nodePath.resolve(
+            dataDir || nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
+            'mpe_config.json'
+        )
+        const config = JSON.parse(fs.readFileSync(configFile, {encoding:'utf8'}).toString())
+        const now = Math.floor(Date.now() / 1000)
+        const latest = {}
+        for (const row of (config.rows || [])) {
+            // colVal, not row.var_name: a row that has been through the app's
+            // own edit and approve flow comes back from the adapter with
+            // upper-case keys, and a direct property read would skip it - which
+            // silently drops the option back to its default.
+            if (colVal(row, 'VAR_SCOPE') !== 'DC') continue
+            if (Number(colVal(row, 'VAR_ACTIVE')) !== 1) continue
+            if (Number(colVal(row, 'TX_TO')) <= now) continue
+            const name = String(colVal(row, 'VAR_NAME') || '').trim()
+            if (!name) continue
+            const from = Number(colVal(row, 'TX_FROM')) || 0
+            if (latest[name] === undefined || from > latest[name]) {
+                const val = colVal(row, 'VAR_VALUE')
+                latest[name] = from
+                out[name] = String(val == null ? '' : val).trim()
+            }
+        }
+    } catch(err) { /* no MPE_CONFIG - every option falls back to its default */ }
+    return out
+}
+
+/**
+ * A numeric DC option out of a loadDcConfig() map, or the service's own default
+ * when the option is absent, blank, or not a number.
+ */
+function dcConfigNumber(cfg, name, fallback) {
+    const raw = cfg ? cfg[name] : undefined
+    if (raw === undefined || raw === '') return fallback
+    const num = Number(raw)
+    return isNaN(num) ? fallback : num
+}
+
 // ─── %webout(OPEN/OBJ/CLOSE) emulation ───────────────────────────────────────
 
 /**
@@ -1176,3 +1352,73 @@ function webOutSend(tables) {
     }
     webOutClose()
 }
+
+// ─── %mpeinit() emulation ────────────────────────────────────────────────────
+
+/**
+ * The `_debug` values that mean "debug on", mirroring the test in mpeinit.sas.
+ *
+ * 131 is what the adapter sends - SASjsApiClient.executeJob() puts it on every
+ * server-mode request, and on the multipart form when the payload goes out as
+ * FormData.  128 is what arrives on the Viya WEB JES path when runAsTask is
+ * enabled, and 2477 / 'fields,log,trace' are the SASjs debug values.
+ */
+const MPEINIT_DEBUG_VALUES = ['2477', 'fields,log,trace', '131', '128']
+
+/* Mirrors the &mpeinit=1 guard in the SAS macro: dump once per execution. */
+let _mpeinitDone = false
+
+/**
+ * Dumps the browser_url_vars and browser_info input tables, mirroring the
+ * debug block of %mpeinit (sas/sasjs/macros/mpeinit.sas).
+ *
+ * The SAS macro reads work.browser_url_vars (name/value pairs taken from the
+ * query string) and work.browser_info (the browser fingerprint).  Both arrive
+ * here as input tables - see fetchRaw() - and are dumped with console.log,
+ * which SASjs Server returns in the request's log (after its
+ * SASJS_LOGS_SEPARATOR marker), so the dump shows up in Data Controller's
+ * SAS Log tab exactly as the SAS version does.
+ *
+ * Called at the bottom of this file, so the services that eval() it dump the
+ * tables at startup - the same place %mpeinit() runs in the SAS services.
+ * That covers every diagnostics-carrying service (startup, editors, auditors);
+ * the mocks that do not load this file (lineage, metanav, usernav, a few
+ * public ones) have no equivalent startup step in the SAS app either.
+ *
+ * Difference from the SAS version: the header line is printed once per table,
+ * where SAS repeats it for every row (its PUTLOG sits inside the SET loop).
+ */
+function mpeinit() {
+    if (_mpeinitDone) return
+
+    const debugValue = typeof _debug === 'undefined' ? '' : _debug
+    if (MPEINIT_DEBUG_VALUES.indexOf(String(debugValue)) < 0) return
+
+    _mpeinitDone = true
+
+    const urlVars = fetchRaw('browser_url_vars')
+    if (urlVars === null) {
+        console.log('NOTE: mpeinit: no work.browser_url_vars on this request')
+    } else {
+        console.log('NOTE: mpeinit: work.browser_url_vars:')
+        parseCsv(urlVars).forEach(row => {
+            console.log(
+                'name=' + colVal(row, 'name') + ' value=' + colVal(row, 'value')
+            )
+        })
+    }
+
+    const browserInfo = fetchRaw('browser_info')
+    if (browserInfo === null) {
+        console.log('NOTE: mpeinit: no work.browser_info on this request')
+    } else {
+        console.log('NOTE: mpeinit: work.browser_info:')
+        parseCsv(browserInfo).forEach(row => {
+            console.log(Object.keys(row).map(col => col + '=' + row[col]).join(' '))
+        })
+    }
+}
+
+/* Every service that eval()s this file runs %mpeinit() at startup, the same
+   place the SAS services call it. */
+mpeinit()

sas/mocks/sasjs/services/editors/getdata.js

@@ -8,6 +8,28 @@ const dcLibref = 'DC_JSLIB'
 // Load shared DC mock utilities
 eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
 
+// Mirrors SAS cats() for the values a dropdown source can hold: a special
+// missing stored in its period form (".a") becomes the bare uppercase letter
+// ("A") that cats() returns, so the mock's dropdown list matches the real one.
+function cats(value) {
+    if (typeof value === 'string') {
+        const m = /^\.(_|[a-z])$/i.exec(value.trim())
+        if (m) return m[1].toUpperCase()
+        return value.trim()
+    }
+    return String(value)
+}
+
+/**
+ * SAS sort position of a special missing, or null when the value is ordinary.
+ * `._` sorts first, then `.a` to `.z` - all of them below every number.
+ */
+function missingRank(value) {
+    const m = /^\.(_|[a-z])$/i.exec(String(value).trim())
+    if (!m) return null
+    return m[1] === '_' ? 0 : m[1].toUpperCase().charCodeAt(0) - 64
+}
+
 // ─── Parse input ──────────────────────────────────────────────────────────────
 
 const _sctRow = fetchTable('SASControlTable')[0] || {}
@@ -130,7 +152,7 @@ if (tableData && tableData.rows && tableData.columns) {
         const colName = parts[parts.length - 1]
         const lcName = colName.toLowerCase()
         const seen = new Set()
-        let order = 1
+        const values = []
         for (const row of tableData.rows) {
             let val = row[colName]
             if (val === undefined) val = row[lcName]
@@ -139,19 +161,39 @@ if (tableData && tableData.rows && tableData.columns) {
                 if (key) val = row[key]
             }
             if (val !== undefined && val !== null) {
-                const strVal = String(val)
+                const strVal = cats(val)
                 if (!seen.has(strVal)) {
                     seen.add(strVal)
-                    dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: strVal, SELECTBOX_ORDER: order++ })
+                    values.push({ raw: val, str: strVal })
                 }
             }
         }
+        // getdata.sas orders the source by the column itself, and a special
+        // missing sorts below every number - so the list reads `._`, `.a`-`.z`,
+        // then the numbers ascending.
+        values.sort((a, b) => {
+            const ra = missingRank(a.raw)
+            const rb = missingRank(b.raw)
+            if (ra !== null && rb !== null) return ra - rb
+            if (ra !== null) return -1
+            if (rb !== null) return 1
+            const na = Number(a.str)
+            const nb = Number(b.str)
+            if (!isNaN(na) && !isNaN(nb)) return na - nb
+            return a.str < b.str ? -1 : a.str > b.str ? 1 : 0
+        })
+        values.forEach((v, i) => {
+            dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: v.str, SELECTBOX_ORDER: i + 1 })
+        })
     }
 }
 
 // ─── Build response ──────────────────────────────────────────────────────────
 
 let response
+// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
+// getdata.sas, and is emitted as sasjsAbort below.
+let abortMsg = ''
 
 if (tableData && tableData.columns && tableData.rows) {
     // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
@@ -165,8 +207,8 @@ if (tableData && tableData.columns && tableData.rows) {
         if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
     }
 
-    const visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
-    const visibleRows = tableData.rows
+    let visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
+    let visibleRows = tableData.rows
         .filter(r => filterPredicate(r))
         .map(r => {
         const copy = {}
@@ -176,6 +218,56 @@ if (tableData && tableData.columns && tableData.rows) {
         return copy
     })
 
+    // ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
+    // Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
+    // been applied and the rows sorted, with the data in work.OUT, and may
+    // replace it.  The mock eval()s the hook's .js counterpart in this scope, so
+    // the hook may reassign visibleRows / visibleColumns - e.g. to show the live
+    // rows of another table behind an empty mirror.
+    if (tableReg && tableReg.pre_edit_hook) {
+        const hookSrc = mockHookSource(tableReg.pre_edit_hook)
+        if (hookSrc) {
+            try {
+                eval(hookSrc)
+            } catch (err) {
+                console.log('Error running pre_edit_hook ' + tableReg.pre_edit_hook + ': ' + err.message)
+            }
+        }
+    }
+
+    // ─── Row and cell limits ─────────────────────────────────────────────────
+    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
+    // cells (rows x columns), whichever is reached first.  Both are read from
+    // MPE_CONFIG, the way the real service picks them up from the settings
+    // program it generates - so editing the option in DC changes what the mock
+    // allows, and an option that is absent (or switched off, var_active=0)
+    // falls back to the shipped default.  The check runs after the
+    // PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
+    // replace the data with a larger table.  The cell count includes the
+    // housekeeping delete column, which is what getdata.sas counts.
+    // MPE_CONFIG lives in the control library, not in the table's own library,
+    // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
+    // uses.  Passing the requested table's dataDir made the read fail for every
+    // non-control libref, and the catch then fell back to the shipped defaults.
+    const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
+    const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
+    const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
+    const editColCount = visibleColumns.length + 1
+    const cellCount = visibleRows.length * editColCount
+    if (visibleRows.length > maxObsWebEdit) {
+        abortMsg =
+            'Table is too big (' +
+            visibleRows.length +
+            ' rows) - please filter and try again!'
+    } else if (cellCount > maxCellsWebEdit) {
+        abortMsg =
+            'Selection is too wide (' +
+            visibleRows.length +
+            ' rows x ' +
+            editColCount +
+            ' cols) - please filter and try again!'
+    }
+
     // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
     const colLookup = {}
     for (const col of visibleColumns) {
@@ -188,16 +280,16 @@ if (tableData && tableData.columns && tableData.rows) {
             const targetKey = info ? info.name : key
             normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
         }
-        return { _____DELETE__THIS__RECORD_____: 'No', ...normalised }
+        return { ...normalised }
     })
 
     // $sasdata vars: temporal columns are char ($200.), others keep their SAS type
-    const vars = { _____DELETE__THIS__RECORD_____: { format: '$3.', label: '_____DELETE__THIS__RECORD_____', length: '3', type: 'char' } }
+    const vars = {}
     for (const col of visibleColumns) {
         vars[col.name] = sasVarsEntry(col)
     }
 
-    const colHeaders = ['_____DELETE__THIS__RECORD_____', ...visibleColumns.map(c => c.name)]
+    const colHeaders = visibleColumns.map(c => c.name)
 
     // Determine DTVARS, DTTMVARS, TMVARS from column formats
     let dtVars = [], dttmVars = [], tmVars = []
@@ -208,22 +300,25 @@ if (tableData && tableData.columns && tableData.rows) {
         else if (ddtype === 'TIME') tmVars.push(col.name)
     }
 
-    // Build cols array
-    const cols = visibleColumns.map(col => {
-        const ddtype = getDdType(col)
-        const fmtname = col.format
-            ? col.format.replace(/[\d.]+$/, '').replace('datetime', 'DATETIME').replace('date', 'DATE').replace('time', 'TIME').replace('best', 'BEST').replace('E8601DT', 'DATETIME')
-            : ' '
+    // Build cols array.  Every column of the table, as getdata.sas emits them:
+    // the temporal columns are dropped from the data (the rows below) but not
+    // from the cols payload, so a filter can still be built on them.  mpGetcols
+    // is the JS counterpart of %mp_getcols.sas - the macro getdata.sas itself
+    // calls to type these columns - so the editor sees the same DDTYPE the
+    // service would give it.
+    const cols = mpGetcols(tableData.columns).map((c, i) => {
+        const raw = tableData.columns[i]
+        const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
         let coltype
-        if (ddtype === 'DATE') coltype = `{"data":"${col.name}","type":"date"}`
-        else if (ddtype === 'DATETIME') coltype = `{"data":"${col.name}","type":"datetime"}`
-        else if (ddtype === 'TIME') coltype = `{"data":"${col.name}","type":"time"}`
-        else if (ddtype === 'N') coltype = `{"data":"${col.name}","type":"numeric","format":"0"}`
-        else coltype = `{"data":"${col.name}"}`
+        if (ddtype === 'DATE') coltype = `{"data":"${raw.name}","type":"date"}`
+        else if (ddtype === 'DATETIME') coltype = `{"data":"${raw.name}","type":"datetime"}`
+        else if (ddtype === 'TIME') coltype = `{"data":"${raw.name}","type":"time"}`
+        else if (ddtype === 'N') coltype = `{"data":"${raw.name}","type":"numeric","format":"0"}`
+        else coltype = `{"data":"${raw.name}"}`
         return {
-            NAME: col.name,
-            LABEL: col.label || col.name,
-            FMTNAME: fmtname || ' ',
+            NAME: c.NAME,
+            LABEL: c.LABEL,
+            FMTNAME: c.FMTNAME || ' ',
             DDTYPE: ddtype,
             CLS_RULE: 'READ',
             MEMLABEL: ' ',
@@ -235,7 +330,6 @@ if (tableData && tableData.columns && tableData.rows) {
 
     // Build maxvarlengths
     const maxvarlengths = [
-        { NAME: '_____DELETE__THIS__RECORD_____', MAXLEN: 3 },
         ...visibleColumns.map(col => {
             let maxLen = 0
             const lcName = col.name.toLowerCase()
@@ -298,15 +392,20 @@ if (tableData && tableData.columns && tableData.rows) {
 }
 
 webOutOpen()
-webOutObj(response.approvers, 'approvers')
-webOutObj(response.cols, 'cols')
-webOutObj(response.dqdata, 'dqdata')
-webOutObj(response.dqrules, 'dqrules')
-webOutObj(response.dsmeta, 'dsmeta')
-webOutObj(response.maxvarlengths, 'maxvarlengths')
-webOutObj(response.query, 'query')
-webOutObj(response.sasdata, 'sasdata', response.$sasdata)
-webOutObj(response.sasparams, 'sasparams')
-webOutObj(response.versions, 'versions')
-webOutObj(response.xl_rules, 'xl_rules')
+if (abortMsg) {
+    webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
+    webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
+} else {
+    webOutObj(response.approvers, 'approvers')
+    webOutObj(response.cols, 'cols')
+    webOutObj(response.dqdata, 'dqdata')
+    webOutObj(response.dqrules, 'dqrules')
+    webOutObj(response.dsmeta, 'dsmeta')
+    webOutObj(response.maxvarlengths, 'maxvarlengths')
+    webOutObj(response.query, 'query')
+    webOutObj(response.sasdata, 'sasdata', response.$sasdata)
+    webOutObj(response.sasparams, 'sasparams')
+    webOutObj(response.versions, 'versions')
+    webOutObj(response.xl_rules, 'xl_rules')
+}
 webOutClose()

sas/mocks/sasjs/services/editors/loadfile.js

@@ -59,8 +59,8 @@ for (let i = 1; i <= 10; i++) {
 
 let libds = ''
 if (typeof table !== 'undefined' && table) libds = String(table).toUpperCase()
-const libref = libds.split('.')[0] || dcLibref
-const dsn = libds.split('.')[1] || ''
+let libref = libds.split('.')[0] || dcLibref
+let dsn = libds.split('.')[1] || ''
 
 // ─── Validate (mirrors the mp_abort checks in loadfile.sas) ─────────────────
 
@@ -167,6 +167,29 @@ if (!msg) {
     }
 
     if (!msg) {
+        // ─── POST_EDIT_HOOK ──────────────────────────────────────────────────
+        // Mirrors mpe_loader.sas: the hook runs before the MPE_SUBMIT record is
+        // written, while LIBREF / DS are still ordinary macro variables, so it
+        // can re-point the changeset at a different table.  The mock eval()s the
+        // hook's .js counterpart in this scope so it can reassign libref / dsn.
+        const hookTables = mpeLoadTableData('MPE_TABLES')
+        const hookReg = (hookTables && hookTables.rows)
+            ? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
+            : null
+        if (hookReg && hookReg.post_edit_hook) {
+            const hookSrc = mockHookSource(hookReg.post_edit_hook)
+            if (hookSrc) {
+                try {
+                    eval(hookSrc)
+                } catch (err) {
+                    console.log(
+                        'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
+                    )
+                }
+                libds = libref + '.' + dsn
+            }
+        }
+
         tableId = makeTableId()
         stageSubmission({
             dataDir: libDataDir(libref),

sas/mocks/sasjs/services/editors/stagedata.js

@@ -20,8 +20,8 @@ const _sctRow = fetchTable('SASControlTable')[0] || {}
 let action = _sctRow.ACTION || 'LOAD'
 let message = _sctRow.MESSAGE || ''
 let libds = _sctRow.LIBDS || ''
-const libref = libds.split('.')[0] || dcLibref
-const dsn = libds.split('.')[1] || ''
+let libref = libds.split('.')[0] || dcLibref
+let dsn = libds.split('.')[1] || ''
 
 // The base table lives in its own library (DC_JSLIB for control tables,
 // TESTDATA for demo user tables); the staging dir stays in the control lib.
@@ -56,6 +56,31 @@ if (schema.length > 0) {
   })
 }
 
+// ─── POST_EDIT_HOOK ──────────────────────────────────────────────────────────
+// Mirrors mpe_loader.sas: %mpe_runhook(POST_EDIT_HOOK) runs before the MPE_SUBMIT
+// record is written, while LIBREF / DS are still ordinary macro variables - which
+// is what lets a hook re-point a changeset at a different table.  The mock
+// eval()s the hook's .js counterpart in this scope so it can reassign
+// libref / dsn / libds; the submit record then names the new base table, exactly
+// as the SAS hook does via call symputx.
+const hookTables = makeTableLoader(mpeDataDir)('MPE_TABLES')
+const hookReg = (hookTables && hookTables.rows)
+    ? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
+    : null
+if (hookReg && hookReg.post_edit_hook) {
+    const hookSrc = mockHookSource(hookReg.post_edit_hook)
+    if (hookSrc) {
+        try {
+            eval(hookSrc)
+        } catch (err) {
+            console.log(
+                'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
+            )
+        }
+        libds = libref + '.' + dsn
+    }
+}
+
 // ─── Stage the data ──────────────────────────────────────────────────────────
 // One subfolder per submission (DSID) under the staging directory, so that the
 // staged data can sit alongside related artifacts (submit/approval logs,

sas/mocks/sasjs/services/hooks/demo_mirror_postedit.js

@@ -0,0 +1,31 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// Mock of a POST_EDIT_HOOK program
+//
+// Registered against TESTDATA.DEMO_MIRROR (MPE_TABLES.post_edit_hook).
+//
+// This file is eval'd by editors/stagedata.js and editors/loadfile.js
+// (mirroring how mpe_loader.sas %includes the hook program via %mpe_runhook),
+// so it shares the caller's scope: libref, dsn, libds and every dcMockUtils
+// function are available directly.
+//
+// The real hook runs in mpe_loader BEFORE the submit record is written, while
+// LIBREF / DS are still ordinary macro variables - which is what lets it
+// re-point a changeset at a different table:
+//
+//     data _null_;
+//       call symputx('libref','TESTDATA');
+//       call symputx('ds','DEMO_ORDERS');
+//     run;
+//
+// (call symputx reaches the calling service's variable because LIBREF/DS are
+// not declared %local in mpe_loader; a %let creates a new variable in the
+// hook's own scope and is discarded.)
+//
+// The mock's equivalent is to reassign libref / dsn, so the MPE_SUBMIT record
+// names the real table and the approval is raised against it rather than
+// against the empty mirror.
+// ─────────────────────────────────────────────────────────────────────────────
+
+libref = 'TESTDATA'
+dsn = 'DEMO_ORDERS'
+console.log('[HOOK DEBUG] demo_mirror_postedit: routing changeset to ' + libref + '.' + dsn)

sas/mocks/sasjs/services/hooks/demo_mirror_preedit.js

@@ -0,0 +1,35 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// Mock of a PRE_EDIT_HOOK program
+//
+// Registered against TESTDATA.DEMO_MIRROR (MPE_TABLES.pre_edit_hook), which is
+// an EMPTY mirror of TESTDATA.DEMO_ORDERS.
+//
+// This file is eval'd by editors/getdata.js (mirroring how getdata.sas
+// %includes the hook program via %mpe_runhook), so it shares the caller's
+// scope: visibleRows, visibleColumns, tableReg, requestedLibref and every
+// dcMockUtils function (loadTableAnyLib, makeTableLoader, libDataDir, ...) are
+// available directly.
+//
+// The real hook is a .sas program which runs after the filter has been applied
+// and the table sorted, with the editor data in work.OUT:
+//
+//     data work.out;
+//       set TESTDATA.DEMO_ORDERS;
+//     run;
+//
+// The mock's equivalent is to replace visibleRows / visibleColumns with the
+// rows and columns of the real table, so an empty mirror still shows live data.
+// (A real hook that must honour the user's filter has to re-apply it - the
+// filter has already been applied to the empty mirror by the time it runs.)
+// ─────────────────────────────────────────────────────────────────────────────
+
+const realTable = loadTableAnyLib('DEMO_ORDERS')
+if (realTable && realTable.data) {
+    visibleRows = realTable.data.rows.map((row) => ({ ...row }))
+    visibleColumns = realTable.data.columns
+    console.log(
+        '[HOOK DEBUG] demo_mirror_preedit: loaded ' +
+            visibleRows.length +
+            ' rows from DEMO_ORDERS'
+    )
+}

sas/mocks/sasjs/services/public/getcols.js

@@ -0,0 +1,80 @@
+const nodePath = require('path')
+
+let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
+const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
+const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
+const dcLibref = 'DC_JSLIB'
+
+// Load shared DC mock utilities
+eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
+
+// ─── Parse input ──────────────────────────────────────────────────────────────
+// getcols.sas reads %mf_getvalue(work.iwant,libds). The adapter serialises the
+// input table as CSV, which fetchTable parses - so the row is `_iwantRow` (the
+// runtime already owns the name `iwant`).
+
+const _iwantRow = fetchTable('iwant')[0] || {}
+const libds = String(_iwantRow.libds || _iwantRow.LIBDS || '').trim()
+
+let libref = dcLibref
+let table = ''
+if (libds) {
+    const parts = libds.split('.')
+    if (parts.length >= 2) {
+        libref = parts[0].trim()
+        table = parts[1].trim()
+    }
+}
+
+// ─── Load the table schema ────────────────────────────────────────────────────
+// Same resolution as getdata.js: the requested libref first, then any libref
+// (a LIBDS that doesn't match the data folder still resolves).
+
+let tableData = table ? makeTableLoader(libDataDir(libref))(table) : null
+if (!tableData) {
+    // Fall back to searching every libref if the table isn't in the requested
+    // libref (e.g. a LIBDS that doesn't match the actual data folder), and take
+    // the libref that actually held it for the registry lookup below.
+    const found = loadTableAnyLib(table)
+    if (found) {
+        tableData = found.data
+        libref = found.libref
+    }
+}
+
+// ─── Build the column list ────────────────────────────────────────────────────
+// Every column, as getdata.sas emits them: the temporal columns are dropped from
+// the data, not from the cols payload, so a filter can still be built on them -
+// which is exactly what "please filter and try again" wants to do.
+
+let cols = []
+
+if (tableData && tableData.columns) {
+    // mpGetcols, then the mapping getcols.sas applies on top of %mp_getcols:
+    // DDTYPE in the client's vocabulary, and TYPE as the SAS C/N it has always
+    // been.  The client maps TYPE to its own num/char vocabulary in
+    // sas-store.getCols, because the picker keys its operator set and its value
+    // quoting on it and this list does not pass through mergeColsRules.
+    cols = mpGetcols(tableData.columns)
+        .map(c => {
+            const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
+            return {
+                NAME: c.NAME,
+                LABEL: c.LABEL,
+                FORMAT: c.FORMAT || ' ',
+                FMTNAME: c.FMTNAME || ' ',
+                DDTYPE: ddtype,
+                TYPE: ddtype === 'C' ? 'C' : 'N',
+                CLS_RULE: 'READ',
+                MEMLABEL: ' ',
+                DESC: ' ',
+                LONGDESC: ' ',
+                COLTYPE: ' '
+            }
+        })
+        .sort((a, b) => a.NAME.localeCompare(b.NAME))
+}
+
+webOutOpen()
+webOutObj(cols, 'cols')
+webOutClose()

sas/mocks/sasjs/services/public/viewdata.js

@@ -8,9 +8,15 @@ const mpeDataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', dcLibref
 
 eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
 
-// Row cap for the viewer - mirrors DC_MAXOBS_WEBVIEW in viewdata.sas, and the
+// Row cap for the viewer - mirrors DC_MAXOBS_WEBVIEW in viewdata.sas, read from
+// MPE_CONFIG the way the real service reads it from the settings program (an
+// absent or switched-off option falls back to the shipped default).  Also the
 // MAXROWS value reported in sasparams.
-const maxObsWebView = 500
+const maxObsWebView = dcConfigNumber(
+    loadDcConfig(driveRoot, appLoc, mpeDataDir),
+    'DC_MAXOBS_WEBVIEW',
+    2000
+)
 
 // ─── Parse input ──────────────────────────────────────────────────────────────
 
@@ -95,16 +101,15 @@ let vars = {}
 let nobs = 0
 
 if (tableData && tableData.columns) {
-    cols = tableData.columns.map(col => {
-        const ddtype = getDdType(col)
-        const fmtname = col.format
-            ? col.format.replace(/[\d.]+$/, '').replace('datetime', 'DATETIME').replace('date', 'DATE').replace('time', 'TIME').replace('best', 'BEST').replace('E8601DT', 'DATETIME')
-            : ' '
-        const formatStr = col.type === 'N' ? (col.format || '8.') : ('$' + col.length + '.')
+    // mpGetcols, the JS counterpart of %mp_getcols.sas - the macro viewdata.sas
+    // calls for its own column list.
+    cols = mpGetcols(tableData.columns).map(c => {
+        const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
+        const formatStr = c.TYPE === 'N' ? (c.FORMAT || '8.') : ('$' + c.LENGTH + '.')
         return {
-            NAME: col.name, TYPE: col.type === 'N' ? 'N' : 'C', LENGTH: col.length,
-            FORMAT: formatStr, FMTNAME: fmtname || ' ', DDTYPE: ddtype,
-            LABEL: col.label || col.name
+            NAME: c.NAME, TYPE: c.TYPE, LENGTH: c.LENGTH,
+            FORMAT: formatStr, FMTNAME: c.FMTNAME || ' ', DDTYPE: ddtype,
+            LABEL: c.LABEL
         }
     })
 

sas/package-lock.json

@@ -6,8 +6,8 @@
     "": {
       "name": "dc-sas",
       "dependencies": {
-        "@sasjs/cli": "4.20.4",
-        "@sasjs/core": "5.2.8"
+        "@sasjs/cli": "4.20.9",
+        "@sasjs/core": "5.5.2"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -153,68 +153,64 @@
         "lodash.isnil": "^4.0.0"
       }
     },
-    "node_modules/@nodelib/fs.scandir": {
-      "version": "2.1.5",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
-      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
-      "license": "MIT",
-      "dependencies": {
-        "@nodelib/fs.stat": "2.0.5",
-        "run-parallel": "^1.1.9"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/@nodelib/fs.stat": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
-      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/@nodelib/fs.walk": {
-      "version": "1.2.8",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
-      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
-      "license": "MIT",
-      "dependencies": {
-        "@nodelib/fs.scandir": "2.1.5",
-        "fastq": "^1.6.0"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/@sasjs/adapter": {
-      "version": "4.19.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.19.0.tgz",
-      "integrity": "sha512-AhSldduDFPAuhRsXdzeiwMqEm0lm6dnh2yVxDSUmxxw8V9YShmeGPmEzvU8n2Cl4VAyrhc+Ietc5YWFYC03G+Q==",
+      "version": "4.19.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.19.2.tgz",
+      "integrity": "sha512-7uxkpk0kjK4PZ552YK0CsTmEKEurK/1uzloR4rxplbViRkVdaewP5l0hIrGDBVM4z0gmV5pf1jh8uil3/NSAzw==",
       "license": "ISC",
       "dependencies": {
-        "@sasjs/utils": "^3.6.0",
-        "axios": "1.18.1",
+        "@sasjs/utils": "3.6.4",
+        "axios": "1.20.0",
         "axios-cookiejar-support": "5.0.5",
         "form-data": "4.0.6",
-        "https": "1.0.0",
         "tough-cookie": "4.1.3"
       }
     },
-    "node_modules/@sasjs/cli": {
-      "version": "4.20.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.20.4.tgz",
-      "integrity": "sha512-hKE002Dm9AzWU62qpRyYpjQv8BIXG951Z0ToNdG9J1wvvJNYH7h6rGv7kdQMbgvXff/KOQvVdnGJZ023SrG+Jw==",
+    "node_modules/@sasjs/adapter/node_modules/@sasjs/utils": {
+      "version": "3.6.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.4.tgz",
+      "integrity": "sha512-tLbvIFMRb3KrHW5EapvMS/0tSa6W/SYCuR+fcdavayQ4eaUFayeRN3HKu2gXLtWJjHf/HijPVTsNwkIRwDvJMA==",
       "license": "ISC",
       "dependencies": {
-        "@sasjs/adapter": "^4.19.0",
-        "@sasjs/core": "5.2.9",
-        "@sasjs/lint": "2.5.0",
-        "@sasjs/utils": "3.6.2",
+        "@fast-csv/format": "4.3.5",
+        "@types/fs-extra": "11.0.4",
+        "@types/prompts": "2.4.9",
+        "chalk": "4.1.1",
+        "cli-table": "0.3.11",
+        "consola": "3.4.2",
+        "fs-extra": "11.4.1",
+        "jwt-decode": "3.1.2",
+        "prompts": "2.4.2"
+      }
+    },
+    "node_modules/@sasjs/adapter/node_modules/chalk": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@sasjs/cli": {
+      "version": "4.20.9",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.20.9.tgz",
+      "integrity": "sha512-wJCdd6fRI0RVwEDLDsauUbsKQsH69aaYJqDkV9WfSKkRH1VxgU9zFGHdSxrKLOzrjwwybn/DGzh2xcgeeyl6jA==",
+      "license": "ISC",
+      "dependencies": {
+        "@sasjs/adapter": "4.19.2",
+        "@sasjs/core": "5.3.0",
+        "@sasjs/lint": "4.1.2",
+        "@sasjs/utils": "3.6.5",
         "chalk": "4.1.2",
         "dotenv": "17.4.2",
-        "find": "0.3.0",
         "js-base64": "3.9.3",
         "jsdom": "26.0.0",
         "jwt-decode": "3.1.2",
@@ -224,7 +220,6 @@
         "node-powershell": "5.0.1",
         "ora": "5.4.1",
         "prompts": "2.4.2",
-        "shelljs": "0.10.0",
         "ssl-root-cas": "1.3.1",
         "xml": "1.0.1",
         "yargs": "17.6.2",
@@ -233,48 +228,84 @@
       },
       "bin": {
         "sasjs": "build/index.js"
+      },
+      "engines": {
+        "node": "^14.18.0 || >=16.10.0"
       }
     },
     "node_modules/@sasjs/cli/node_modules/@sasjs/core": {
-      "version": "5.2.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.9.tgz",
-      "integrity": "sha512-R5wtJF0ANHchaURqTF2t1vRXrhUJ+uDfj21ewzdOq4Lp14CWW22+yYeRQooY4QOpyHUyFDjReA5q6SoJt2Tz1g==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.3.0.tgz",
+      "integrity": "sha512-6x1hEPa1pCENyeG0Nro3YcZZ+A8Ns60pO6gDuvG1X6oWdwOKKPpZztQNKE6khFmXXIGl0uUgT9CbappUET+odA==",
       "license": "MIT"
     },
     "node_modules/@sasjs/core": {
-      "version": "5.2.8",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.8.tgz",
-      "integrity": "sha512-+K8tXTrsw52NR1y5MFHW0+Jf9euL+GukwqA79FIHHu4Oj+BhHBm1uZiyB5tPrvwl3V8mY1CQhMOT83//bV+jWg==",
+      "version": "5.5.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.5.2.tgz",
+      "integrity": "sha512-puojJhAKAKg5o0uHdGSkpQ3cXgzajMY9AXAm3R22l6bZ4GKvC03qRq1ueGjBBF3npNxviwIKCABTsNEtjs1LAw==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-2.5.0.tgz",
-      "integrity": "sha512-bVUtYnKhCigXylCCvQ6dsAIywpPKkX3BXatMqOWjDVw9R30jfj4M3/qn05fcR45L42Y7e5Tuzlt+LYs+9k95Yg==",
-      "hasInstallScript": true,
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-4.1.2.tgz",
+      "integrity": "sha512-P7QqeMPnhg/AvqH40dmyY2OArlvAYAp81/ijK/PhmS+x/HslpypdLoNy5iN7SwixUWPde5fPwV/ZBmcjVK0uLA==",
       "license": "MIT",
       "dependencies": {
-        "@sasjs/utils": "3.6.2",
-        "ignore": "7.0.8"
+        "@sasjs/utils": "3.6.4",
+        "ignore": "7.0.11"
+      },
+      "engines": {
+        "node": "^14.18.0 || >=16.10.0"
       }
     },
-    "node_modules/@sasjs/utils": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.2.tgz",
-      "integrity": "sha512-AfC5OJuJx3NAbntKq2J8E8NNpRXthYfuqOyp2lOmOkTctL3mAfrA54U/cdPQvrlftXNxpK9P2U0RU10qNjlEzQ==",
+    "node_modules/@sasjs/lint/node_modules/@sasjs/utils": {
+      "version": "3.6.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.4.tgz",
+      "integrity": "sha512-tLbvIFMRb3KrHW5EapvMS/0tSa6W/SYCuR+fcdavayQ4eaUFayeRN3HKu2gXLtWJjHf/HijPVTsNwkIRwDvJMA==",
       "license": "ISC",
       "dependencies": {
         "@fast-csv/format": "4.3.5",
         "@types/fs-extra": "11.0.4",
-        "@types/prompts": "2.0.13",
+        "@types/prompts": "2.4.9",
         "chalk": "4.1.1",
-        "cli-table": "0.3.6",
-        "consola": "2.15.0",
-        "find": "0.3.0",
-        "fs-extra": "11.3.0",
+        "cli-table": "0.3.11",
+        "consola": "3.4.2",
+        "fs-extra": "11.4.1",
         "jwt-decode": "3.1.2",
-        "prompts": "2.4.1",
-        "valid-url": "1.0.9"
+        "prompts": "2.4.2"
+      }
+    },
+    "node_modules/@sasjs/lint/node_modules/chalk": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@sasjs/utils": {
+      "version": "3.6.5",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.5.tgz",
+      "integrity": "sha512-2iCy0EmZJCQSG1JsVa1t/ulUb7lfdBnp6wiRPRl9/AaxRu8D30ypqUKn6heos1lxhNxg42JFsry8rlupiajdHg==",
+      "license": "ISC",
+      "dependencies": {
+        "@fast-csv/format": "4.3.5",
+        "@types/fs-extra": "11.0.4",
+        "@types/prompts": "2.4.9",
+        "chalk": "4.1.1",
+        "cli-table": "0.3.11",
+        "consola": "3.4.2",
+        "fs-extra": "11.4.1",
+        "jwt-decode": "3.1.2",
+        "prompts": "2.4.2"
       }
     },
     "node_modules/@sasjs/utils/node_modules/chalk": {
@@ -293,19 +324,6 @@
         "url": "https://github.com/chalk/chalk?sponsor=1"
       }
     },
-    "node_modules/@sasjs/utils/node_modules/prompts": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.1.tgz",
-      "integrity": "sha512-EQyfIuO2hPDsX1L/blblV+H7I0knhgAd82cVneCwcdND9B8AuCDuRcBH6yIcG4dFzlOUqbazQqwGjx5xmsNLuQ==",
-      "license": "MIT",
-      "dependencies": {
-        "kleur": "^3.0.3",
-        "sisteransi": "^1.0.5"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/@types/fs-extra": {
       "version": "11.0.4",
       "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz",
@@ -332,12 +350,13 @@
       "license": "MIT"
     },
     "node_modules/@types/prompts": {
-      "version": "2.0.13",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "version": "2.4.9",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.4.9.tgz",
+      "integrity": "sha512-qTxFi6Buiu8+50/+3DGIWLHM6QuWsEKugJnnP6iv2Mc4ncxE4A/OJkjuVOA+5X0X1S/nq5VJRa8Lu+nwcvbrKA==",
       "license": "MIT",
       "dependencies": {
-        "@types/node": "*"
+        "@types/node": "*",
+        "kleur": "^3.0.3"
       }
     },
     "node_modules/accumulate-stream": {
@@ -390,13 +409,13 @@
       "license": "MIT"
     },
     "node_modules/axios": {
-      "version": "1.18.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz",
-      "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==",
+      "version": "1.20.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz",
+      "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==",
       "license": "MIT",
       "dependencies": {
         "follow-redirects": "^1.16.0",
-        "form-data": "^4.0.5",
+        "form-data": "^4.0.6",
         "https-proxy-agent": "^5.0.1",
         "proxy-from-env": "^2.1.0"
       }
@@ -451,18 +470,6 @@
         "readable-stream": "^3.4.0"
       }
     },
-    "node_modules/braces": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
-      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
-      "license": "MIT",
-      "dependencies": {
-        "fill-range": "^7.1.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -574,9 +581,9 @@
       }
     },
     "node_modules/cli-table": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.6.tgz",
-      "integrity": "sha512-ZkNZbnZjKERTY5NwC2SeMeLeifSPq/pubeRoTpdr3WchLlnZg6hEgvHkK5zL7KNFdd9PmHN8lxrENUwI3cE8vQ==",
+      "version": "0.3.11",
+      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.11.tgz",
+      "integrity": "sha512-IqLQi4lO0nIB4tcdTpN4LCB9FI3uqrJZK7RC515EnhZ6qBaglkIgICb1wjeAqpdoOabm1+SuQtkXIPdYC93jhQ==",
       "dependencies": {
         "colors": "1.0.3"
       },
@@ -647,23 +654,12 @@
       }
     },
     "node_modules/consola": {
-      "version": "2.15.0",
-      "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.0.tgz",
-      "integrity": "sha512-vlcSGgdYS26mPf7qNi+dCisbhiyDnrN1zaRbw3CSuc2wGOMEGGPsp46PdRG5gqXwgtJfjxDkxRNAgRPr1B77vQ==",
-      "license": "MIT"
-    },
-    "node_modules/cross-spawn": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
-      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz",
+      "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==",
       "license": "MIT",
-      "dependencies": {
-        "path-key": "^3.1.0",
-        "shebang-command": "^2.0.0",
-        "which": "^2.0.1"
-      },
       "engines": {
-        "node": ">= 8"
+        "node": "^14.18.0 || >=16.10.0"
       }
     },
     "node_modules/cssstyle": {
@@ -840,79 +836,10 @@
       "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
       "license": "MIT"
     },
-    "node_modules/execa": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
-      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
-      "license": "MIT",
-      "dependencies": {
-        "cross-spawn": "^7.0.3",
-        "get-stream": "^6.0.0",
-        "human-signals": "^2.1.0",
-        "is-stream": "^2.0.0",
-        "merge-stream": "^2.0.0",
-        "npm-run-path": "^4.0.1",
-        "onetime": "^5.1.2",
-        "signal-exit": "^3.0.3",
-        "strip-final-newline": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/execa?sponsor=1"
-      }
-    },
-    "node_modules/fast-glob": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
-      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
-      "license": "MIT",
-      "dependencies": {
-        "@nodelib/fs.stat": "^2.0.2",
-        "@nodelib/fs.walk": "^1.2.3",
-        "glob-parent": "^5.1.2",
-        "merge2": "^1.3.0",
-        "micromatch": "^4.0.8"
-      },
-      "engines": {
-        "node": ">=8.6.0"
-      }
-    },
-    "node_modules/fastq": {
-      "version": "1.20.3",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
-      "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
-      "license": "ISC",
-      "dependencies": {
-        "reusify": "^1.0.4"
-      }
-    },
-    "node_modules/fill-range": {
-      "version": "7.1.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
-      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
-      "license": "MIT",
-      "dependencies": {
-        "to-regex-range": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/find": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
-      "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==",
-      "license": "MIT",
-      "dependencies": {
-        "traverse-chain": "~0.1.0"
-      }
-    },
     "node_modules/follow-redirects": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
-      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
+      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
       "funding": [
         {
           "type": "individual",
@@ -946,9 +873,9 @@
       }
     },
     "node_modules/fs-extra": {
-      "version": "11.3.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
-      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
+      "version": "11.4.1",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.1.tgz",
+      "integrity": "sha512-KYAb4c9BJQI6QqGKthV68OHe0badztdXJWKo0WtBA9IuCFPTKvE5ZdUBglP833aMjhaSPNO4A5j/EkzZtGlKjA==",
       "license": "MIT",
       "dependencies": {
         "graceful-fs": "^4.2.0",
@@ -1014,30 +941,6 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/get-stream": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/glob-parent": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
-      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
-      "license": "ISC",
-      "dependencies": {
-        "is-glob": "^4.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/gopd": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -1153,12 +1056,6 @@
         "node": ">= 14"
       }
     },
-    "node_modules/https": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/https/-/https-1.0.0.tgz",
-      "integrity": "sha512-4EC57ddXrkaF0x83Oj8sM6SLQHAWXw90Skqu2M4AEWENZ3F02dFJE/GARA8igO79tcgYqGrD7ae4f5L3um2lgg==",
-      "license": "ISC"
-    },
     "node_modules/https-proxy-agent": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
@@ -1184,15 +1081,6 @@
         "node": ">= 6.0.0"
       }
     },
-    "node_modules/human-signals": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
-      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
-      "license": "Apache-2.0",
-      "engines": {
-        "node": ">=10.17.0"
-      }
-    },
     "node_modules/iconv-lite": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
@@ -1226,9 +1114,9 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/ignore": {
-      "version": "7.0.8",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.8.tgz",
-      "integrity": "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q==",
+      "version": "7.0.11",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.11.tgz",
+      "integrity": "sha512-YChdK5txDjwGUvgR7oCJcLGkwi3LDh8Zx8tS7ndYciLEg/oOCL26VUBSUEtgS7EjiybcBqFxh5j0rAdDX7/bbg==",
       "license": "MIT",
       "engines": {
         "node": ">= 4"
@@ -1255,15 +1143,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/is-extglob": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -1273,18 +1152,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/is-glob": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
-      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
-      "license": "MIT",
-      "dependencies": {
-        "is-extglob": "^2.1.1"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/is-interactive": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
@@ -1294,33 +1161,12 @@
         "node": ">=8"
       }
     },
-    "node_modules/is-number": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
-      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.12.0"
-      }
-    },
     "node_modules/is-potential-custom-element-name": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
       "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
       "license": "MIT"
     },
-    "node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/is-unicode-supported": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz",
@@ -1345,12 +1191,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-      "license": "ISC"
-    },
     "node_modules/js-base64": {
       "version": "3.9.3",
       "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.3.tgz",
@@ -1532,34 +1372,6 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/merge-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
-      "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
-      "license": "MIT"
-    },
-    "node_modules/merge2": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
-      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/micromatch": {
-      "version": "4.0.8",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
-      "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
-      "license": "MIT",
-      "dependencies": {
-        "braces": "^3.0.3",
-        "picomatch": "^2.3.1"
-      },
-      "engines": {
-        "node": ">=8.6"
-      }
-    },
     "node_modules/mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@@ -1630,18 +1442,6 @@
         "is-wsl": "^2.2.0"
       }
     },
-    "node_modules/npm-run-path": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
-      "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
-      "license": "MIT",
-      "dependencies": {
-        "path-key": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/nwsapi": {
       "version": "2.2.27",
       "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.27.tgz",
@@ -1744,33 +1544,12 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/path-key": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/pend": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
       "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
       "license": "MIT"
     },
-    "node_modules/picomatch": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
-      "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/jonschlinkert"
-      }
-    },
     "node_modules/prompts": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
@@ -1820,26 +1599,6 @@
       "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
       "license": "MIT"
     },
-    "node_modules/queue-microtask": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
-      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "license": "MIT"
-    },
     "node_modules/readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -1882,45 +1641,12 @@
         "node": ">=8"
       }
     },
-    "node_modules/reusify": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
-      "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
-      "license": "MIT",
-      "engines": {
-        "iojs": ">=1.0.0",
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/rrweb-cssom": {
       "version": "0.8.0",
       "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz",
       "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==",
       "license": "MIT"
     },
-    "node_modules/run-parallel": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
-      "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "license": "MIT",
-      "dependencies": {
-        "queue-microtask": "^1.2.2"
-      }
-    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1959,40 +1685,6 @@
         "node": ">=v12.22.7"
       }
     },
-    "node_modules/shebang-command": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-      "license": "MIT",
-      "dependencies": {
-        "shebang-regex": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/shebang-regex": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/shelljs": {
-      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.10.0.tgz",
-      "integrity": "sha512-Jex+xw5Mg2qMZL3qnzXIfaxEtBaC4n7xifqaqtrZDdlheR70OGkydrPJWT0V1cA1k3nanC86x9FwAmQl6w3Klw==",
-      "license": "BSD-3-Clause",
-      "dependencies": {
-        "execa": "^5.1.1",
-        "fast-glob": "^3.3.2"
-      },
-      "engines": {
-        "node": ">=18"
-      }
-    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -2049,15 +1741,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/strip-final-newline": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
-      "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/supports-color": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -2094,18 +1777,6 @@
       "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
       "license": "MIT"
     },
-    "node_modules/to-regex-range": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
-      "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-      "license": "MIT",
-      "dependencies": {
-        "is-number": "^7.0.0"
-      },
-      "engines": {
-        "node": ">=8.0"
-      }
-    },
     "node_modules/tough-cookie": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
@@ -2142,12 +1813,6 @@
         "node": ">=18"
       }
     },
-    "node_modules/traverse-chain": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
-      "integrity": "sha512-up6Yvai4PYKhpNp5PkYtx50m3KbwQrqDwbuZP/ItyL64YEWHAvH6Md83LFLV/GRSk/BoUVwwgUzX6SOQSbsfAg==",
-      "license": "MIT"
-    },
     "node_modules/trim-buffer": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/trim-buffer/-/trim-buffer-5.0.0.tgz",
@@ -2179,11 +1844,6 @@
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
       "license": "MIT"
     },
-    "node_modules/valid-url": {
-      "version": "1.0.9",
-      "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz",
-      "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA=="
-    },
     "node_modules/w3c-xmlserializer": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz",
@@ -2249,21 +1909,6 @@
         "node": ">=18"
       }
     },
-    "node_modules/which": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-      "license": "ISC",
-      "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/node-which"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",

sas/package.json

@@ -30,8 +30,8 @@
   },
   "private": true,
   "dependencies": {
-    "@sasjs/cli": "4.20.4",
-    "@sasjs/core": "5.2.8"
+    "@sasjs/cli": "4.20.9",
+    "@sasjs/core": "5.5.2"
   },
   "overrides": {
     "nanoid": "3.3.18"

sas/sasjs/db/migrations/20261003_row_and_cell_limits.sas

@@ -0,0 +1,119 @@
+/**
+  @file
+  @brief migration script - raise the VIEW row limit, add the EDIT cell limit
+
+  OPTIONAL CHANGE - updates three options in MPE_CONFIG:
+
+  |option|old|new|
+  |---|---|---|
+  |DC_MAXOBS_WEBVIEW|500|2000|
+  |DC_MAXOBS_WEBEDIT|100|250|
+  |DC_MAXCELLS_WEBEDIT|(new)|200000|
+
+  MPE_CONFIG is bitemporal, so the current row for each option is closed and a
+  new row written with the new value.  A site that has already chosen its own
+  values for these options will have them replaced - edit the values below
+  first if that is not what you want.
+
+  An option that is switched off - a current row with var_active=0 - is left
+  alone, and a note is written to the log.  Writing an active row for it would
+  silently switch it back on.
+
+  Without this script the code defaults still apply (the same numbers), but
+  only for options that are not present at all - an existing install carries
+  its own rows, so it keeps 500 / 100 until this is run.
+
+  Each block below uses %do/%end on BOTH branches.  A single-statement
+  %then/%else in open code (outside a %macro) is rejected by the Viya compute
+  engine with "Expected %DO not found", so it must not be shortened back.
+
+**/
+
+%let dclib=YOURDCLIB;
+%let now=%sysfunc(datetime());
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+/* which of the three options the site has switched off */
+proc sql noprint;
+  select count(*) into :off_view trimmed
+    from &dclib..mpe_config
+    where var_name='DC_MAXOBS_WEBVIEW' and var_active=0 and &now lt tx_to;
+  select count(*) into :off_edit trimmed
+    from &dclib..mpe_config
+    where var_name='DC_MAXOBS_WEBEDIT' and var_active=0 and &now lt tx_to;
+  select count(*) into :off_cells trimmed
+    from &dclib..mpe_config
+    where var_name='DC_MAXCELLS_WEBEDIT' and var_active=0 and &now lt tx_to;
+quit;
+
+%if &off_view=0 %then %do;
+  proc sql noprint;
+    update &dclib..mpe_config
+      set tx_to=&now
+      where var_name='DC_MAXOBS_WEBVIEW' and var_active=1 and &now lt tx_to;
+
+    insert into &dclib..mpe_config
+      set tx_from=&now
+        ,tx_to='31DEC9999:23:59:59'dt
+        ,var_scope="DC"
+        ,var_name="DC_MAXOBS_WEBVIEW"
+        ,var_value='2000'
+        ,var_active=1
+        ,var_desc='This sets the maximum number of observations that can be'
+          !!' loaded into the browser in the VIEW screen.';
+  quit;
+%end;
+%else %do;
+  %put NOTE: DC_MAXOBS_WEBVIEW is switched off (var_active=0) - not changed;
+%end;
+
+%if &off_edit=0 %then %do;
+  proc sql noprint;
+    update &dclib..mpe_config
+      set tx_to=&now
+      where var_name='DC_MAXOBS_WEBEDIT' and var_active=1 and &now lt tx_to;
+
+    insert into &dclib..mpe_config
+      set tx_from=&now
+        ,tx_to='31DEC9999:23:59:59'dt
+        ,var_scope="DC"
+        ,var_name="DC_MAXOBS_WEBEDIT"
+        ,var_value='250'
+        ,var_active=1
+        ,var_desc='This sets the maximum number of observations that can be'
+          !!' loaded into the browser for editing in the EDIT screen.  The'
+          !!' DC_MAXCELLS_WEBEDIT limit is applied at the same time, whichever'
+          !!' is reached first.';
+  quit;
+%end;
+%else %do;
+  %put NOTE: DC_MAXOBS_WEBEDIT is switched off (var_active=0) - not changed;
+%end;
+
+%if &off_cells=0 %then %do;
+  proc sql noprint;
+    update &dclib..mpe_config
+      set tx_to=&now
+      where var_name='DC_MAXCELLS_WEBEDIT' and var_active=1 and &now lt tx_to;
+
+    /* Why the cell limit exists, kept out of var_desc - MPE_CONFIG.VAR_DESC
+      is varchar(300), and a truncation WARNING aborts the build.  A wide
+      table is expensive long before it is tall - 250 rows of a 1000 column
+      table is a quarter of a million cells. */
+    insert into &dclib..mpe_config
+      set tx_from=&now
+        ,tx_to='31DEC9999:23:59:59'dt
+        ,var_scope="DC"
+        ,var_name="DC_MAXCELLS_WEBEDIT"
+        ,var_value='200000'
+        ,var_active=1
+        ,var_desc='This sets the maximum number of CELLS (rows multiplied by'
+          !!' columns) that can be loaded into the browser for editing in the'
+          !!' EDIT screen.  Whichever of this and DC_MAXOBS_WEBEDIT is reached'
+          !!' first applies.';
+  quit;
+%end;
+%else %do;
+  %put NOTE: DC_MAXCELLS_WEBEDIT is switched off (var_active=0) - not changed;
+%end;

sas/sasjs/macros/mpe_loader.sas

@@ -462,6 +462,32 @@ run;
   %return;
 %end;
 
+/* The post edit hook may have re-pointed the changeset at a different table -
+  * that is what lets an empty mirror stand in for a real one.  The target must
+  * itself be registered in MPE_TABLES: the approval screen resolves the table's
+  * audit settings from that row, and a changeset whose base table has no row
+  * cannot be reviewed.  Fail here, while the submitter is still watching. */
+%local target_chk;
+proc sql noprint;
+select count(*) into: target_chk
+  from &mpelib..mpe_tables
+  where tx_from le &now and &now lt tx_to
+    and upcase(libref)="%upcase(&libref)"
+    and upcase(dsn)="%upcase(&ds)";
+%if &target_chk=0 %then %do;
+  %let msg=%str(ERR)OR: target table &libref..&ds is not registered in%trim(
+    ) &mpelib..mpe_tables - a post edit hook may only route a changeset to a%trim(
+    ) registered table;
+  %mpe_loadfail(
+      status=FAILED - TARGET NOT REGISTERED
+    ,now=&now
+    ,mperef=&mperef
+    ,reason_txt=%quote(&msg)
+    ,dc_dttmtfmt=&dc_dttmtfmt.
+  )
+  %return;
+%end;
+
 
 /**
   * send to approve process

sas/sasjs/macros/mpe_makedata.sas

@@ -72,12 +72,37 @@ insert into &lib..mpe_config set
     ,tx_to='31DEC9999:23:59:59'dt
     ,var_scope="DC"
     ,var_name="DC_MAXOBS_WEBEDIT"
-    ,var_value="100"
+    ,var_value="250"
     ,var_active=1
     ,var_desc='This sets the maximum number of observations that can be loaded'
       !!' into the browser for editing in the EDIT screen.  A higher number'
       !!' will require a decent browser (ie, not IE) and more memory on the'
-      !!' client side.';
+      !!' client side.  The DC_MAXCELLS_WEBEDIT limit is applied at the same'
+      !!' time, whichever is reached first.';
+/* Why the cell limit exists, kept out of var_desc - MPE_CONFIG.VAR_DESC is
+  varchar(300), and a truncation WARNING aborts this whole build.  A wide
+  table is expensive long before it is tall - 250 rows of a 1000 column
+  table is a quarter of a million cells. */
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC"
+    ,var_name="DC_MAXCELLS_WEBEDIT"
+    ,var_value="200000"
+    ,var_active=1
+    ,var_desc='This sets the maximum number of CELLS (rows multiplied by'
+      !!' columns) that can be loaded into the browser for editing in the EDIT'
+      !!' screen.  Whichever of this and DC_MAXOBS_WEBEDIT is reached first'
+      !!' applies.';
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC"
+    ,var_name="DC_MAXOBS_WEBVIEW"
+    ,var_value="2000"
+    ,var_active=1
+    ,var_desc='This sets the maximum number of observations that can be'
+      !!' loaded into the browser in the VIEW screen.';
 insert into &lib..mpe_config set
     tx_from=0
     ,tx_to='31DEC9999:23:59:59'dt

sas/sasjs/macros/mpeinit.sas

@@ -115,9 +115,34 @@ run;
   ,msg=%str(Problem during compilation or with STP precode (&syswarningtext))
 )
 
+/* `_debug` is 131 by default, but the adapter sends 128 on the Viya WEB JES
+  * path when runAsTask is enabled (see WebJobExecutor.getRequestParams), and
+  * 128 also arrives when a `log` debug value is requested.  All of them mean
+  * "debug on" - so all of them enable the extra logging below. */
 %if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131"
+  or "&_debug"="128"
 %then %do;
   %let sasjs_mdebug=1;
 %end;
 
+%if "&sasjs_mdebug"="1" %then %do;
+  %if %sysfunc(exist(work.browser_url_vars)) %then %do;
+    data _null_;
+      length name value $1024;
+      set work.browser_url_vars;
+      putlog "NOTE: &sysmacroname: work.browser_url_vars:";
+      putlog name= value=;
+    run;
+  %end;
+  %else %put NOTE: &sysmacroname: no work.browser_url_vars on this request;
+  %if %sysfunc(exist(work.browser_info)) %then %do;
+    data _null_;
+      set work.browser_info;
+      putlog "NOTE: &sysmacroname: work.browser_info:";
+      putlog (_all_)(=);
+    run;
+  %end;
+  %else %put NOTE: &sysmacroname: no work.browser_info on this request;
+%end;
+
 %mend mpeinit;

sas/sasjs/services/admin/exportdb.sas

@@ -4,9 +4,12 @@
   @details If user is in the administrator group, they can call this
   service directly adding the following URL params:
 
-    @li &flavour= (only PGSQL supported at this time)
+    @li &flavour= (SAS, PGSQL or TSQL - defaults to SAS)
     @li &schema= (optional, if target schema is needed)
 
+  Data inserts are generated for the SAS and PGSQL flavours only; the TSQL
+  export contains DDL without inserts.
+
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
   @li mp_abort.sas

sas/sasjs/services/admin/exportdb.test.sas

@@ -72,3 +72,51 @@ run;
   test=EQUALS 1,
   outds=work.test_results
 )
+
+/* test 3 - TSQL flavour (DDL only, no inserts) */
+data work.params;
+  length name $32 value $1000;
+  name='flavour';value='TSQL';output;
+  name='schema';value='public';output;
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params,
+  outref=web3,
+  viyaresult=WEBOUT_TXT,
+  mdebug=&sasjs_mdebug
+)
+
+data work.results3;
+  infile web3;
+  input;
+  putlog _infile_;
+  if index(upcase(_infile_),'CREATE TABLE') then do;
+    putlog 'test passed';
+    output;
+    stop;
+  end;
+run;
+
+/* scan the whole file for any INSERT - TSQL is DDL only */
+data work.results4;
+  infile web3;
+  input;
+  if index(upcase(_infile_),'INSERT INTO') then do;
+    putlog 'unexpected INSERT found';
+    output;
+    stop;
+  end;
+run;
+
+%mp_assertdsobs(work.results3,
+  desc=TSQL flavour DDL file is successfully returned,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+%mp_assertdsobs(work.results4,
+  desc=TSQL flavour export contains no INSERTs (DDL only),
+  test=EQUALS 0,
+  outds=work.test_results
+)

sas/sasjs/services/editors/getdata.sas

@@ -55,6 +55,8 @@
   @li mf_getvarlist.sas
   @li mf_existds.sas
   @li mf_getquotedstr.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mf_getuser.sas
   @li mf_nobs.sas
   @li mf_verifymacvars.sas
@@ -205,7 +207,14 @@ select upcase(loadtype)
     %put NOTE- Please add to &mpelib..MPE_CONFIG table;
     %put NOTE-;%put NOTE-;
     %global DC_MAXOBS_WEBEDIT;
-    %let DC_MAXOBS_WEBEDIT=500;
+    %let DC_MAXOBS_WEBEDIT=250;
+  %end;
+  %if not %symexist(DC_MAXCELLS_WEBEDIT) %then %do;
+    %put NOTE:;%put NOTE- DC_MAXCELLS_WEBEDIT not found!;
+    %put NOTE- Please add to &mpelib..MPE_CONFIG table;
+    %put NOTE-;%put NOTE-;
+    %global DC_MAXCELLS_WEBEDIT;
+    %let DC_MAXCELLS_WEBEDIT=200000;
   %end;
   /* for tables which use RKs/SKs then we just expose the business key to
       users - this lets uploads be sent to multiple environments (with
@@ -228,7 +237,6 @@ select upcase(loadtype)
 
   %if &loadtype=BITEMPORAL %then %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
-      _____DELETE__THIS__RECORD_____="No";
       set &libds;
       where %inc filtref;;
     run;
@@ -243,7 +251,6 @@ select upcase(loadtype)
   %end;
   %else %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
-      _____DELETE__THIS__RECORD_____="No";
       set &libds;
       where %inc filtref;;
     run;
@@ -254,40 +261,61 @@ select upcase(loadtype)
     ,msg=%str(Issue with filtering (line 165) )
   )
 
-  options obs=&DC_MAXOBS_WEBEDIT;
+  /* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
+    then run the hook, then check the row and cell limits.  Checking once,
+    after both, means the counts cover everything that could reach the
+    browser: the rows the selection matched, and anything the hook has added
+    to work.out.  A wide table exhausts the cell budget long before it
+    reaches the row limit - 250 rows of a 1000 column table is a quarter of
+    a million cells - so capping rows alone does not bound the work the
+    browser is asked to do.  The payload itself is capped further down, when
+    work.outdata is built. */
   %let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
   proc sort data=work.out; by &sortPK; run;
-  options obs=max;
 
   %mpe_runhook(PRE_EDIT_HOOK)
 
   %let obscnt=%mf_getattrn(work.out,NLOBS);
+  %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
+  %let cellcnt=%eval(&obscnt * &colcnt);
   %mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
     ,mac=&_program
     ,msg=Table is too big (&obscnt rows) - please filter and try again!
   )
+  %mp_abort(iftrue=(&cellcnt>&DC_MAXCELLS_WEBEDIT)
+    ,mac=&_program
+    ,msg=Selection is too wide (&obscnt rows x &colcnt cols) - please filter and try again!
+  )
 
 
-  /* order delete var and pk fields at start of table */
+  /* order pk fields at start of table.  The delete flag is NOT added here:
+    _____DELETE__THIS__RECORD_____ is a client-owned column, created by the
+    editor itself (see deleteRecordColumnRule.ts), and getdata must never
+    emit it - a real table column of that name is refused by the frontend. */
   %let sourcevars=%mf_wordsInStr1ButNotStr2(
                             Str1=%mf_getvarlist(work.out)
-                            ,Str2= _____DELETE__THIS__RECORD_____ &pk
+                            ,Str2=&pk
                           );
   %put sourcevars=&sourcevars;
+  /* the payload is capped here, at the row limit - the limits above are
+    checked against the whole selection, this is what goes to the browser */
+  options obs=&DC_MAXOBS_WEBEDIT;
   data outdata;
-    /* delete & pk fields come first */
-    attrib _____DELETE__THIS__RECORD_____ &pk label='';
+    /* pk fields come first */
+    %if %length(&pk)>0 %then %do;
+      attrib &pk label='';
+    %end;
     /* keep remaining variable order */
     %if %length(&sourcevars)>0 %then %do;
       attrib &sourcevars label='';
     %end;
-    _____DELETE__THIS__RECORD_____="No ";
     %if %mf_nobs(work.out)=0 %then %do;
       /* send empty row if empty table to help with hot rendering */
       output;
     %end;
     set work.out ;
   run;
+  options obs=max;
 
 
 
@@ -300,7 +328,7 @@ select upcase(loadtype)
     by varnum;
   run;
 
-  data vars3(keep=name type length format label pk varnum  ctrloptions formatd);
+  data vars3(keep=name type length format label pk varnum formatd);
     set vars(rename=(format=format2 type=type2));
     name=upcase(name);
     /* not interested in transaction or processing dates
@@ -321,10 +349,6 @@ select upcase(loadtype)
 
     if name in ('',%upcase(%mf_getQuotedStr(&pk,dlm=%str(,),quote=S)))
       then PK='YES';
-
-    length ctrlOptions $500;
-    if name="_____DELETE__THIS__RECORD_____" then ctrlOptions='["No","Yes"]';
-    else ctrlOptions='';
   run;
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&_program..sas
@@ -339,9 +363,7 @@ select upcase(loadtype)
     length coltype $500.;
     format=upcase(format);
     coltype=cats('{"data":"',name,'"');
-    if ctrlOptions ne '' then
-      colType=cats(coltype,',"type":"dropdown","source":',ctrlOptions,"}");
-    else if type='num' then do;
+    if type='num' then do;
       if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
       then do;
         colType=cats(coltype,',"type":"datetime"}');
@@ -624,7 +646,7 @@ create table dqdata as
 
 /* extract selectbox data */
 %macro dq_selects();
-  %local x source lib ds col;
+  %local x source lib ds col filtref filterds srcview;
   %do x=1 %to &sourcecnt;
     %let source=&&source&x;
     %let lib=%scan(&source,1,.);
@@ -634,13 +656,31 @@ create table dqdata as
     %put &=lib;
     %dc_assignlib(READ,&lib)
     %dc_casload(&lib..&ds)
+    /* Selectbox values are read straight from the source table, so the same
+      Row Level Security that filters the grid must be applied here too -
+      otherwise values from rows the user is not permitted to see leak into
+      the cell dropdown.  %mpe_filtermaster builds the filter; a data step
+      view is used because a fileref cannot be %inc'd inside a proc sql where
+      clause.  EDIT mode mirrors the grid on this (editor) page. */
+    %let filtref=%mf_getuniquefileref();
+    %let filterds=%mf_getuniquename();
+    %mpe_filtermaster(EDIT,&lib..&ds,
+      dclib=&mpelib,
+      outref=&filtref,
+      outds=work.&filterds
+    )
+    %let srcview=%mf_getuniquename();
+    data work.&srcview / view=work.&srcview;
+      set &lib..&ds;
+      where %inc &filtref;;
+    run;
     proc sql;
     create table dqdata&x as
       select distinct "&&base_col&x" as base_col length=32
         ,"&source" as rule_value length=74
         ,cats(&col) as rule_data length=1000
         ,&col as tmp_order
-      from &lib..&ds
+      from work.&srcview
       order by tmp_order;
     /* ensure both numerics and char vals are ordered correctly */
     data work.dqdata&x (drop=tmp_order);
@@ -653,6 +693,12 @@ create table dqdata as
     )
     proc append base=dqdata data=dqdata&x;run;
     proc sql; drop table dqdata&x;
+    /* drop the per-source filter artifacts - WORK is session scoped, but the
+      loop should not leave a view and a filter table behind per source */
+    proc datasets lib=work nolist;
+      delete &srcview &filterds;
+    quit;
+    filename &filtref clear;
   %end;
 %mend dq_selects;
 %dq_selects()
@@ -664,11 +710,6 @@ run;
 
 %mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)
 
-data maxvarlengths;
-  set maxvarlengths;
-  if name='_____DELETE__THIS__RECORD_____' then mAXLEN=3;
-run;
-
 data xl_rules;
   set &mpelib..mpe_excel_config;
   where &dc_dttmtfmt. lt tx_to;

sas/sasjs/services/editors/getdata.test.1.sas

@@ -0,0 +1,124 @@
+/**
+  @file
+  @brief testing getdata row and cell limits
+
+  @details The EDIT screen refuses a selection that is too big on either
+  count - rows (DC_MAXOBS_WEBEDIT) or cells (DC_MAXCELLS_WEBEDIT).  The row
+  check runs first, so a selection over the row limit reports "Table is too
+  big" and one under the row limit but over the cell limit reports
+  "Selection is too wide".  Both aborts are proven by the child job state
+  (an aborted service registers no webout).
+
+  Fixture: dctest.wideboy (1000 rows x 1004 columns), created and registered
+  by testsetup.  Filtered to 300 rows it is over the 250 row limit; filtered
+  to 200 rows it is under the row limit but 200 x 1004 = 200,800 cells, over
+  the 200,000 cell limit.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_filterstore.sas
+  @li mf_getuniquefileref.sas
+  @li mx_execute.sas
+
+**/
+
+%let _program=&appLoc/services/editors/getdata;
+
+/**
+  * build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
+  * in &filter_rk
+  */
+%macro wideboy_filter(maxrow);
+  data work.inquery;
+    length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8
+      VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
+    GROUP_LOGIC='AND';
+    SUBGROUP_LOGIC='AND';
+    SUBGROUP_ID=1;
+    VARIABLE_NM='ROW_ID';
+    OPERATOR_NM='LE';
+    RAW_VALUE="&maxrow";
+    output;
+  run;
+  %mp_filterstore(
+    libds=dctest.wideboy,
+    filter_summary=&dc_libref..mpe_filteranytable,
+    filter_detail=&dc_libref..mpe_filtersource,
+    lock_table=&dc_libref..mpe_lockanytable,
+    maxkeytable=&dc_libref..mpe_maxkeyvalues,
+    queryds=work.inquery,
+    outresult=work.result,
+    outquery=work.query
+  )
+  data _null_;
+    set work.result;
+    call symputx('filter_rk',filter_rk,'g');
+  run;
+%mend wideboy_filter;
+
+/**
+  * Test 1 - row limit (300 rows > 250)
+  */
+%wideboy_filter(300)
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "dctest.wideboy,&filter_rk";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:sascontroltable,
+  outref=web1
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+%let syscc=0;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=Test 1 - a selection over the row limit is refused,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - cell limit (200 rows x 1004 cols = 200,800 > 200,000)
+  */
+%wideboy_filter(200)
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "dctest.wideboy,&filter_rk";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:sascontroltable,
+  outref=web2
+)
+
+%let abort2=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort2',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort2=1),
+  desc=Test 2 - a selection over the cell limit is refused,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/editors/getdata.test.2.sas

@@ -0,0 +1,271 @@
+/**
+  @file
+  @brief testing that selectbox (cell dropdown) values respect Row Level Security
+
+  @details A HARDSELECT / SOFTSELECT rule whose RULE_VALUE is a
+  library.member.column reference has its distinct values read straight from
+  that source table by getdata.sas.  If the source table carries Row Level
+  Security rules, the restricted rows must be filtered out of the dropdown too -
+  otherwise values the user is not permitted to see appear in the cell dropdown
+  while the grid itself is correctly filtered.
+
+  This test makes the runner a 'regular' (non admin) user and restricts
+  SOME_BESTNUM in MPE_X_TEST to one value selected from the data (the demo
+  data holds random SOME_BESTNUM values, refreshed by testsetup, so the
+  value cannot be hardcoded).  The demo data must hold more than one
+  distinct SOME_BESTNUM - true of the SAS-seeded estate, but NOT of the
+  SASjs server mock, which seeds the seven 44 rows only (see the
+  'distinct_bestnum' precondition assertion below).
+
+  Runners whose SAS identity group is the admin group cannot be demoted by
+  editing MPE_GROUPS alone - Row Level Security is skipped for them - so the
+  test reports a skip row instead of running the assertions.
+
+  The call passes a filter (PRIMARY_KEY_FIELD < 100) because getdata.sas also
+  enforces the row / cell limits, and the unfiltered MPE_X_TEST is above the
+  row limit (642 rows against DC_MAXOBS_WEBEDIT:250) - the limits test
+  (getdata.test.1.sas) covers that abort separately.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getuser.sas
+  @li mpe_getgroups.sas
+  @li mp_assert.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_filterstore.sas
+  @li mx_execute.sas
+
+**/
+
+%let _program=&appLoc/services/editors/getdata;
+
+/**
+  * The body is wrapped in a macro so the conditional skip below uses
+  * %if/%then/%do inside a macro definition.  Open-code %if needs SAS 9.4M5+,
+  * and the suite also runs on the sas9 target, so the wrapper keeps the test
+  * portable across the supported estates (as testsetup.sas's update_settings
+  * does for its own conditionals).
+  */
+%macro getdata_rls_test();
+
+/**
+  * Row Level Security is skipped for admins, so only a non admin runner can
+  * verify it.  The group membership is checked against &dc_admin_group (the
+  * mpeinit-resolved &mpeadmins is not available in a test session, see
+  * testinit.sas) and taken from the SAS identity groups, NOT the MPE_GROUPS
+  * table this test goes on to edit.  Admin runners (eg a shared test account
+  * that is a member of the admin group) simply skip the RLS assertions.
+  */
+%mpe_getgroups(user=%mf_getuser(),outds=work.testgroups)
+proc sql noprint;
+select count(*) into: admin_check
+  from work.testgroups
+  where groupname="&dc_admin_group";
+quit;
+%if &admin_check>0 %then %do;
+  %put WARNING: %mf_getuser() is in the &dc_admin_group group;
+  %put WARNING: Row Level Security is skipped for admins - test skipped;
+  data work.test_results;
+    length test_result $4 test_description $256 test_comments $256;
+    test_result='PASS';
+    test_description="GETDATA_TEST_2 skipped - runner is an admin";
+    test_comments="RLS is skipped for admins so the assertions cannot run";
+  run;
+%end;
+%else %do;
+
+/**
+  * Make the runner a 'regular' user and grant the temp group EDIT access so
+  * the getdata access check still passes.
+  */
+data work.groups_backup;
+  set &dc_libref..MPE_GROUPS;
+  where upcase(user_name)="%upcase(%mf_getuser())";
+run;
+
+%let tempgroup=%mf_getuniquename();
+
+proc sql;
+delete from &dc_libref..MPE_GROUPS
+  where upcase(user_name)="%upcase(%mf_getuser())";
+
+insert into &dc_libref..MPE_GROUPS
+  set user_name="%mf_getuser()"
+    ,group_name="&tempgroup"
+    ,group_desc="temp group"
+    ,tx_from='01Jan1960:00:00:00'dt
+    ,tx_to='31Dec9999:23:59:59'dt;
+
+insert into &dc_libref..MPE_SECURITY
+  set tx_from=0
+    ,libref="*ALL*"
+    ,dsn="*ALL*"
+    ,access_level="EDIT"
+    ,sas_group="&tempgroup"
+    ,tx_to='31DEC9999:23:59:59'dt;
+quit;
+
+/**
+  * Row Level Security on MPE_X_TEST - only one SOME_BESTNUM value (selected
+  * from the data, see header) is visible.  The value is taken from the rows
+  * that match the user filter below, so the filtered grid cannot be empty.
+  */
+proc sql noprint;
+delete from &dc_libref..MPE_ROW_LEVEL_SECURITY
+  where RLS_TABLE="MPE_X_TEST" and rls_group="&tempgroup";
+
+select distinct some_bestnum into :rls_value trimmed
+  from &dc_libref..MPE_X_TEST
+  where some_bestnum is not null and primary_key_field < 100;
+select coalesce(max(rls_rk),0) into: max_rk
+  from &dc_libref..MPE_ROW_LEVEL_SECURITY;
+quit;
+
+proc sql;
+insert into &dc_libref..mpe_row_level_security set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,RLS_RK=&max_rk+1
+    ,RLS_SCOPE='ALL'
+    ,RLS_GROUP="&tempgroup"
+    ,RLS_LIBREF="&dc_libref"
+    ,RLS_TABLE="MPE_X_TEST"
+    ,RLS_GROUP_LOGIC='AND'
+    ,RLS_SUBGROUP_LOGIC='OR'
+    ,RLS_SUBGROUP_ID=0
+    ,RLS_VARIABLE_NM='SOME_BESTNUM'
+    ,RLS_OPERATOR_NM='IN'
+    ,RLS_RAW_VALUE="(&rls_value)"
+    ,RLS_ACTIVE=1;
+quit;
+
+/**
+  * Preconditions - the source column really does hold more than one distinct
+  * value (so the assertions are meaningful) and the user filter matches at
+  * least one of them (so the grid cannot be empty).
+  */
+proc sql noprint;
+select count(distinct some_bestnum) into: distinct_bestnum
+  from &dc_libref..MPE_X_TEST;
+quit;
+%mp_assert(
+  iftrue=(&distinct_bestnum>1),
+  desc=Precondition - MPE_X_TEST.SOME_BESTNUM has more than one distinct value,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%length(&rls_value)>0),
+  desc=Precondition - the user filter matches at least one SOME_BESTNUM value,
+  outds=work.test_results
+)
+
+/**
+  * Store a filter (PRIMARY_KEY_FIELD < 100) so the selection stays under the
+  * DC_MAXOBS_WEBEDIT row limit.
+  */
+data work.inquery;
+  length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8
+    VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
+  GROUP_LOGIC='AND';
+  SUBGROUP_LOGIC='AND';
+  SUBGROUP_ID=1;
+  VARIABLE_NM='PRIMARY_KEY_FIELD';
+  OPERATOR_NM='<';
+  RAW_VALUE='100';
+run;
+%mp_filterstore(
+  libds=&dclib..MPE_X_TEST,
+  filter_summary=&dc_libref..mpe_filteranytable,
+  filter_detail=&dc_libref..mpe_filtersource,
+  lock_table=&dc_libref..mpe_lockanytable,
+  maxkeytable=&dc_libref..mpe_maxkeyvalues,
+  queryds=work.inquery,
+  outresult=work.filter_result,
+  outquery=work.filter_query
+)
+data _null_;
+  set work.filter_result;
+  call symputx('f1_rk',filter_rk,'l');
+run;
+
+/**
+  * Call the getdata service - filtered to keep the row count under
+  * DC_MAXOBS_WEBEDIT (see header).
+  */
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "&dclib..MPE_X_TEST,&f1_rk";
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:sascontroltable,
+  outlib=web1,
+  mdebug=&sasjs_mdebug
+)
+
+data work.dqdata;
+  set web1.dqdata;
+run;
+data work.sasdata;
+  set web1.sasdata;
+run;
+
+/* the selectbox source rows only (exclude MPE_SELECTBOX-derived values) */
+data work.dqtest;
+  set work.dqdata;
+  where upcase(base_col)='SOME_BESTNUM';
+run;
+
+data work.checkchar;
+  val="&rls_value"; output;
+run;
+data work.checknum;
+  val=&rls_value; output;
+run;
+
+/**
+  * Test 1 - control: the grid itself is filtered by RLS.  If this fails the
+  * runner is an admin and the dropdown assertion below cannot be trusted.
+  */
+%mp_assertcolvals(work.sasdata.SOME_BESTNUM,
+  checkvals=work.checknum.val,
+  desc=Test 1 - grid rows respect Row Level Security (control),
+  test=ALLVALS,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - the selectbox values sourced from MPE_X_TEST.SOME_BESTNUM must
+  * also be restricted to the RLS-visible value (44).
+  */
+%mp_assertdsobs(work.dqtest,
+  desc=Test 2 precondition - selectbox values were returned for SOME_BESTNUM,
+  test=HASOBS,
+  outds=work.test_results
+)
+%mp_assertcolvals(work.dqtest.RULE_DATA,
+  checkvals=work.checkchar.val,
+  desc=Test 2 - selectbox values respect Row Level Security,
+  test=ALLVALS,
+  outds=work.test_results
+)
+
+/**
+  * Cleanup - restore the group / security / RLS rows
+  */
+proc sql;
+delete from &dc_libref..MPE_ROW_LEVEL_SECURITY where rls_group="&tempgroup";
+delete from &dc_libref..MPE_SECURITY where sas_group="&tempgroup";
+delete from &dc_libref..MPE_GROUPS where group_name="&tempgroup";
+quit;
+proc append base=&dc_libref..MPE_GROUPS data=work.groups_backup;
+run;
+%end;
+
+%mend getdata_rls_test;
+
+%getdata_rls_test()

sas/sasjs/services/editors/getdata.test.sas

@@ -17,13 +17,43 @@
 
 /**
   * Test 1 - basic fetch
+
+  * The whole of MPE_TABLES is over the EDIT row limit on a populated estate,
+  * so fetch a single library's rows - the filter is what keeps the selection
+  * under the limit.  The assertion is simply that data still comes back.
   */
 
+data work.inquery1;
+  length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8
+    VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
+  GROUP_LOGIC='AND';
+  SUBGROUP_LOGIC='AND';
+  SUBGROUP_ID=1;
+  VARIABLE_NM='LIBREF';
+  OPERATOR_NM='=';
+  RAW_VALUE="'&dclib'";
+  output;
+run;
+%mp_filterstore(
+  libds=&dclib..MPE_TABLES,
+  filter_summary=&dc_libref..mpe_filteranytable,
+  filter_detail=&dc_libref..mpe_filtersource,
+  lock_table=&dc_libref..mpe_lockanytable,
+  maxkeytable=&dc_libref..mpe_maxkeyvalues,
+  queryds=work.inquery1,
+  outresult=work.result1,
+  outquery=work.query1
+)
+data _null_;
+  set work.result1;
+  call symputx('filter_rk1',filter_rk,'g');
+run;
+
 %let f1=%mf_getuniquefileref();
 data _null_;
   file &f1 termstr=crlf;
-  put 'LIBDS:$43.';
-  put "&dclib..MPE_TABLES";
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "&dclib..MPE_TABLES,&filter_rk1";
 run;
 %mx_execute(&_program,
   viyacontext=&defaultcontext,
@@ -58,9 +88,9 @@ data query;
 run;
 
 %mp_assertcols(work.query,
-  desc=Query dataset is empty (test cols not obs as table has one row),
+  desc=Query dataset carries the filter columns,
   cols=VARIABLE_NM RAW_VALUE SUBGROUP_ID,
-  test=NONE,
+  test=ALL,
   outds=work.test_results
 )
 %mp_assertdsobs(work.sasdata,

sas/sasjs/services/public/getcols.sas

@@ -6,6 +6,7 @@
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
   @li mp_abort.sas
+  @li mp_getcols.sas
   @li mp_validatecol.sas
 
   @version 9.2
@@ -43,32 +44,26 @@ run;
 
 %dc_assignlib(READ,%scan(&ds,1,.))
 
-proc contents noprint data=&ds
-  out=droplist1 (keep=name type length label varnum format:);
-run;
-data cols(keep=name type length varnum format label);
-  set droplist1(rename=(format=format2 type=type2));
-  name=upcase(name);
-  if type2=2 then do;
-    length format $49.;
-    if format2='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else format=cats(format2,formatl,'.');
-    type='C';
-    ddtype='C';
-  end;
-  else do;
-    if format2='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
-    type='N';
-    if format=:'DATETIME' then ddtype='DATETIME';
-    else if format=:'DATE' then ddtype='DATE';
-    else if format=:'TIME' then ddtype='TIME';
-    else ddtype='N';
-  end;
-  if label='' then label=name;
+/* Column metadata, typed the way the editor's own load types it: %mp_getcols
+  infers DATE and DATETIME across the whole format list (YYMMDD, MMDDYY,
+  E8601DA, B8601DA, NLDATE, NLDATM, E8601DT, ...), which a test against the
+  format prefix does not - a column formatted YYMMDD would otherwise reach the
+  picker as numeric.  getdata.sas makes the same call for the same reason. */
+%mp_getcols(&ds, outds=cols1)
+
+/* Response contract.  `type` keeps the SAS vocabulary this public service has
+  always used (C/N), and `format` and `label` are carried over from the
+  previous shape, so an existing caller still finds them unchanged.  `fmtname`
+  (the bare format name) and `ddtype` (which now distinguishes DATE, DATETIME
+  and TIME) are additions.  The client derives its own num/char vocabulary from
+  ddtype at the store boundary - see sas-store.getCols. */
+data cols(keep=name type length varnum fmtname format label ddtype);
+  set cols1;
+  /* DDTYPE in the client's vocabulary, as getdata.sas maps it.  ddtype needs
+    no length statement - the macro creates it with 'CHARACTER', so it is
+    already $9. */
+  if ddtype='CHARACTER' then ddtype='C';
+  else if ddtype='NUMERIC' then ddtype='N';
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)

sas/sasjs/services/public/getcols.test.sas

@@ -66,8 +66,20 @@ run;
 )
 
 %let nobs=0;
+%let nbad=0;
 proc sql noprint;
 select count(*) into: nobs from web2.cols;
+/**
+  * The service keeps the SAS vocabulary this public contract has always used:
+  * TYPE is C/N, and DDTYPE separates the date and time types.  The client maps
+  * TYPE to its own num/char vocabulary in sas-store.getCols, so a change here
+  * that silently switched vocabulary would not be caught by the client suite.
+  * Every column must be C or N, and the two must agree with ddtype.
+  */
+select count(*) into: nbad from web2.cols
+  where type not in ('C','N')
+  or (ddtype='C' and type ne 'C')
+  or (ddtype in ('N','DATE','DATETIME','TIME') and type ne 'N');
 quit;
 
 %mp_assert(
@@ -76,6 +88,12 @@ quit;
   outds=work.test_results
 )
 
+%mp_assert(
+  iftrue=(&nbad=0),
+  desc=%str(TYPE stays in the SAS C/N vocabulary and agrees with ddtype),
+  outds=work.test_results
+)
+
 /**
   * dump results to the log for offline inspection
   */

sas/sasjs/services/public/viewdata.sas

@@ -181,7 +181,7 @@ run;
   %put NOTE- Please add to &mpelib..MPE_CONFIG table;
   %put NOTE-;%put NOTE-;
   %global DC_MAXOBS_WEBVIEW;
-  %let DC_MAXOBS_WEBVIEW=500;
+  %let DC_MAXOBS_WEBVIEW=2000;
 %end;
 %if &existds>0 %then %do;