test(getcols): assert the TYPE vocabulary in the SAS suite

Review feedback on this PR.  The cypress cases exercise the mock, which derives
TYPE in JS and never passes through the $1 variable the length statement
widens, so dropping `length type $4;` again would pass everything CI runs.

getcols.test.sas already runs the service end to end, so its second case now
also counts the rows whose TYPE does not match their DDTYPE - 'char' for C,
'num' for N/DATE/DATETIME/TIME.  That covers the vocabulary and the mapping it
is derived from, so a truncated 'n'/'c' and a swapped mapping both fail it.
The suite runs via `sasjs test` against a Viya target, not in CI.

sasjs lint reports no warning on the file.

.gitea/scripts/verify-section.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Emits the "Verifying the download" section appended to every release body
+# by the "Upload assets to release" step of release.yaml.
+# Keep this text in sync with the docs page:
+#   docs.datacontroller.io/docs/downloads.md
+set -euo pipefail
+cat <<'EOF'
+
+## Verifying the download
+
+Every asset on this release is covered by the `SHA256SUMS` file. To verify a download, fetch `SHA256SUMS` from the release assets alongside the files you downloaded, then run:
+
+```
+sha256sum --check SHA256SUMS --ignore-missing
+```
+
+The command reports `OK` for each asset that matches. `--ignore-missing` lets you check just the files you downloaded rather than all of them.
+
+What this protects: a matching checksum confirms the asset is byte-for-byte the file this pipeline uploaded, guarding against corrupted or tampered downloads (mirrors, proxies, interrupted transfers). It is not a signature: the hashes travel in the same release as the files, so a compromise of the forge itself could alter both. Treat it as an integrity check, not a trust anchor.
+EOF

.gitea/workflows/build.yaml

@@ -15,6 +15,22 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
+      - name: Scan for secrets (gitleaks)
+        # The local pre-commit hook is suppressed entirely when .npmrc sets
+        # ignore-scripts=true (the prepare script that would activate the
+        # hooks never runs), so CI is the enforcement point: a leaked
+        # credential must fail the build, not just a commit.
+        # Uses the same @nogoo9/gitleaks binary the pre-commit hook pins - the
+        # gitleaks-action requires a license key for organization repos.
+        # Runs before client/.npmrc is written below, because that file is
+        # created from the NPMRC secret and a later scan would flag the
+        # registry token this job injects itself.
+        # --no-git scans the working tree, which is what a pull request gate
+        # is about.
+        run: |
+          npm ci
+          node_modules/.bin/gitleaks detect --no-git --source . --redact --no-banner -v
+
       - name: Install Google Chrome
         run: |
           apt-get update
@@ -107,6 +123,15 @@ jobs:
           echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
           npm ci
 
+      - name: Build frontend for web streaming
+        # The server-ci target streams client/dist as the 'clickme' web app (see
+        # CONTRIBUTING.md), so the production build has to exist before the mock
+        # services are deployed. Without it the deploy logs
+        # "webSourcePath: .../client/dist ... doesn't exist" and streams no frontend.
+        run: |
+          cd client
+          npm run build
+
       - name: Setup and start SASjs server
         run: |
           npm i -g pm2
@@ -117,8 +142,27 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          # The server refuses to run as root unless RUN_AS names an account
+          # (a security change in the recent releases). This job runs as root
+          # in a throwaway container, so opt in explicitly.
+          echo RUN_AS=root >> .env
           cat .env
           pm2 start api-linux --wait-ready
+          # `pm2 start --wait-ready` can return while the app is still
+          # launching (it does not always wait for the ready signal), and the
+          # next step hits :5000 immediately. Wait for the server to answer,
+          # and dump its log if it never does.
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5000/; then
+              echo "SASjs server answered on attempt ${i}"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "SASjs server did not come up within 120s"
+          pm2 list
+          pm2 logs api-linux --lines 100 --nostream
+          exit 1
 
       - name: Deploy mocked services
         run: |
@@ -154,7 +198,7 @@ jobs:
           # Start frontend and run cypress
           # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

.gitea/workflows/lighthouse.yaml

@@ -33,10 +33,29 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          # The server refuses to run as root unless RUN_AS names an account
+          # (a security change in the recent releases). This job runs as root
+          # in a throwaway container, so opt in explicitly.
+          echo RUN_AS=root >> .env
           cat .env
           curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > linux.zip
           unzip linux.zip
           pm2 start api-linux --wait-ready
+          # `pm2 start --wait-ready` can return while the app is still
+          # launching (it does not always wait for the ready signal), and the
+          # next step hits :5000 immediately. Wait for the server to answer,
+          # and dump its log if it never does.
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5000/; then
+              echo "SASjs server answered on attempt ${i}"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "SASjs server did not come up within 120s"
+          pm2 list
+          pm2 logs api-linux --lines 100 --nostream
+          exit 1
 
       - name: Write .npmrc file
         run: echo "$NPMRC" > client/.npmrc

.gitea/workflows/release.yaml

@@ -117,8 +117,27 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          # The server refuses to run as root unless RUN_AS names an account
+          # (a security change in the recent releases). This job runs as root
+          # in a throwaway container, so opt in explicitly.
+          echo RUN_AS=root >> .env
           cat .env
           pm2 start api-linux --wait-ready
+          # `pm2 start --wait-ready` can return while the app is still
+          # launching (it does not always wait for the ready signal), and the
+          # next step hits :5000 immediately. Wait for the server to answer,
+          # and dump its log if it never does.
+          for i in $(seq 1 60); do
+            if curl -sf -o /dev/null http://localhost:5000/; then
+              echo "SASjs server answered on attempt ${i}"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "SASjs server did not come up within 120s"
+          pm2 list
+          pm2 logs api-linux --lines 100 --nostream
+          exit 1
 
       - name: Deploy mocked services
         run: |
@@ -302,8 +321,10 @@ jobs:
         run: |
           cd sas
           cp sasjsbuild/viya.json ../client/dist/viya.json
-          cd ..
-          zip -r frontend.zip ./client/dist
+          # Zip from *inside* dist so the archive holds the frontend files at its
+          # root rather than under a client/dist/ prefix (see #147).
+          cd ../client/dist
+          zip -r ../../frontend.zip .
 
       - name: Release Typedoc
         run: |
@@ -336,23 +357,43 @@ jobs:
           RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
           RELEASE_BODY=$(echo "$RELEASE_JSON" | jq -r '.body')
 
-          # Update body (also confirms the token has contents:write on this repo)
+          # Generate SHA-256 checksums over the release assets, so downloads can
+          # be verified. The file uses basename-only paths so that a plain
+          # "sha256sum -c SHA256SUMS" works in the folder the assets were
+          # downloaded to (the upload below stores them under their basename).
+          ASSETS=(frontend.zip
+                  sas/demostream_sas9.sas
+                  sas/viya.sas
+                  sas/sasjs_server.json.zip
+                  sas/sas9.sas
+                  sas/viya_noweb.sas
+                  sas/viya_noweb.json)
+
+          : > SHA256SUMS
+          for f in "${ASSETS[@]}"; do
+            sum=$(sha256sum "$f" | cut -d' ' -f1)
+            printf '%s  %s\n' "$sum" "${f##*/}" >> SHA256SUMS
+          done
+          cat SHA256SUMS
+
+          # Update body: keep the existing notes, add a verification section
+          # ahead of the installation footer. The section text lives in a
+          # dedicated script below (.gitea/scripts/verify-section.sh) so this
+          # YAML scalar stays clean.
+          VERIFY_SECTION=$(./.gitea/scripts/verify-section.sh)
+          NEW_BODY=$(jq -n --arg body "$RELEASE_BODY" --arg verify "$VERIFY_SECTION" \
+            '$body + "\n" + $verify + "\n\nFor installation instructions, please visit https://docs.datacontroller.io/"')
           curl -k --fail-with-body -sS -X PATCH \
             -H "$AUTH_HEADER" \
             -H 'Content-Type: application/json' \
-            --data "$(jq -n --arg body "$RELEASE_BODY"$'\n\nFor installation instructions, please visit https://docs.datacontroller.io/' \
-                        '{draft:false, body:$body}')" \
+            --data "$(jq -n --arg body "$NEW_BODY" '{draft:false, body:$body}')" \
             "$BASE/releases/$RELEASE_ID"
 
           # Upload assets
           URL="$BASE/releases/$RELEASE_ID/assets"
-          for f in frontend.zip \
-                   sas/demostream_sas9.sas \
-                   sas/viya.sas \
-                   sas/sasjs_server.json.zip \
-                   sas/sas9.sas \
-                   sas/viya_noweb.sas \
-                   sas/viya_noweb.json; do
+          for f in "${ASSETS[@]}"; do
             echo "Uploading $f ..."
             curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@$f"
           done
+          echo "Uploading SHA256SUMS ..."
+          curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@SHA256SUMS"

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+# Hooks for the pre-commit framework - https://pre-commit.com
+#
+# The repo also ships native hooks in .git-hooks/, covering the secret scan, a
+# 2MB commit size limit, a prettier check and Conventional Commits. But .npmrc
+# sets ignore-scripts=true, so the `prepare` script that points git at them
+# never runs on `npm i` - a fresh clone commits unchecked until the one-time
+# command in CONTRIBUTING.md is run by hand.
+#
+# This file is the standard route instead: install the framework once with
+# `pre-commit install`, and the secret scan runs on every commit. The hook is
+# pinned to an exact gitleaks release tag.
+repos:
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.30.1
+    hooks:
+      - id: gitleaks

CHANGELOG.md

@@ -1,3 +1,56 @@
+# [7.16.0](https://git.datacontroller.io/dc/dc/compare/v7.15.0...v7.16.0) (2026-09-28)
+
+
+### Bug Fixes
+
+* **ci:** correct server-ci webSourcePath and build frontend before mock deploy ([a4ad4cb](https://git.datacontroller.io/dc/dc/commit/a4ad4cba2861fe76f7ee5ea72ccf8c6e61c888a5))
+* **client:** parse iOS browsers in parseUserAgent ([eec3438](https://git.datacontroller.io/dc/dc/commit/eec3438de9735e0feb74c95d0b28128af6677477))
+* **diagnostics:** give the startup service a body for browser_info ([5538e0c](https://git.datacontroller.io/dc/dc/commit/5538e0c5752553a7c61903c62e4fdc82b6206826))
+* **diagnostics:** treat _debug=128 as debug on ([8de9978](https://git.datacontroller.io/dc/dc/commit/8de99782134e37c5e9c26c3117b083f49fa65189))
+* **loader:** abort when a post edit hook routes to an unregistered table ([171974b](https://git.datacontroller.io/dc/dc/commit/171974bd412be3efafd792e6c61c23f2fb442ace))
+* **mocks:** keep the mock hook reader inside the Drive ([7fd8ae4](https://git.datacontroller.io/dc/dc/commit/7fd8ae462993f0a1a8a034b5938d309d63920043))
+* **sas:** drop the duplicated abort in the target re-registration check ([48d45ac](https://git.datacontroller.io/dc/dc/commit/48d45ac7f8c32e39ff2d0e1ceac50252a49ebadd))
+* **sidebar:** drop the query string from the sub-page label ([d2dd46b](https://git.datacontroller.io/dc/dc/commit/d2dd46bfdfddd1695d1b3915d20ce79330cf275b))
+
+
+### Features
+
+* **client:** report the browser and both versions in browser_info ([a770f36](https://git.datacontroller.io/dc/dc/commit/a770f3695ceac441f48527f606843602c39fcc13))
+* **client:** send session_info with every service request ([50282f2](https://git.datacontroller.io/dc/dc/commit/50282f2ddb1250e048c55c04b395b428167142f7))
+* **filters:** make the applied-filter panel expandable ([fdaa249](https://git.datacontroller.io/dc/dc/commit/fdaa249f447b3a2c87da4b2aa1e5978c09f53ff8))
+* **mocks:** emulate the %mpeinit diagnostics dump ([200ee89](https://git.datacontroller.io/dc/dc/commit/200ee8931d3952624434567b5b95c9e7e53c6508))
+* **mocks:** run pre/post edit hook programs in the JS mock services ([b5f2283](https://git.datacontroller.io/dc/dc/commit/b5f228351bca18cbc8a3032e60846d05192668f6))
+* **mpeinit:** dump session_info to the log when debug is on ([fbd77a5](https://git.datacontroller.io/dc/dc/commit/fbd77a533417b1c8549a339b94785f7af0a30ca8))
+* **release:** publish SHA256SUMS and verify instructions with each release ([f1ae501](https://git.datacontroller.io/dc/dc/commit/f1ae501d49b510a1870f3682bb7d83de7446e061))
+
+# [7.15.0](https://git.datacontroller.io/dc/dc/compare/v7.14.2...v7.15.0) (2026-09-23)
+
+
+### Bug Fixes
+
+* **mocks:** keep special missings in the DIFF, and add the clip recording spec ([d124ce3](https://git.datacontroller.io/dc/dc/commit/d124ce3b36f8f5153e8d7a576d4bec3189524ed5))
+* **mocks:** list a column's own special missing in its dropdown ([b8f16a6](https://git.datacontroller.io/dc/dc/commit/b8f16a63827310f68629f2e016a4fb876593c37c))
+* **mocks:** make the approval path work, and record the approval scene ([12847cf](https://git.datacontroller.io/dc/dc/commit/12847cf07165d5aefa61592815cab0a37245e31f))
+* **validator:** a range rule ignores a missing value ([70dae4b](https://git.datacontroller.io/dc/dc/commit/70dae4b7013ac825eb77d3dc2b622980c0e4af05))
+* **validator:** keep the regular missing in a numeric dropdown source too ([a639bca](https://git.datacontroller.io/dc/dc/commit/a639bca70264e8a6a81795892b9ed63ba58d5f73))
+* **validator:** primary keys are NOT NULL, and a strict dropdown can match a special missing ([586a41a](https://git.datacontroller.io/dc/dc/commit/586a41ad49670ee795284a83cf8535711ce14d2c))
+* **validator:** reject a special missing on a NOT NULL column ([6d85bce](https://git.datacontroller.io/dc/dc/commit/6d85bce2a0c528e8eab9ee4abbade8c682ec3a1b))
+
+
+### Features
+
+* **validator:** compare MINVAL and MAXVAL in SAS's own order ([3467322](https://git.datacontroller.io/dc/dc/commit/3467322e99b0c3f6fda5654d35b507fdb2cb1c22))
+
+## [7.14.2](https://git.datacontroller.io/dc/dc/compare/v7.14.1...v7.14.2) (2026-09-22)
+
+
+### Bug Fixes
+
+* **release:** put frontend files at the root of frontend.zip ([7771a24](https://git.datacontroller.io/dc/dc/commit/7771a24b9ce55751253686885e4848a292817a38)), closes [#147](https://git.datacontroller.io/dc/dc/issues/147)
+* **security:** accept the format-catalog form when validating a libds ([0fa5da8](https://git.datacontroller.io/dc/dc/commit/0fa5da8abfc303ce3b18beb4c01484dcce508439))
+* **security:** escape col-info dropdown and origin-check VA replay ([c9eed6d](https://git.datacontroller.io/dc/dc/commit/c9eed6dae717414b874ef6f3d60312ea0bb5a0da))
+* validate request inputs and add admin gates to public services ([be86000](https://git.datacontroller.io/dc/dc/commit/be86000fe0f0858edcbecf71d426a2ff81257247))
+
 ## [7.14.1](https://git.datacontroller.io/dc/dc/compare/v7.14.0...v7.14.1) (2026-09-17)
 
 

CONTRIBUTING.md

@@ -19,6 +19,8 @@ git config core.hooksPath ./.git-hooks
 
 The pre-commit hook requires the gitleaks binary provided by the root `@nogoo9/gitleaks` devDependency, so make sure `npm i` has run in the repo root before your first commit.
 
+The standard [pre-commit](https://pre-commit.com) route is available as well, and does not depend on the lifecycle scripts above: install it once with `pre-commit install` and the secret scan in [`.pre-commit-config.yaml`](./.pre-commit-config.yaml) runs on every commit. The same scan also runs in CI, so a leaked credential fails the build regardless of which local hooks are active.
+
 ## Dependencies that requires licences
 
 [SheetJS Pro Version](https://www.npmjs.com/package/sheetjs)

client/cypress/clips/special-missings-clip.cy.ts

@@ -0,0 +1,592 @@
+// Clip script: special missings inside Data Controller's validation rules.
+//
+// This is not a test - it is the recording script for the companion clip, and
+// it deliberately pauses between steps so each beat is readable on video. It
+// lives in cypress/clips (outside cypress/e2e) so the default spec pattern does
+// not pick it up in CI.
+//
+// Record it with:
+//
+//   npx cypress run --spec cypress/clips/special-missings-clip.cy.ts \
+//     --config video=true,viewportWidth=1280,viewportHeight=720
+//
+// The table is TESTDATA.DEMO_01 (seven columns, one rule each):
+//   ID (NOTNULL), AMOUNT (MINVAL 1), SCORE (MAXVAL 100),
+//   REF (SOFTREGEX /^[0-9]+$/), STATUS (SOFTSELECT TESTDATA.DEMO_01.STATUS),
+//   RATING (no rule), LAST_REVIEWED (date9.)
+//
+// One editing session, one submission. The rule scenes are played on row 1 and
+// each value is put back afterwards, so the DIFF that follows carries only the
+// two changes the clip is about; the review screens are reached through the
+// app's own navigation rather than cy.visit, so nothing reloads mid-clip.
+//
+// The beats are deliberately short - the take is played back at ~1.4x on the
+// way out, so a hold that looks tight here reads as a normal pause on the
+// finished clip.
+
+// Caption track: each mark opens a caption and carries the text it shows.
+// The encoder turns consecutive marks into subtitle cues, so the captions are
+// timed by the recording itself rather than by guessed offsets.
+const BEATS = '/tmp/clip-beats.tsv'
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+// A beat long enough to read on video.
+const beat = (ms = 1500) => cy.wait(ms)
+
+context('special missings clip (DEMO_01)', function () {
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    // The mock estate carries a valid licence key (mock-storage/licence.json),
+    // so the app activates directly with no free-tier banner in shot.
+    cy.get('.nav-tree', { timeout: longerCommandTimeout }).should('exist')
+  })
+
+  it('records the demo in one editing session', () => {
+    cy.writeFile(BEATS, '', { flag: 'w' })
+
+    // ---- Scene 1: the table and its rules ----------------------------------
+    openTableFromTree('testdata', 'demo_01')
+    beat(1200)
+
+    clickOnEdit(() => {
+      // Seven columns, five rules: ID is the key, AMOUNT the floor, SCORE the
+      // ceiling, REF the pattern, STATUS the dropdown, and RATING and
+      // LAST_REVIEWED carry nothing.
+      cy.get('.ht_master tbody tr', { timeout: longerCommandTimeout }).should(
+        'have.length.greaterThan',
+        3
+      )
+      mark(
+        'grid',
+        'TESTDATA.DEMO_01 - eight columns, six rules. ID is the key, and RATING and LAST_REVIEWED carry none.'
+      )
+      beat(2600)
+
+      // ---- Scene 2: typing a special missing ------------------------------
+      // RATING has no rule, so this shows entry on its own. Every beat asserts
+      // the settled cell state before holding, so the recording always rests on
+      // the outcome (flagged or accepted) rather than on a blind pause that
+      // might land before the rule engine has run.
+      mark(
+        'entry',
+        'A numeric cell takes a special missing as a letter or an underscore, with or without a period - row 2 already holds .a.'
+      )
+      typeAndHold(0, 'RATING', 'a', 'accepted') // a single letter is taken
+      mark(
+        'reject',
+        'Two letters, or a letter mixed with a number, are refused. Red means it will not submit.'
+      )
+      typeAndHold(0, 'RATING', 'AB', 'rejected', 2200) // two letters are not
+      typeAndHold(0, 'RATING', '1a', 'rejected', 2200) // nor a number and a letter
+      typeAndHold(0, 'RATING', 'a', 'accepted') // leave it as a special missing
+
+      // ---- Scene 3: NOTNULL refuses both --------------------------------
+      // ID is the key, so NOTNULL applies. A special missing is not a value
+      // here: like a blank, it fails the rule (a real SAS NOT NULL constraint
+      // rejects a special missing as well), and only a number satisfies it -
+      // the original key, put back so the row carries no change into the DIFF.
+      mark(
+        'notnull_blank',
+        "ID is the table's primary key, so NOT NULL is applied to it automatically - a blank fails..."
+      )
+      typeAndHold(0, 'ID', '', 'rejected', 2200) // a blank fails NOTNULL
+      mark(
+        'notnull_missing',
+        '...and so does a special missing: a missing is not a value to NOT NULL.'
+      )
+      typeAndHold(0, 'ID', 'A', 'rejected', 2200) // so does a special missing
+      mark(
+        'notnull_number',
+        'A number satisfies it, and the row key goes back.'
+      )
+      typeAndHold(0, 'ID', '1', 'accepted') // a number is what it wants
+
+      // ---- Scene 4: the pattern still applies -----------------------------
+      // REF carries SOFTREGEX /^[0-9]+$/, and a special missing is not exempt
+      // from it. The amber cell is a soft rule warning rather than a block,
+      // which the caption on this beat says out loud - the pattern itself is
+      // only visible in the cell's native title, which a screencast does not
+      // capture.
+      mark(
+        'softregex',
+        'REF carries SOFTREGEX /^[0-9]+$/. Amber is a soft warning - it warns, it does not block.'
+      )
+      typeIntoCell(0, 'REF', 'A')
+      getCellByHeaderAndRow(0, 'REF').should('have.class', 'dc-warning-cell')
+      beat(3400)
+      typeIntoCell(0, 'REF', '1001') // put the reference back
+      getCellByHeaderAndRow(0, 'REF').should(
+        'not.have.class',
+        'dc-warning-cell'
+      )
+      beat(600)
+
+      // ---- Scene 5: the dropdown lists the missing -------------------------
+      // STATUS carries a SOFTSELECT whose list is taken from the column itself
+      // (the library.member.column form), and that column holds a special
+      // missing - so the dropdown offers it alongside the ordinary values, as
+      // the bare letter SAS produces for it.
+      mark(
+        'dropdown',
+        "STATUS carries a SOFTSELECT, and its list is the column's own values."
+      )
+      openDropdown(0, 'STATUS')
+        .should('have.length', 4)
+        .then(($items: any) => {
+          const texts = [...$items].map((td: any) => td.innerText.trim())
+          expect(texts).to.include('A')
+        })
+      beat(2400)
+
+      mark(
+        'dropdown_pick',
+        'So the special missing the column holds is offered as a bare letter, first - a missing sorts below every number.'
+      )
+      pickFromDropdown('A')
+      getCellByHeaderAndRow(0, 'STATUS').should('contain.text', 'A')
+      beat(1800)
+      typeIntoCell(0, 'STATUS', '1') // put the status back
+      beat(600)
+
+      // ---- Scene 6: a range rule compares in SAS order, missings included
+      mark(
+        'minval',
+        'AMOUNT has MINVAL 1. A missing sorts below every number, so it is below the floor.'
+      )
+      typeAndHold(0, 'AMOUNT', 'A', 'rejected', 2200) // a missing is below the floor
+      mark(
+        'maxval',
+        'SCORE has MAXVAL 100 - the same missing is below the ceiling, so it passes.'
+      )
+      typeAndHold(0, 'SCORE', 'A', 'accepted', 2200) // a missing is below the ceiling
+      mark(
+        'grade',
+        'GRADE takes MINVAL .A and MAXVAL .C. The missings have an order of their own: .B is inside the range, .D is outside it.'
+      )
+      typeAndHold(0, 'GRADE', '.B', 'accepted', 2400) // .B is between .A and .C
+      typeAndHold(0, 'GRADE', '.D', 'rejected', 2400) // .D is above .C
+
+      mark('abort', 'Submitting with an invalid cell aborts.')
+
+      // Submit while AMOUNT is invalid - the modal reports it.
+      submitTable(() => {
+        cy.get('.modal-body', { timeout: longerCommandTimeout }).should(
+          'contain.text',
+          'Invalid Values are Present'
+        )
+        beat(2400)
+        // Close the abort so the editor is clean for the next scene.
+        cy.get('clr-modal.clr-abort-modal .modal-footer button')
+          .contains('Close')
+          .click({ force: true })
+        beat(800)
+      })
+
+      // ---- Scene 7: put the test values back, then make the real change ----
+      mark(
+        'clean',
+        'Back to a clean row. Row 2 already holds .a, so this changes one special missing to another.'
+      )
+      typeIntoCell(0, 'AMOUNT', '120')
+      typeIntoCell(0, 'SCORE', '82')
+      typeIntoCell(0, 'GRADE', '.a')
+      typeIntoCell(0, 'RATING', '4')
+      beat(800)
+
+      // Row 2 (ID 2) already carries a special missing in RATING, so this is a
+      // change from one special missing to another, and the date column gives
+      // the formatted / unformatted switch something to switch.
+      typeAndHold(1, 'RATING', 'B', 'accepted')
+      typeIntoDateCell(1, 'LAST_REVIEWED', '2026-01-15')
+      beat(1500)
+
+      submitTable(() => {
+        cy.get('#submitBtn', { timeout: longerCommandTimeout })
+          .should('exist')
+          .should('not.be.disabled')
+          .click()
+        beat(2500)
+      })
+    })
+
+    // ---- Scene 8: the queue, then the DIFF (in-app navigation) ------------
+    mark('submitted', 'Submitted - the queue shows it waiting for approval.')
+    goToReviewNav()
+    beat(1800)
+
+    // The submit queue lists oldest first, so the row we just created is last.
+    cy.get('app-submitter clr-datagrid clr-dg-row', {
+      timeout: longerCommandTimeout
+    })
+      .should('exist')
+      .last()
+      .click({ force: true })
+    beat(1800)
+
+    cy.get('app-approve-details .card', { timeout: longerCommandTimeout })
+      .should('exist')
+      .should('be.visible')
+    beat(1000)
+
+    // The DIFF table is wider than the frame, so the two columns that matter
+    // (RATING and LAST_REVIEWED) sit off the right edge until it is scrolled.
+    scrollDiffToEnd()
+    beat(1200)
+
+    // Only those two cells changed: the rule scenes were put back, so the DIFF
+    // is the two changes the clip is about and nothing else.
+    getDiffCell('RATING')
+      .should('contain.text', '.b')
+      .should('have.class', 'ch')
+    getDiffCell('LAST_REVIEWED')
+      .should('contain.text', '15JAN2026')
+      .should('have.class', 'ch')
+    cy.get('app-approve-details .tableCont tbody tr td:not(.ch)').should(
+      'have.length.greaterThan',
+      3
+    )
+    mark(
+      'diff',
+      'The DIFF compares staged with base: one changed row, two changed cells.'
+    )
+    beat(2600)
+
+    // ---- Scene 9: the staged data -----------------------------------------
+    // What the approval is actually acting on: the staged row, still holding
+    // the special missing, before it reaches the base table.
+    mark(
+      'staged',
+      'The staged row, before approval - still holding the special missing.'
+    )
+    clickButton('VIEW STAGED DATA')
+    // 'Basic Submitted Details' is on the staged screen only - asserting
+    // 'Staged Data' alone would be satisfied by the button that was just
+    // clicked, which is how a beat can pass without ever leaving the DIFF.
+    cy.get('body', { timeout: longerCommandTimeout })
+      .should('contain.text', 'Basic Submitted Details')
+      .and('contain.text', 'Base Table')
+    beat(4200)
+    mark('staged_end', '')
+
+    // ---- Scene 10: the approver opens it, and switches the format ----------
+    goToReviewNav()
+    beat(1500)
+    openApproveTab()
+    beat(1500)
+
+    cy.get('app-approve clr-datagrid clr-dg-row a.color-green', {
+      timeout: longerCommandTimeout
+    })
+      .should('exist')
+      .last()
+      .click({ force: true })
+    beat(1800)
+
+    cy.get('#acceptBtn', { timeout: longerCommandTimeout })
+      .should('exist')
+      .should('not.be.disabled')
+    beat(1000)
+
+    // Same scroll as the submitter view, so the date column is in frame when
+    // the format is switched.
+    scrollDiffToEnd()
+    beat(1200)
+
+    mark(
+      'approve',
+      'The approver opens the same submission. Hovering a changed cell shows the value it replaced.'
+    )
+
+    // Hovering the two changed cells shows what each replaced - that is how the
+    // two special missings are told apart, not just the before/after of one.
+    hoverDiffCell('RATING', 'Original value is: .a')
+    beat(2800)
+
+    mark('hover_date', '...including the date it replaced.')
+    hoverDiffCell('LAST_REVIEWED', 'Original value is: 29FEB2024')
+    beat(2800)
+
+    mark(
+      'toggle',
+      'The formatted / unformatted switch shows the value as SAS stores it: 24121.'
+    )
+    cy.get('.formatted-values-toggle').should('have.text', 'Formatted').click()
+    cy.get('.formatted-values-toggle').should('have.text', 'Unformatted')
+    getDiffCell('LAST_REVIEWED').should('contain.text', '24121')
+    getDiffCell('RATING').should('contain.text', '.b')
+    beat(2800)
+
+    cy.get('.formatted-values-toggle').click()
+    cy.get('.formatted-values-toggle').should('have.text', 'Formatted')
+    getDiffCell('LAST_REVIEWED').should('contain.text', '15JAN2026')
+    beat(1000)
+
+    // ---- Scene 11: approve, and the change is in the history --------------
+    mark('accepted', 'Accepted - the history records it as APPROVED.')
+    cy.get('#acceptBtn').click()
+    cy.url({ timeout: longerCommandTimeout }).should(
+      'include',
+      '/review/history'
+    )
+    cy.get('app-history clr-datagrid clr-dg-row', {
+      timeout: longerCommandTimeout
+    })
+      .should('exist')
+      .first()
+      .should('contain.text', 'APPROVED')
+    beat(2800)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Helpers (mirrored from the e2e specs so the clip drives the same UI paths)
+// ---------------------------------------------------------------------------
+
+/**
+ * Caption track. Each mark opens a caption and carries the text it shows; the
+ * encoder turns consecutive marks into subtitle cues, so the captions are timed
+ * by the recording itself rather than by guessed offsets.
+ *
+ * The timestamp has to be taken inside a `cy.then()`: Cypress evaluates a
+ * command's arguments when the command is *queued*, so `Date.now()` passed to
+ * `cy.writeFile` directly would give every mark the same value - the moment the
+ * spec body ran.
+ */
+const mark = (label: string, text: string) => {
+  cy.then(() => {
+    cy.writeFile(BEATS, `${label}\t${text}\t${Date.now()}\n`, { flag: 'a+' })
+  })
+}
+
+const typeIntoCell = (rowIndex: number, header: string, value: string) => {
+  getCellByHeaderAndRow(rowIndex, header)
+    .dblclick({ force: true })
+    .then(() => {
+      cy.focused().clear().type(`${value}{enter}`)
+    })
+}
+
+/**
+ * A date-formatted column edits through an HTML date input, where cy.type()
+ * refuses anything but a bare YYYY-MM-DD string (so no {enter} in the same
+ * call). Set the value through the DOM and commit it with the Enter keydown
+ * that Handsontable listens for.
+ */
+const typeIntoDateCell = (rowIndex: number, header: string, value: string) => {
+  getCellByHeaderAndRow(rowIndex, header)
+    .dblclick({ force: true })
+    .then(() => {
+      cy.focused()
+        .then(($i: any) => {
+          const el = $i[0]
+          el.value = value
+          el.dispatchEvent(new Event('input', { bubbles: true }))
+          el.dispatchEvent(new Event('change', { bubbles: true }))
+        })
+        .trigger('keydown', { key: 'Enter', keyCode: 13, which: 13 })
+    })
+}
+
+/**
+ * Opens the selectbox on a cell and returns its list entries. Handsontable
+ * renders the arrow itself (`.htAutocompleteArrow`); the list is a
+ * `.handsontable.listbox` in the app document.
+ */
+const openDropdown = (rowIndex: number, header: string) => {
+  getCellByHeaderAndRow(rowIndex, header).within(() => {
+    cy.get('.htAutocompleteArrow').click({ force: true })
+  })
+
+  return cy.get('.handsontable.listbox td', { timeout: longerCommandTimeout })
+}
+
+/** Picks an entry from the open selectbox. */
+const pickFromDropdown = (value: string) => {
+  cy.get('.handsontable.listbox td').contains(value).click({ force: true })
+}
+
+/** Reaches the review area through the app's own navigation (no reload). */
+const goToReviewNav = () => {
+  cy.get('.nav-link', { timeout: longerCommandTimeout })
+    .contains('REVIEW')
+    .click({ force: true })
+}
+
+/** Opens the APPROVE tab within the review area. */
+const openApproveTab = () => {
+  cy.get('.nav-link', { timeout: longerCommandTimeout })
+    .contains('APPROVE')
+    .click({ force: true })
+}
+
+/**
+ * Clicks a button by its visible text. The match is case-insensitive: the app
+ * uppercases button labels in CSS, so the rendered text and the DOM's
+ * textContent differ.
+ */
+const clickButton = (text: string) => {
+  cy.contains('button', new RegExp(text, 'i'), {
+    timeout: longerCommandTimeout
+  }).click({ force: true })
+}
+
+/**
+ * The DIFF table is wider than the recording frame, so scroll its container to
+ * the end - that is what puts the changed RATING and the date column on screen.
+ */
+const scrollDiffToEnd = () => {
+  cy.get('app-approve-details .tableCont', { timeout: longerCommandTimeout })
+    .should('exist')
+    .scrollTo('right', { duration: 1200 })
+}
+
+/**
+ * The DIFF on the review screen is a plain HTML table (`.tableCont`), not
+ * Handsontable: headers are `th`, cells `td`, and a changed cell also carries
+ * the `ch` class whose tooltip holds the value it replaced.
+ */
+const getDiffCell = (headerText: string) => {
+  return cy
+    .get('app-approve-details .tableCont thead tr th', {
+      timeout: longerCommandTimeout
+    })
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => th.innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => th.innerText.trim() === headerText
+      )
+
+      return cy
+        .get('app-approve-details .tableCont tbody tr')
+        .first()
+        .then(($tr: any) => $tr[0].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}
+
+/**
+ * Reveals the value a changed DIFF cell replaced, and waits until it is really
+ * on screen.
+ *
+ * Clarity shows the tooltip through CSS :hover, which a synthetic
+ * `trigger('mouseover')` does NOT activate - the tooltip stays
+ * `visibility: hidden`, and a `contain.text` assertion still passes because the
+ * text is in the DOM. So this moves the real mouse (cypress-real-events) and
+ * then asserts the computed style. The first real move after another action can
+ * be swallowed, hence the repeat; if the tooltip ever fails to appear the
+ * recording fails rather than quietly showing nothing.
+ */
+const hoverDiffCell = (headerText: string, expected: string) => {
+  getDiffCell(headerText).realHover()
+  beat(300)
+  getDiffCell(headerText).realHover()
+
+  getDiffCell(headerText)
+    .find('.tooltip-content')
+    .should(($t) => {
+      const style = getComputedStyle($t[0] as HTMLElement)
+      expect(style.visibility, 'tooltip visibility').to.eq('visible')
+      expect(Number(style.opacity), 'tooltip opacity').to.be.greaterThan(0)
+      expect($t[0].textContent || '', 'tooltip text').to.contain(expected)
+    })
+}
+
+/**
+ * Types a value into a cell and holds on the settled result. The rule engine
+ * flags the cell (htInvalid) once the edit commits, so asserting the expected
+ * state before the hold means the recording always rests on the outcome, and
+ * waits for it however long the engine takes.
+ *
+ * @param expected 'rejected' when the rule engine should flag the cell,
+ *   'accepted' when the value should settle unflagged.
+ */
+const typeAndHold = (
+  rowIndex: number,
+  header: string,
+  value: string,
+  expected: 'accepted' | 'rejected',
+  hold = 1800
+) => {
+  typeIntoCell(rowIndex, header, value)
+
+  getCellByHeaderAndRow(rowIndex, header).should(
+    expected === 'rejected' ? 'have.class' : 'not.have.class',
+    'htInvalid'
+  )
+
+  beat(hold)
+}
+
+const getCellByHeaderAndRow = (rowIndex: number, headerText: string) => {
+  return cy
+    .get('.ht_clone_top .htCore thead tr th')
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => th.innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => th.innerText.trim() === headerText
+      )
+
+      return cy
+        .get('.ht_master tbody tr')
+        .then((rows: any) => rows[rowIndex].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}
+
+const clickOnEdit = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+const submitTable = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary')
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let libNode
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            libNode = node
+            break
+          }
+        }
+
+        cy.get(libNode).within(() => {
+          cy.get('.clr-tree-node-content-container > button').click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+}

client/cypress/e2e/browser-info.cy.ts

@@ -0,0 +1,165 @@
+// Marks this file as an ES module so its top-level consts don't collide with
+// other spec files under the TS type-checker.
+export {}
+//
+// The client sends `browser_info` (and `browser_url_vars`, when the page URL
+// has parameters) with the startup service and with the services that execute
+// customer-provided code - hook scripts and dynamic cell dropdown programs.
+// Other services do not receive the tables.
+//
+// Payloads are keyed by the _program value in the request URL, so the
+// assertions can tell one service's payload from another's. Asserted on the
+// adapter interface, which is the only place the payload is observable end
+// to end.
+//
+// Each test starts from a plain visit of the app root rather than assuming
+// the previous test's session: Cypress clears cookies between tests, the
+// SASjs Server session drops, and the app lands back on the evaluation
+// agreement card. The acceptance is guarded, so it is a no-op when the
+// session survived.
+const hostUrl = Cypress.env('hosturl')
+
+/** The services the app sends diagnostics to (see DIAGNOSTICS_SERVICES in
+ * client/src/app/services/sas.service.ts). */
+const DIAGNOSTICS_SERVICES = [
+  'public/startupservice',
+  'editors/getdata',
+  'editors/getdynamiccolvals',
+  'editors/stagedata',
+  'editors/loadfile',
+  'editors/restore',
+  'auditors/postdata'
+]
+
+/** Boots the app at the root, accepting the evaluation agreement if shown. */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: 60000 })
+  cy.get('body').then(($body) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Captures each service's payload, keyed by the _program URL parameter. */
+const capturePrograms = (): Record<string, string> => {
+  const programs: Record<string, string> = {}
+  cy.intercept('**/stp/execute/**', (req) => {
+    // The handler must never throw: an exception inside an intercept handler
+    // aborts the request, which breaks the app under test. A lone '%' in the
+    // URL would make decodeURIComponent throw, hence the guarded parse.
+    let program = req.url
+    try {
+      program = decodeURIComponent(
+        (req.url.match(/_program=([^&]+)/) || [])[1] || ''
+      )
+    } catch (e) {
+      program = (req.url.match(/_program=([^&]+)/) || [])[1] || req.url
+    }
+    if (typeof req.body === 'string') {
+      programs[program] = req.body
+    } else {
+      programs[program] = JSON.stringify(req.body)
+    }
+  })
+  return programs
+}
+
+context('browser_info input table: ', function () {
+  it('1 | services that run no customer code do not receive it', () => {
+    const programs = capturePrograms()
+    bootApp()
+
+    // The viewer route fires public/viewlibs and a usernav service - plain
+    // data services with no hook scripts and no customer includes. Taken
+    // from a fresh boot, before anything caches the library tree.
+    cy.visit(`${hostUrl}/#/view/data`, { timeout: 60000 })
+    cy.wait(8000)
+
+    cy.then(() => {
+      const captured = Object.keys(programs)
+      // The startup service is a diagnostics service by design - it runs on
+      // every boot, which is the point of it - so it is excluded from this
+      // sweep. On a real estate it would otherwise appear here carrying
+      // browser_info and fail the test.
+      const plain = captured.filter(
+        (program) => !DIAGNOSTICS_SERVICES.some((s) => program.includes(s))
+      )
+      expect(
+        plain.length,
+        `non-diagnostics service calls captured (${captured.join(', ')})`
+      ).to.be.greaterThan(0)
+
+      plain.forEach((program) => {
+        expect(
+          programs[program],
+          `${program} carries no browser_info`
+        ).not.to.contain('browser_info')
+      })
+    })
+  })
+
+  it('2 | hook-running services receive it', () => {
+    const programs = capturePrograms()
+    bootApp()
+
+    // Opening the editor directly makes the app call editors/getdata, which
+    // runs the pre-edit hook and so receives the diagnostics tables.
+    cy.visit(`${hostUrl}/#/editor/DC_JSLIB.MPE_X_TEST`, { timeout: 60000 })
+    cy.get('#hotTable', { timeout: 60000 }).should('exist')
+    cy.wait(8000)
+
+    cy.then(() => {
+      const captured = Object.keys(programs)
+      expect(
+        captured.length,
+        `service calls captured (${captured.join(', ')})`
+      ).to.be.greaterThan(0)
+
+      // The startup service is the other diagnostics recipient, but it is not
+      // reachable from a mock estate: `checkSasjsDeploy` finds makedata in
+      // services/admin and routes to the deploy screen instead, so the app
+      // never calls it on boot. Its null-payload path is covered by the unit
+      // spec (src/app/services/sas.service.spec.ts).
+      const getdata = captured.find((p) => p.includes('editors/getdata'))
+      expect(getdata, 'editors/getdata captured').to.not.be.undefined
+      expect(
+        programs[getdata!],
+        'getdata payload carries browser_info'
+      ).to.contain('browser_info')
+      // TIMEZONE is a column of browser_info and of no other input table, so
+      // this shows the row itself arrived, not just the table name.
+      expect(
+        programs[getdata!],
+        'browser_info row content in the payload'
+      ).to.contain('timezone')
+    })
+  })
+
+  it('3 | browser_url_vars carries the URL parameters as name/value pairs', () => {
+    const programs = capturePrograms()
+    bootApp()
+
+    // The labels parameter is read by the editor from the hash query string,
+    // so the diagnostics tables must carry it as a name/value row.
+    cy.visit(`${hostUrl}/#/editor/DC_JSLIB.MPE_X_TEST?labels=true`, {
+      timeout: 60000
+    })
+    cy.get('#hotTable', { timeout: 60000 }).should('exist')
+    cy.wait(8000)
+
+    cy.then(() => {
+      const getdata = Object.keys(programs).find((p) =>
+        p.includes('editors/getdata')
+      )
+      expect(getdata, 'editors/getdata captured').to.not.be.undefined
+      expect(
+        programs[getdata!],
+        'getdata payload carries browser_url_vars'
+      ).to.contain('browser_url_vars')
+      // the labels parameter, as a name in the name/value table
+      expect(programs[getdata!], 'labels parameter row').to.contain('labels')
+    })
+  })
+})

client/cypress/e2e/ddl-export.cy.ts

@@ -0,0 +1,164 @@
+export {}
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('Export DC Library DDL: ', function () {
+  this.beforeAll(() => {
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+    visitPage('home')
+  })
+
+  it('1 | the admin action opens the flavour chooser and exports the SAS flavour by default', () => {
+    visitPage('system')
+
+    cy.get('.admin-action', { timeout: longerCommandTimeout })
+      .contains('Export DC Library DDL')
+      .should('exist')
+    cy.get('.admin-action').contains('button', 'EXPORT').should('exist')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().should('be.visible')
+    ddlModal().within(() => {
+      cy.get('select').should('have.value', 'SAS')
+      // the schema box only applies to the DB flavours
+      cy.contains('label', 'Schema (optional)').should('not.exist')
+    })
+
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').should(
+      'have.been.calledWithMatch',
+      /\/services\/admin\/exportdb&flavour=SAS$/
+    )
+  })
+
+  it('2 | the PGSQL flavour reveals the optional schema box and passes it to the service', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().within(() => {
+      cy.get('select').select('PGSQL')
+      cy.contains('label', 'Schema (optional)').should('be.visible')
+      cy.get('input[type="text"]').clear().type('DC')
+    })
+
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').should(
+      'have.been.calledWithMatch',
+      /\/services\/admin\/exportdb&flavour=PGSQL&schema=DC$/
+    )
+  })
+
+  it('3 | the TSQL flavour exports without a schema when the box is left empty', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().within(() => {
+      cy.get('select').select('TSQL')
+      cy.contains('label', 'Schema (optional)').should('be.visible')
+    })
+
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').should(
+      'have.been.calledWithMatch',
+      /\/services\/admin\/exportdb&flavour=TSQL$/
+    )
+  })
+
+  it('4 | the SAS export streams the DC library DDL', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().contains('button', 'Export').click()
+
+    // the URL the action opens is served by the backend - against the mocks
+    // that is services/admin/exportdb.js, which renders the library as basic
+    // DDL per flavour
+    cy.get('@open').then((open: any) => {
+      const url = open.getCall(0).args[0]
+      expect(url).to.match(/\/services\/admin\/exportdb&flavour=SAS$/)
+
+      cy.request(url).then((res) => {
+        expect(res.status).to.eq(200)
+        expect(res.headers['content-disposition']).to.contain('DC_JSLIB.ddl')
+        expect(res.body).to.contain('create table DC_JSLIB.MPE_CONFIG(')
+        expect(res.body).to.contain('VAR_NAME char(32)')
+        expect(res.body).to.contain('create table DC_JSLIB.MPE_X_TEST(')
+        // the SAS flavour has no schema statement and no unique index
+        expect(res.body).not.to.contain('CREATE SCHEMA')
+        expect(res.body).not.to.contain('UNIQUE INDEX')
+      })
+    })
+  })
+
+  it('5 | the PGSQL export honours the schema and quotes its identifiers', () => {
+    visitPage('system')
+
+    cy.window().then((win) => {
+      cy.stub(win, 'open').as('open')
+    })
+
+    cy.get('.admin-action').contains('button', 'EXPORT').click()
+
+    const ddlModal = () => cy.contains('clr-modal', 'Export DC Library DDL')
+
+    ddlModal().within(() => {
+      cy.get('select').select('PGSQL')
+      cy.get('input[type="text"]').clear().type('DC')
+    })
+    ddlModal().contains('button', 'Export').click()
+
+    cy.get('@open').then((open: any) => {
+      const url = open.getCall(0).args[0]
+      expect(url).to.match(/&flavour=PGSQL&schema=DC$/)
+
+      cy.request(url).then((res) => {
+        expect(res.status).to.eq(200)
+        expect(res.body).to.contain('CREATE SCHEMA DC;')
+        expect(res.body).to.contain('CREATE TABLE DC.MPE_CONFIG (')
+        expect(res.body).to.contain('"VAR_NAME" VARCHAR(32)')
+        expect(res.body).to.contain('CREATE UNIQUE INDEX "MPE_CONFIG_pk"')
+      })
+    })
+  })
+})
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}

client/cypress/e2e/filter-panel.cy.ts

@@ -0,0 +1,257 @@
+export {}
+//
+// The applied-filter panel shows the clause that is in force. A long clause
+// does not fit the collapsed single line, so the panel carries a chevron that
+// expands it to show the clause in full - in the viewer and in the editor.
+//
+// The chevron is only rendered when the clause actually overflows, so a short
+// filter looks exactly as it did before.
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('applied filter panel: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  // the panel behaviour depends on how much room the clause has, so pin a
+  // laptop-sized window rather than the wider CI viewport
+  this.beforeEach(() => {
+    cy.viewport(1280, 800)
+    bootApp()
+  })
+
+  it('1 | without a filter the panel is not rendered', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    cy.get('.infoBar').should('not.exist')
+
+    cy.screenshot('filter-panel-1-no-filter', { capture: 'viewport' })
+  })
+
+  it('2 | a short filter is shown in full, with no chevron', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    openFilterPopup()
+    setFilterVariable('SOME_DROPDOWN')
+    setFilterValue('Option 1')
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', "SOME_DROPDOWN = 'Option 1'")
+    // the clause fits the collapsed line, so there is nothing to expand
+    assertClauseFitsCollapsedLine()
+    cy.get('.infoBar-toggle').should('not.exist')
+
+    cy.screenshot('filter-panel-2-filter', { capture: 'viewport' })
+  })
+
+  it('3 | a long filter is collapsed to a chevron and expands to the full clause', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    // SOME_CHAR holds a very long free-text value, so an IN over every value
+    // produces a clause of several hundred characters
+    openFilterPopup()
+    setFilterVariable('SOME_CHAR')
+    setFilterOperator('IN')
+    chooseAllFilterValues()
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', 'SOME_CHAR IN')
+    assertClauseOverflowsCollapsedLine()
+    cy.get('.infoBar-toggle')
+      .should('exist')
+      .and('have.attr', 'aria-expanded', 'false')
+
+    cy.screenshot('filter-panel-3-big-filter-collapsed', {
+      capture: 'viewport'
+    })
+
+    // expand: the whole clause is rendered, on several lines, unclipped
+    cy.get('.infoBar-toggle').click()
+    cy.get('.infoBar').should('have.class', 'expanded')
+    cy.get('.infoBar-toggle').should('have.attr', 'aria-expanded', 'true')
+    assertClauseFullyVisible()
+    assertClauseSpansSeveralLines()
+
+    cy.screenshot('filter-panel-3-big-filter-expanded', { capture: 'viewport' })
+
+    // collapse again
+    cy.get('.infoBar-toggle').click()
+    cy.get('.infoBar').should('not.have.class', 'expanded')
+    assertClauseOverflowsCollapsedLine()
+  })
+
+  it('4 | the editor panel collapses and expands in the same way', () => {
+    openTable('DC_JSLIB.MPE_X_TEST')
+
+    openFilterPopup()
+    setFilterVariable('SOME_CHAR')
+    setFilterOperator('IN')
+    chooseAllFilterValues()
+    submitFilter()
+
+    // the editor shows the filter the viewer just stored
+    cy.url().then((url) => {
+      const filterId = url.split('/').pop()
+      cy.visit(
+        `${hostUrl}${appLocation}/#/editor/DC_JSLIB.MPE_X_TEST/${filterId}`,
+        { timeout: longerCommandTimeout }
+      )
+    })
+
+    // wait for the editor itself - .editor-title/.btnCtrl are editor-only, so
+    // this cannot pass on the viewer the visit came from
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should('exist')
+    cy.get('.btnCtrl', { timeout: longerCommandTimeout }).should('exist')
+    cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'SOME_CHAR IN'
+    )
+    assertClauseOverflowsCollapsedLine()
+
+    cy.screenshot('filter-panel-4-editor-collapsed', { capture: 'viewport' })
+
+    cy.get('.infoBar-toggle').click()
+    cy.get('.infoBar').should('have.class', 'expanded')
+    assertClauseFullyVisible()
+    assertClauseSpansSeveralLines()
+
+    cy.screenshot('filter-panel-4-editor-expanded', { capture: 'viewport' })
+  })
+})
+
+/**
+ * Cypress clears cookies between tests, which drops the SASjs Server session
+ * and lands the app back on the evaluation agreement card. Accept it when it is
+ * shown - a no-op when the session survived.
+ */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: longerCommandTimeout })
+  cy.get('body').then(($body: any) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Opens a table in the viewer and waits for its grid to render. */
+const openTable = (libMem: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/view/data/${libMem}`, {
+    timeout: longerCommandTimeout
+  })
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).should('exist')
+  cy.wait(2500)
+}
+
+const openFilterPopup = () => {
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).click()
+  cy.get('clr-dropdown-menu', { timeout: longerCommandTimeout })
+    .contains('Filter')
+    .click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/**
+ * The soft-select inputs drop a transparent .overlay click-catcher over the
+ * modal while their suggestion list is open, so they need force.
+ */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterOperator = (operator: string) => {
+  cy.get('.filter-modal .operator-col select').first().select(operator)
+  cy.wait(500)
+}
+
+/** IN/NOT IN take their values from a modal of checkboxes - take them all. */
+const chooseAllFilterValues = () => {
+  cy.contains('.filter-modal button', 'Choose values').click()
+  cy.get('.in-values-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+  cy.get('.in-values-modal input[type=checkbox]').then(($checkboxes: any) => {
+    for (let i = 0; i < $checkboxes.length; i++) {
+      cy.get('.in-values-modal input[type=checkbox]')
+        .eq(i)
+        .click({ force: true })
+    }
+  })
+  cy.contains('.in-values-modal button', 'Apply').click()
+  cy.wait(500)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+/** The clause is wider than the collapsed line, so it is clipped. */
+const assertClauseOverflowsCollapsedLine = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    expect(
+      el.scrollWidth,
+      'the clause is wider than the collapsed line'
+    ).to.be.greaterThan(el.clientWidth)
+    expect(
+      el.ownerDocument.defaultView!.getComputedStyle(el).whiteSpace,
+      'collapsed panel does not wrap'
+    ).to.equal('nowrap')
+  })
+}
+
+/** The clause fits the collapsed line, so nothing is hidden. */
+const assertClauseFitsCollapsedLine = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    expect(el.scrollWidth, 'the clause fits the collapsed line').to.be.at.most(
+      el.clientWidth
+    )
+  })
+}
+
+/** Expanded, the whole clause is rendered and nothing is clipped. */
+const assertClauseFullyVisible = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    const style = el.ownerDocument.defaultView!.getComputedStyle(el)
+    expect(style.whiteSpace, 'expanded panel wraps').to.equal('normal')
+    expect(el.scrollWidth, 'expanded panel clips nothing').to.be.at.most(
+      el.clientWidth
+    )
+  })
+}
+
+/** Expanded, the clause occupies several lines rather than one. */
+const assertClauseSpansSeveralLines = () => {
+  cy.get('.infoBar-text').then(($text: any) => {
+    const el = $text[0] as HTMLElement
+    const lineHeight =
+      parseFloat(
+        el.ownerDocument.defaultView!.getComputedStyle(el).lineHeight
+      ) || 20
+    expect(
+      el.clientHeight / lineHeight,
+      'the clause spans several lines'
+    ).to.be.greaterThan(3)
+  })
+}

client/cypress/e2e/hook-programs.cy.ts

@@ -0,0 +1,209 @@
+// Marks this file as an ES module (rather than a global script) so its
+// top-level consts don't collide, under the TS type-checker, with the same
+// names declared in other spec files.
+//
+// RUN ORDER MATTERS, and build.yaml reflects it: this spec must run after
+// csv-limited.cy.ts (which asserts the free tier and so needs the first slot,
+// before any spec applies a licence key) and before every spec that submits a
+// changeset. Test 2 reads the approval queue, which is paged oldest first, and
+// a changeset submitted by an earlier spec pushes this one off the first page.
+export {}
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('hook program tests: ', function () {
+  this.beforeAll(() => {
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+
+    visitPage('home')
+  })
+
+  // TESTDATA.DEMO_MIRROR is an empty mirror of TESTDATA.DEMO_ORDERS, registered
+  // with a PRE_EDIT_HOOK and a POST_EDIT_HOOK (services/hooks/demo_mirror_*.js).
+  // The mock services run those hooks the way the SAS backend %includes the
+  // real hook programs - see dcMockUtils.mockHookSource.
+
+  it('1 | PRE_EDIT_HOOK: an empty mirror displays the live rows of its target table', () => {
+    openTableFromTree('testdata', 'demo_mirror')
+
+    // The mirror holds no rows of its own: everything on screen comes from the
+    // pre-edit hook, which loads TESTDATA.DEMO_ORDERS.
+    cy.get('#hotTable .ht_master tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 4)
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .first()
+      .should('contain.text', 'Acme Corp')
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .last()
+      .should('contain.text', 'Delta Systems')
+
+    // The grid is validated against the MIRROR's own rule set (AMOUNT <= 2000),
+    // not the real table's (AMOUNT <= 100000) - that is the point of routing
+    // edits through a mirror.
+    clickOnEdit(() => {
+      editCell(0, 'AMOUNT', '5000')
+    })
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .first()
+      .find('td.htInvalid')
+      .should('exist')
+  })
+
+  it('2 | POST_EDIT_HOOK: a change submitted against the mirror is raised against the target', () => {
+    openTableFromTree('testdata', 'demo_mirror')
+
+    clickOnEdit(() => {
+      editCell(0, 'AMOUNT', '750')
+    })
+
+    submitTable(() => {
+      cy.get('textarea.submit-reason', { timeout: longerCommandTimeout }).type(
+        'hook routing test'
+      )
+      submitTableMessage()
+    })
+
+    // A successful submit sends the app to the staged-data page; wait for that
+    // before navigating away, or the app's own redirect wins the race.
+    cy.url({ timeout: longerCommandTimeout }).should('include', '/stage/')
+
+    // The post-edit hook re-points the changeset at TESTDATA.DEMO_ORDERS, so the
+    // approval queue names the real table rather than the mirror. Search the
+    // whole grid rather than one row: the queue also holds changesets from
+    // earlier specs in the same run, so position is not a stable anchor.
+    visitPage('review/approve')
+
+    cy.get('clr-datagrid clr-dg-row', { timeout: longerCommandTimeout }).should(
+      ($rows) => {
+        const texts = Array.from($rows).map((row: any) =>
+          row.innerText.trim().replace(/\s+/g, ' ')
+        )
+
+        expect(
+          texts.some((text) => text.includes('TESTDATA.DEMO_ORDERS')),
+          `no approval row names TESTDATA.DEMO_ORDERS - rows: ${texts
+            .slice(0, 5)
+            .join(' | ')
+            .slice(0, 400)}`
+        ).to.be.true
+      }
+    )
+  })
+})
+
+// ─── helpers (declared at the bottom, per the other specs) ───────────────────
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let viyaLib
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            viyaLib = node
+            break
+          }
+        }
+
+        cy.get(viyaLib).within(() => {
+          cy.wait(300)
+
+          cy.get(
+            '.clr-tree-node-content-container .clr-treenode-content p'
+          ).click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+
+  cy.get('#hotTable .ht_clone_top .htCore thead button.changeType', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+}
+
+const clickOnEdit = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+// Edits a single cell, locating the column by its header label. Handsontable
+// renders the frozen header in a separate clone pane (.ht_clone_top) - the
+// header row inside .ht_master is kept visibility:hidden - so the labels are
+// read from the clone.
+const editCell = (rowIndex: number, colName: string, value: string) => {
+  cy.get('#hotTable .ht_clone_top .htCore thead th', {
+    timeout: longerCommandTimeout
+  }).then((headers: any) => {
+    let colIndex = -1
+    for (let i = 0; i < headers.length; i++) {
+      if (
+        String(headers[i].innerText || '')
+          .trim()
+          .toUpperCase() === colName.toUpperCase()
+      ) {
+        colIndex = i
+        break
+      }
+    }
+    expect(colIndex, `column ${colName} present`).to.be.greaterThan(-1)
+
+    cy.get('#hotTable .ht_master tbody tr')
+      .eq(rowIndex)
+      .then((row: any) => {
+        cy.get(row[0].childNodes[colIndex])
+          .dblclick({ force: true })
+          .then(() => {
+            cy.focused()
+              .clear()
+              .type(value + '{enter}')
+          })
+      })
+  })
+}
+
+const submitTable = (callback?: any) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+const submitTableMessage = (callback?: any) => {
+  cy.get('.modal-footer .btn.btn-sm.btn-success-outline', {
+    timeout: longerCommandTimeout
+  })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}

client/cypress/e2e/row-cell-limits.cy.ts

@@ -0,0 +1,415 @@
+// The EDIT screen refuses a selection that is too large - DC_MAXOBS_WEBEDIT
+// rows, or DC_MAXCELLS_WEBEDIT cells (rows x columns), whichever is reached
+// first - and the VIEW screen caps at DC_MAXOBS_WEBVIEW rows.
+//
+// This spec pins those limits against the mock, which mirrors the shipped
+// defaults (250 rows / 200000 cells for EDIT, 2000 rows for VIEW).  It uses
+// two existing fixtures rather than new ones:
+//   MPE_X_SEARCH - 1000 rows x 9 cols, so it trips the ROW limit
+//   MPE_X_WIDE   - 200 rows x 1001 cols (200,400 cells), under the row limit
+//                  but over the CELL limit
+export {}
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+const serverType = Cypress.env('serverType')
+const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
+
+context('EDIT row and cell limits, VIEW row limit: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`)
+    // forceLicenceKey=true: applies a valid licence rather than relying on an
+    // earlier spec in the run having done so (see commands.ts).
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.visit(hostUrl + appLocation)
+    visitPage('home')
+    visitPage('view/data')
+  })
+
+  it('1 | VIEW renders up to the raised row cap, past the old 500', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
+
+    // Neither the header nor the cell text can pin the cap: the header reports
+    // the table's true row count whether or not the grid is capped, and the
+    // fixture's own numeric columns carry values above 1500 (DEPTH_M reaches
+    // 4293, SAMPLE_COUNT 4210), so a text match can hit fixture data.
+    //
+    // The PK can pin it.  The first td of a .ht_master row is the PK - the row
+    // header, where present, is a th - and MPE_X_SEARCH holds PK 1001-2000, so
+    // the range check below is what stops a mis-selected column passing.  With
+    // the old 500 row cap the grid stopped at PK 1500, so a rendered PK above
+    // 1500 can only appear if the cap was raised.
+    scrollGridToBottom()
+    cy.get('#hotTable .ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should(($rows) => {
+      const pks = $rows
+        .toArray()
+        .map((row) => row.querySelectorAll('td')[0])
+        .filter((td) => !!td)
+        .map((td) => parseInt((td.textContent || '').trim(), 10))
+        .filter((pk) => !isNaN(pk))
+
+      expect(pks.length).to.be.greaterThan(0)
+      pks.forEach((pk) => expect(pk).to.be.within(1001, 2000))
+      expect(Math.max(...pks)).to.be.greaterThan(1500)
+    })
+  })
+
+  it('2 | EDIT refuses a table over the row limit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
+    openEdit()
+
+    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'Table is too big (1000 rows) - please filter and try again!'
+    )
+  })
+
+  it('3 | EDIT refuses a table over the cell limit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_wide')
+    openEdit()
+
+    // 200 rows x 1002 columns (the 1001 stored columns plus the housekeeping
+    // delete column) = 200,400 cells, over the 200,000 cell limit, while the
+    // row count is under the 250 row limit.
+    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'Selection is too wide (200 rows x 1002 cols) - please filter and try again!'
+    )
+  })
+
+  it('4 | the EDIT limits follow MPE_CONFIG, not the shipped defaults', () => {
+    // Two things at once.  The seeded config carries the shipped defaults (250
+    // rows / 200000 cells), which are the numbers the mock used to hard-code -
+    // so nothing above would notice if the mock stopped reading MPE_CONFIG at
+    // all.  And MPE_CONFIG lives in the control library, so reading it from the
+    // table's own library would fail for this table in particular.
+    //
+    // The cell limit is the one to move: 7 rows x 18 columns is 126 cells, so
+    // at 110 the TESTDATA.MPE_X_NEW edit must be refused, where a hard-coded
+    // 200000 - or a config read that fails and falls back to it - lets it
+    // through.  110 is still above MPE_CONFIG's own 17 x 6 = 102, so the option
+    // can be put back afterwards.
+    setConfigValue('DC_MAXCELLS_WEBEDIT', '110', () => {
+      visitPage('view/data/TESTDATA.MPE_X_NEW')
+      closeAbortModalIfPresent()
+      openEdit()
+
+      cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+        'contain.text',
+        'Selection is too wide (7 rows x 18 cols) - please filter and try again!'
+      )
+    })
+
+    // Put the option back, so the rest of the estate sees the shipped default.
+    setConfigValue('DC_MAXCELLS_WEBEDIT', '200000', () => {
+      visitPage('view/data/TESTDATA.MPE_X_NEW')
+      closeAbortModalIfPresent()
+      openEdit()
+
+      // 126 cells under a 200000 cell limit: the editor loads, no abort.
+      cy.get('#hotTable .ht_master .htCore tbody tr', {
+        timeout: longerCommandTimeout
+      }).should('exist')
+    })
+  })
+
+  it('5 | the filter picker still lists the variables after the abort', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
+    openEdit()
+
+    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      'Table is too big (1000 rows) - please filter and try again!'
+    )
+
+    closeAbortModal()
+
+    // The abort returns no table, and the picker's variable list normally comes
+    // from the table's own columns - so without the public/getcols fallback the
+    // "please filter and try again" advice has nothing to pick from.  Every
+    // column of MPE_X_SEARCH must be on offer.
+    openFilterPopup()
+
+    filterVariables().should(($options) => {
+      const names = $options
+        .toArray()
+        .map((o) => (o.textContent || '').trim())
+        .filter((n) => n.length > 0)
+
+      expect(names, 'the picker offers the table columns').to.include.members([
+        'PRIMARY_KEY_FIELD',
+        'SITE_NAME',
+        'VESSEL',
+        'SAMPLE_COUNT',
+        'DEPTH_M',
+        'SPECIES',
+        'CRUISE_DATE',
+        'EXPEDITION_ID',
+        'NOTES'
+      ])
+    })
+
+    // and the suggestion list is usable, not just populated
+    setFilterVariable('SITE_NAME')
+
+    // Submitting is what exercises TYPE.  The picker keys its operator set and
+    // its value quoting on it, and the fallback list never passes through
+    // mergeColsRules, so without TYPE on this payload a character value would be
+    // submitted unquoted and the backend's %mp_filtercheck would reject it.
+    setFilterValue('Bristol')
+    submitFilter()
+
+    cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      "SITE_NAME = 'Bristol'"
+    )
+  })
+})
+
+const visitPage = (url: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/${url}`)
+}
+
+// The viewer's options dropdown holds the Edit item; it only renders for a
+// table that is registered in MPE_TABLES (tableEditExists()).
+const openEdit = () => {
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).click()
+  cy.contains('[clrDropdownItem]', 'Edit', {
+    timeout: longerCommandTimeout
+  }).click()
+}
+
+/** The abort modal offers Close, and the filter button sits behind it. */
+const closeAbortModal = () => {
+  cy.get('app-info-modal', { timeout: longerCommandTimeout })
+    .contains('button', 'Close')
+    .click()
+}
+
+/**
+ * The EDIT screen's Filter button opens the same .filter-modal the viewer uses.
+ * It only renders once the editor is in its read-only (non-editing) state,
+ * which is where a failed EDIT load leaves it.
+ */
+const openFilterPopup = () => {
+  cy.contains('button.btnView', 'Filter', {
+    timeout: longerCommandTimeout
+  }).click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/** The picker's variable suggestion list. */
+const filterVariables = () =>
+  cy.get('#datalist_vals_var_id0_0 option', { timeout: longerCommandTimeout })
+
+/** The soft-select drops a transparent overlay over the modal, so it needs force. */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+}
+
+/** The value input of the same soft-select row. */
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+/**
+ * Sets a DC option in MPE_CONFIG through the editor and approves it, then hands
+ * back to the callback.  The same route licensing.cy.ts uses for the licence
+ * keys: the config table needs an approval like any other.
+ */
+const setConfigValue = (
+  varName: string,
+  varValue: string,
+  callback: () => void
+) => {
+  reloadApp()
+  // straight to the config table's editor, so the flow does not depend on
+  // whichever view mode the previous step left behind
+  visitPage('editor/DC_JSLIB.MPE_CONFIG')
+  closeAbortModalIfPresent()
+  clickOnEdit()
+
+  editTableField([{ varName, varValue }], () => {
+    submitTable(() => {
+      cy.wait(2000)
+      approveTable(() => {
+        reloadApp()
+        callback()
+      })
+    })
+  })
+}
+
+/**
+ * A full page load.  A visit that only changes the hash does not reload the
+ * app, so an open modal - the abort message from the step just gone, say -
+ * would still be covering the page.
+ */
+const reloadApp = () => {
+  cy.visit(hostUrl + appLocation)
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+}
+
+/**
+ * DC puts abort messages up as modals over whatever page is showing, and they
+ * outlive a hash-only navigation, so clear one before driving the page.
+ */
+const closeAbortModalIfPresent = () => {
+  cy.get('body').then(($body) => {
+    if ($body.find('app-info-modal').length > 0) {
+      cy.get('app-info-modal')
+        .contains('button', 'Close')
+        .click({ force: true })
+    }
+  })
+}
+
+/**
+ * The viewer's Edit button.  The options dropdown (.filterSide) that openEdit()
+ * uses is not rendered for every table, so the config table is opened the way
+ * licensing.cy.ts opens it.
+ */
+const clickOnEdit = (callback?: () => void) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
+/** Types a new value into the row whose cell carries varName. */
+const editTableField = (
+  edits: { varName: string; varValue: string }[],
+  callback: () => void
+) => {
+  cy.get('#hotTable .ht_master .htCore tbody td', {
+    timeout: longerCommandTimeout
+  }).then(($tds: any) => {
+    const nodes = $tds.toArray()
+
+    for (const edit of edits) {
+      for (let i = 0; i < nodes.length; i++) {
+        // exact match on the cell that holds the option name; the cell after it
+        // is the value
+        if ((nodes[i].textContent || '').trim() === edit.varName) {
+          cy.wrap(nodes[i + 1])
+            .dblclick()
+            .then(() => {
+              cy.focused()
+                .type('{selectall}')
+                .type(edit.varValue)
+                .type('{enter}')
+            })
+          break
+        }
+      }
+    }
+
+    callback()
+  })
+}
+
+const submitTable = (callback: () => void) => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => {
+      cy.get(".modal.ng-star-inserted button[type='submit']").click()
+      callback()
+    })
+}
+
+const approveTable = (callback: () => void) => {
+  cy.get('button', { timeout: longerCommandTimeout })
+    .should('contain', 'Approve')
+    .then((allButtons: any) => {
+      for (const approvalButton of allButtons) {
+        if (approvalButton.innerText.toLowerCase().includes('approve')) {
+          approvalButton.click()
+          break
+        }
+      }
+
+      cy.get('button#acceptBtn', { timeout: longerCommandTimeout })
+        .should('exist')
+        .should('not.be.disabled')
+        .click()
+        .then(() => {
+          cy.get('app-history', { timeout: longerCommandTimeout }).should(
+            'exist'
+          )
+          callback()
+        })
+    })
+}
+
+// Handsontable virtualises rows, so the last row is only in the DOM once the
+// grid has been scrolled to the bottom.  The scroller is the .wtHolder inside
+// the .ht_master pane.
+const scrollGridToBottom = () => {
+  cy.get('#hotTable .ht_master .wtHolder', { timeout: longerCommandTimeout })
+    .scrollTo('bottom', { duration: 0 })
+    .trigger('scroll')
+}
+
+const openTableFromTree = (libNameIncludes: string, tablename: string) => {
+  cy.get('.app-loading', { timeout: longerCommandTimeout })
+    .should('not.exist')
+    .then(() => {
+      cy.get('.nav-tree clr-tree > clr-tree-node', {
+        timeout: longerCommandTimeout
+      }).then((treeNodes: any) => {
+        let targetLib
+
+        for (let node of treeNodes) {
+          if (node.innerText.toLowerCase().includes(libNameIncludes)) {
+            targetLib = node
+            break
+          }
+        }
+
+        cy.get(targetLib).within(() => {
+          // Small settle wait: right after a fresh visit the tree component can
+          // still be re-rendering, causing this node to be found then swapped
+          // out mid-click.
+          cy.wait(300)
+
+          cy.get(
+            '.clr-tree-node-content-container .clr-treenode-content p'
+          ).click()
+
+          cy.get('.clr-treenode-link').then((innerNodes: any) => {
+            for (let innerNode of innerNodes) {
+              if (innerNode.innerText.toLowerCase().includes(tablename)) {
+                innerNode.click()
+                break
+              }
+            }
+          })
+        })
+      })
+    })
+
+  cy.get('#hotTable .ht_clone_top .htCore thead button.changeType', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+}

client/licenseChecker.js

@@ -10,7 +10,7 @@ const check = (cwd) => {
         onlyAllow:
           'AFLv2.1;Apache 2.0;Apache-2.0;Apache*;Artistic-2.0;BlueOak-1.0.0;0BSD;BSD*;BSD-2-Clause;BSD-3-Clause;CC0-1.0;CC-BY-3.0;CC-BY-4.0;ISC;MIT;MPL-2.0;ODC-By-1.0;Python-2.0;Unlicense;',
         excludePackages:
-          '@cds/city@1.1.0;@handsontable/angular-wrapper@16.0.1;@handsontable/angular-wrapper@17.1.0;@handsontable/angular-wrapper@18.0.0;handsontable@^16.0.1;handsontable@16.2.0;handsontable@17.1.0;handsontable@18.0.0;hyperformula@2.7.1;hyperformula@3.0.0;hyperformula@3.1.0;hyperformula@3.2.0;hyperformula@3.3.0;jackspeak@3.4.3;path-scurry@1.11.1;package-json-from-dist@1.0.1;buffers@0.1.1'
+          '@cds/city@1.1.0;@handsontable/angular-wrapper@16.0.1;@handsontable/angular-wrapper@17.1.0;@handsontable/angular-wrapper@18.0.0;@handsontable/angular-wrapper@18.1.1;handsontable@^16.0.1;handsontable@16.2.0;handsontable@17.1.0;handsontable@18.0.0;handsontable@18.1.1;hyperformula@2.7.1;hyperformula@3.0.0;hyperformula@3.1.0;hyperformula@3.2.0;hyperformula@3.3.0;jackspeak@3.4.3;path-scurry@1.11.1;package-json-from-dist@1.0.1;buffers@0.1.1'
       },
       (error, json) => {
         if (error) {

client/package.json

@@ -41,21 +41,21 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "^20.3.27",
+    "@angular/animations": "^20.3.33",
     "@angular/cdk": "^20.2.14",
-    "@angular/common": "^20.3.27",
-    "@angular/compiler": "^20.3.27",
-    "@angular/core": "^20.3.27",
-    "@angular/forms": "^20.3.27",
-    "@angular/platform-browser": "^20.3.27",
-    "@angular/platform-browser-dynamic": "^20.3.27",
-    "@angular/router": "^20.3.27",
+    "@angular/common": "^20.3.33",
+    "@angular/compiler": "^20.3.33",
+    "@angular/core": "^20.3.33",
+    "@angular/forms": "^20.3.33",
+    "@angular/platform-browser": "^20.3.33",
+    "@angular/platform-browser-dynamic": "^20.3.33",
+    "@angular/router": "^20.3.33",
     "@cds/core": "^6.15.1",
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
     "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
-    "@handsontable/angular-wrapper": "18.0.0",
-    "@sasjs/adapter": "^4.18.0",
+    "@handsontable/angular-wrapper": "18.1.1",
+    "@sasjs/adapter": "^4.19.1",
     "@sasjs/utils": "^3.6.0",
     "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
     "@types/d3-graphviz": "^2.6.7",
@@ -69,7 +69,7 @@
     "d3-graphviz": "^5.0.2",
     "exceljs": "^4.4.0",
     "fs-extra": "^7.0.1",
-    "handsontable": "18.0.0",
+    "handsontable": "18.1.1",
     "https-browserify": "1.0.0",
     "hyperformula": "^2.5.0",
     "iconv-lite": "^0.5.0",
@@ -100,7 +100,7 @@
     "@angular-eslint/schematics": "19.8.1",
     "@angular-eslint/template-parser": "19.8.1",
     "@angular/cli": "^20.3.32",
-    "@angular/compiler-cli": "^20.3.27",
+    "@angular/compiler-cli": "^20.3.33",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@compodoc/compodoc": "^2.0.0",
     "@cypress/webpack-preprocessor": "^5.17.1",
@@ -148,7 +148,7 @@
     "ajv": "8.18.0",
     "uuid": "11.1.1",
     "lighthouse": "13.4.0",
-    "fast-uri": "3.1.7",
+    "fast-uri": "3.1.8",
     "readdir-glob": {
       "brace-expansion": "^5.0.9"
     },

client/src/_hot-icons.scss

@@ -3,8 +3,7 @@
 
 [class*=ht-theme-classic] .htDropdownMenu table tbody tr td.htSubmenu .htItemWrapper::after,
 [class*=ht-theme-classic] .htContextMenu table tbody tr td.htSubmenu .htItemWrapper::after,
-[class*=ht-theme-classic] .htFiltersConditionsMenu table tbody tr td.htSubmenu .htItemWrapper::after,
-[class*=ht-theme-classic] .pika-single .pika-next {
+[class*=ht-theme-classic] .htFiltersConditionsMenu table tbody tr td.htSubmenu .htItemWrapper::after {
   width: var(--ht-icon-size);
   height: var(--ht-icon-size);
   -webkit-mask-size: contain;
@@ -12,14 +11,6 @@
   background-color: currentColor;
 }
 
-[class*=ht-theme-classic] .pika-single .pika-prev {
-  width: var(--ht-icon-size);
-  height: var(--ht-icon-size);
-  -webkit-mask-size: contain;
-  -webkit-mask-image: url("./assets/hot-icons/arrow-left.svg");
-  background-color: currentColor;
-}
-
 [class*=ht-theme-classic] .ht-page-size-section__select-wrapper::after {
   width: var(--ht-icon-size);
   height: var(--ht-icon-size);

client/src/app/editor/editor.component.html

@@ -369,11 +369,30 @@
           </div>
         }
         @if (!['', ' '].includes(queryText)) {
-          <div class="clr-col-md-12 infoBar">
-            <span
+          <div
+            #infoBar
+            class="clr-col-md-12 infoBar"
+            [class.expanded]="filterExpanded"
+          >
+            <span class="infoBar-text"
               >FILTER :
               <b>{{ queryText }}</b>
             </span>
+            @if (filterOverflows) {
+              <button
+                type="button"
+                class="infoBar-toggle"
+                [attr.aria-expanded]="filterExpanded"
+                [attr.aria-label]="
+                  filterExpanded
+                    ? 'Collapse the filter clause'
+                    : 'Expand the filter clause'
+                "
+                (click)="toggleFilterPanel()"
+              >
+                <clr-icon aria-hidden="true" shape="caret down"></clr-icon>
+              </button>
+            }
           </div>
         }
       </div>

client/src/app/editor/editor.component.ts

@@ -1,4 +1,5 @@
 import {
+  AfterViewChecked,
   AfterViewInit,
   ChangeDetectorRef,
   Component,
@@ -112,7 +113,9 @@ import { ParseResult } from '../models/ParseResult.interface'
   encapsulation: ViewEncapsulation.None,
   standalone: false
 })
-export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
+export class EditorComponent
+  implements OnInit, AfterViewInit, AfterViewChecked, OnDestroy
+{
   @ViewChildren('uploadStater')
   uploadStaterCompList: QueryList<UploadStaterComponent> = new QueryList()
   @ViewChildren('queryFilter')
@@ -525,6 +528,19 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
   public tableData: Array<any> = []
   public queryText = ''
   public queryTextSaved = ''
+
+  /**
+   * The applied-filter panel is collapsed to a single line by default. The
+   * chevron that expands it is only useful when the clause does not fit that
+   * line, which depends on the rendered width - so it is measured from the DOM
+   * rather than guessed from the length of the text.
+   */
+  public filterExpanded = false
+  public filterOverflows = false
+
+  @ViewChild('infoBar') infoBar?: ElementRef<HTMLElement>
+
+  private filterMeasuredKey = ''
   public showApprovers = false
   public pkDups = false
   public validationDone = 0
@@ -1190,7 +1206,16 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
       this.cdf.detectChanges()
 
       this.submitLoading = false
-      this.sasStoreService.setQueryVariables(this.libds, this.cols)
+      // The columns that came with the table are the cheapest source for the
+      // picker. When the table was not returned - the row/cell limits abort the
+      // request before the data arrives - that list is empty, so the store
+      // falls back to public/getcols rather than leaving the picker with
+      // nothing to pick from.
+      this.sasStoreService
+        .setQueryVariablesFromTable(this.libds, this.cols)
+        .catch((err: any) =>
+          this.eventService.catchResponseError('public/getcols', err)
+        )
     }
   }
 
@@ -3547,6 +3572,48 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
     }
   }
 
+  ngAfterViewChecked() {
+    this.scheduleFilterOverflowCheck()
+  }
+
+  /**
+   * Measures whether the applied-filter clause fits the collapsed single line,
+   * and shows or hides the chevron accordingly. The measurement is deferred
+   * because `filterOverflows` and `filterExpanded` are bound in this view -
+   * setting them inside the change-detection cycle would raise
+   * ExpressionChangedAfterItHasBeenChecked.
+   */
+  private scheduleFilterOverflowCheck() {
+    const el = this.infoBar?.nativeElement
+    if (!el) {
+      return
+    }
+
+    const text = el.querySelector('.infoBar-text') as HTMLElement | null
+    if (!text) {
+      return
+    }
+
+    // re-measure only when the clause, the available width, or the state changes
+    const key = `${this.queryText}|${text.clientWidth}|${this.filterExpanded}`
+    if (key === this.filterMeasuredKey) {
+      return
+    }
+    this.filterMeasuredKey = key
+
+    setTimeout(() => {
+      // while expanded the clause wraps, so there is nothing to measure
+      if (this.filterExpanded) {
+        return
+      }
+      this.filterOverflows = text.scrollWidth > text.clientWidth
+    })
+  }
+
+  public toggleFilterPanel() {
+    this.filterExpanded = !this.filterExpanded
+  }
+
   ngAfterViewInit() {
     // Fix ARIA accessibility issues after table initialization
     setTimeout(() => {

client/src/app/editor/utils/characterColumnFormula.integration.spec.ts

@@ -417,9 +417,14 @@ describe('formulas plugin must not be disabled/re-enabled around a table load',
  * capping maxRows at editor_rows_allowed alone makes HyperFormula reject
  * any table bigger than that cap outright. See initSetup's own
  * Math.max(dataSource.length, editor_rows_allowed) fix.
+ *
+ * Handsontable 18.1 fixed the engine coupling upstream (#10672): the grid's
+ * maxRows/maxCols no longer limit the HyperFormula engine, so a capped
+ * maxRows no longer rejects a larger existing dataset. DC's own guard is
+ * still in place; the first test below now pins the fixed behaviour.
  */
 describe('maxRows must never be smaller than the actual loaded row count', () => {
-  it('throws when maxRows is capped below the real row count (proves the mechanism)', () => {
+  it('no longer throws when maxRows is capped below the real row count (HOT 18.1 #10672)', () => {
     const container = document.createElement('div')
     document.body.appendChild(container)
     const colNames = ['PK', 'A', 'B']
@@ -440,7 +445,7 @@ describe('maxRows must never be smaller than the actual loaded row count', () =>
       error = e
     }
 
-    expect(error?.message).toEqual('Sheet size limit exceeded')
+    expect(error).toBeNull()
   })
 
   it('does not throw when maxRows is never let below the actual row count (the fix)', () => {

client/src/app/editor/utils/sortedGridRowSync.integration.spec.ts

@@ -287,9 +287,13 @@ describe('syncOverwrittenCommentForCell resolves the correct visual cell on a so
  * trigger it, independent of what it actually changes. editTable(),
  * cancelEdit(), and every other place editor.component.ts calls
  * updateSettings() after the grid could already be sorted (e.g. the user
- * sorted while still read-only, then clicked Edit) hits this. The fix
- * (updateSettingsSortSafe) clears the sort before the call and restores it
- * immediately afterward.
+ * sorted while still read-only, then clicked Edit) hits this.
+ *
+ * Fixed upstream in Handsontable 18.1 (the formulas plugin now keeps its
+ * index translations across updateSettings), so the corruption no longer
+ * occurs. DC's own guard (updateSettingsSortSafe - clear the sort, call
+ * updateSettings, restore the sort) is still in place and still correct;
+ * this test now pins the fixed behaviour so a future regression shows up.
  */
 describe('updateSettings must not run while a sort is active - it corrupts formula cell references', () => {
   const buildSortedGrid = (): Handsontable => {
@@ -340,13 +344,13 @@ describe('updateSettings must not run while a sort is active - it corrupts formu
       hot.getDataAtRowProp(r, 'FORMULA_HARD_COL')
     )
 
-  it('demonstrates the failure mode: updateSettings while sorted corrupts formula cells', () => {
+  it('no longer corrupts formula cells when updateSettings runs while sorted (HOT 18.1)', () => {
     const hot = buildSortedGrid()
 
     hot.updateSettings({}, false)
     hot.render()
 
-    expect(formulaHardColValues(hot)).toContain('#REF!')
+    expect(formulaHardColValues(hot)).not.toContain('#REF!')
 
     hot.destroy()
   })

client/src/app/services/sas-store.service.ts

@@ -295,6 +295,40 @@ export class SasStoreService {
     this.columns.next(columnsData)
   }
 
+  /**
+   * Column metadata for the filter picker, read from the service.
+   *
+   * The picker's variable list normally comes from the columns that arrived
+   * with the table (see setQueryVariablesFromTable). This is the fallback for
+   * when there is no table - public/getcols reads the metadata only.
+   */
+  public async getCols(libds: string) {
+    const tables = { iwant: [{ libds: libds }] }
+
+    const res = await this.sasService.request('public/getcols', tables)
+
+    return res.adapterResponse.cols || []
+  }
+
+  /**
+   * Feeds the filter picker its variable list.
+   *
+   * `cols` is the column metadata that came back with the table, which costs
+   * nothing extra and is the normal path. When the table was not returned - the
+   * EDIT/VIEW row and cell limits abort the request before the data arrives -
+   * that list is empty, which is what makes "please filter and try again"
+   * unactionable: the picker has nothing to offer. Fall back to
+   * public/getcols, which reads the column metadata without the rows.
+   */
+  public async setQueryVariablesFromTable(libds: string, cols: any[]) {
+    if (cols && cols.length > 0) {
+      this.setQueryVariables(libds, cols)
+      return
+    }
+
+    this.setQueryVariables(libds, await this.getCols(libds))
+  }
+
   public async getChangeInfo(tableId: any) {
     let obj = { TABLE: tableId }
     let table = { SASControlTable: [obj] }

client/src/app/services/sas.service.spec.ts

@@ -0,0 +1,89 @@
+import { SasService } from './sas.service'
+
+/**
+ * SasService's constructor is a plain 8-argument constructor with an empty
+ * body, and `request()` touches only a handful of collaborators, so this
+ * stubs exactly that surface - no TestBed/DI needed, same precedent as
+ * app.service.spec.ts and va-filter.service.spec.ts.
+ *
+ * The SASjs adapter is not built by the constructor either (sasServiceInit
+ * does that), so it is assigned directly - and its `request` spy is where the
+ * outgoing body is observable, which is the whole point of these tests.
+ */
+const buildService = () => {
+  const sasjsAdapter: any = {
+    request: jasmine.createSpy('request').and.resolveTo({}),
+    getSasRequests: () => []
+  }
+  const loggerService: any = {
+    logRequestData: jasmine.createSpy('logRequestData')
+  }
+  const userService: any = { user: undefined }
+  const eventService: any = {
+    showAbortModal: jasmine.createSpy('showAbortModal'),
+    startupDataLoaded: jasmine.createSpy('startupDataLoaded')
+  }
+  const router: any = { navigateByUrl: jasmine.createSpy('navigateByUrl') }
+
+  const service = new SasService(
+    {} as any, // appStoreService
+    userService,
+    eventService,
+    {} as any, // sasjsService
+    {} as any, // sasViyaService
+    loggerService,
+    {} as any, // startupCheckService
+    router
+  )
+  ;(service as any).sasjsAdapter = sasjsAdapter
+
+  return { service, sasjsAdapter }
+}
+
+/** The body the adapter was asked to send for the most recent request. */
+const sentBody = (sasjsAdapter: any) =>
+  sasjsAdapter.request.calls.mostRecent().args[1]
+
+describe('SasService diagnostics payload', () => {
+  it('gives the startup service a body, so browser_info can ride on it', async () => {
+    const { service, sasjsAdapter } = buildService()
+
+    // Both startup service call sites pass null - there is nothing to send -
+    // and the service is the one whose log a support ticket is read from.
+    await service.request('public/startupservice', null)
+
+    const body = sentBody(sasjsAdapter)
+    expect(body).toBeTruthy()
+    expect(body.browser_info.length).toBe(1)
+    expect(body.browser_info[0].user_agent).toBe(navigator.userAgent)
+    expect('timezone' in body.browser_info[0]).toBe(true)
+    expect('tz_offset' in body.browser_info[0]).toBe(true)
+  })
+
+  it('leaves a non-diagnostics service payload alone', async () => {
+    const { service, sasjsAdapter } = buildService()
+
+    await service.request('public/viewlibs', null)
+
+    expect(sentBody(sasjsAdapter)).toBeNull()
+  })
+
+  it('adds browser_info to a body that already carries content', async () => {
+    const { service, sasjsAdapter } = buildService()
+
+    await service.request('editors/getdata', { table: 'MPE_X_TEST' })
+
+    const body = sentBody(sasjsAdapter)
+    expect(body.table).toBe('MPE_X_TEST')
+    expect(body.browser_info.length).toBe(1)
+  })
+
+  it('does not overwrite a browser_info the caller supplied', async () => {
+    const { service, sasjsAdapter } = buildService()
+    const supplied = [{ url: 'supplied by the caller' }]
+
+    await service.request('editors/getdata', { browser_info: supplied })
+
+    expect(sentBody(sasjsAdapter).browser_info).toBe(supplied)
+  })
+})

client/src/app/services/sas.service.ts

@@ -20,6 +20,26 @@ import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapper
 import { SasViyaService } from './sas-viya.service'
 import { ViyaApiFolder } from '../viya-api-explorer/models/viya-api-folder.model'
 import { ViyaApiFolderMembers } from '../viya-api-explorer/models/viya-api-folder-content.model'
+import { parseUserAgent } from '../shared/utils/parse-user-agent'
+import { VERSION } from '../../environments/version'
+
+/**
+ * Services that receive the `browser_info` / `browser_url_vars` diagnostics
+ * tables: the startup service (so every session logs its context once) and
+ * every service that executes customer-provided code - hook scripts via
+ * %mpe_runhook (getdata, stagedata, loadfile, restore, postdata) and the
+ * dynamic cell dropdown programs (getdynamiccolvals). Other services never
+ * see the tables, so the payload stays off the high-frequency calls.
+ */
+const DIAGNOSTICS_SERVICES = [
+  'services/public/startupservice',
+  'services/editors/getdata',
+  'services/editors/getdynamiccolvals',
+  'services/editors/stagedata',
+  'services/editors/loadfile',
+  'services/editors/restore',
+  'services/auditors/postdata'
+]
 
 @Injectable({
   providedIn: 'root'
@@ -118,6 +138,50 @@ export class SasService {
 
     if (!wrapperOptions) wrapperOptions = {}
 
+    // Support diagnostics: tell the backend where the request came from.
+    // Sent only to the services that can act on it - the startup service
+    // (so every session logs its context once) and the services that
+    // %include customer-provided code (hook scripts, dynamic cell dropdown
+    // programs). This is the URL of this page - the Data Controller iframe -
+    // not the document that embeds it, so an embedded report can be
+    // distinguished. The browser_url_vars table carries the URL parameters
+    // as name/value pairs, which is easier for a SAS developer to read than
+    // parsing the url string.
+    // The startup service is called with a null payload - there is nothing to
+    // send - so give it one to carry the diagnostics. It is the service whose
+    // log a support ticket is read from, and without this it never receives
+    // them.
+    if (
+      DIAGNOSTICS_SERVICES.includes(url) &&
+      (data === null || data === undefined)
+    ) {
+      data = {}
+    }
+
+    if (data && typeof data === 'object' && !data.browser_info) {
+      if (DIAGNOSTICS_SERVICES.includes(url)) {
+        const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
+        const ua = parseUserAgent(navigator.userAgent)
+        data.browser_info = [
+          {
+            url: window.location.href,
+            referrer: document.referrer,
+            timezone: tz || '',
+            tz_offset: new Date().getTimezoneOffset(),
+            locale: navigator.language || '',
+            dc_version: VERSION.semverString,
+            adapter_version: VERSION.adapterVersion,
+            browser: ua.browser,
+            browser_version: ua.browserVersion,
+            platform: ua.platform,
+            user_agent: navigator.userAgent || ''
+          }
+        ]
+        const urlVars = this.collectBrowserUrlVars()
+        if (urlVars.length) data.browser_url_vars = urlVars
+      }
+    }
+
     // If debug is on it will print what is going inside the adapter
     this.loggerService.logRequestData(url, data)
 
@@ -263,6 +327,34 @@ export class SasService {
     })
   }
 
+  /**
+   * Collects the page URL parameters as name/value pairs for the
+   * `browser_url_vars` table. Parameters appear both in the search string
+   * (before the hash) and in the hash query string (Angular routes carry
+   * them after the `#`), so both are read - search first, and the first
+   * occurrence of a name wins, so the search string value beats the hash
+   * value on a collision.
+   */
+  private collectBrowserUrlVars(): Array<{ name: string; value: string }> {
+    const vars: { name: string; value: string }[] = []
+    const seen = new Set<string>()
+
+    const collect = (search: string) => {
+      new URLSearchParams(search).forEach((value, name) => {
+        if (!seen.has(name)) {
+          seen.add(name)
+          vars.push({ name, value })
+        }
+      })
+    }
+
+    if (window.location.search) collect(window.location.search.slice(1))
+    const hashQuery = window.location.hash.split('?')[1]
+    if (hashQuery) collect(hashQuery)
+
+    return vars
+  }
+
   /**
    * Uploads a file to the backend, using the adapter upload function.
    *

client/src/app/services/va-messaging.service.ts

@@ -138,13 +138,38 @@ export class VaMessagingService {
     this.earlyDrained = true
     const captured = (window as unknown as { __vaLastMessage?: any })
       .__vaLastMessage
-    const parsed = this.parseData(captured && captured.data)
+    // The early listener (va-early.js) captures from any origin, so re-apply
+    // the live-path trust rule here before acting on it: only replay a message
+    // that came from our own origin or from the frame that embedded us (whose
+    // URL is document.referrer). Without this, a same-origin sibling frame
+    // could inject a crafted DDC message that the live isTrustedSource check
+    // would have rejected.
+    if (!captured || typeof captured.origin !== 'string') return
+    if (!this.isTrustedEarlyOrigin(captured.origin)) return
+    const parsed = this.parseData(captured.data)
     if (!parsed) return
     this.resultName = parsed.resultName
-    if (captured.origin) this.parentOrigin = captured.origin
+    this.parentOrigin = captured.origin
     callback(parsed)
   }
 
+  /**
+   * Origin check for the pre-bootstrap replay. Mirrors isTrustedSource: the
+   * live path trusts a message whose event.source IS the parent frame; for a
+   * captured message we cannot reference its source Window, so we trust an
+   * origin that is this window's origin or the embedding frame's origin
+   * (document.referrer). Unverifiable/absent referrer -> reject.
+   */
+  private isTrustedEarlyOrigin(origin: string): boolean {
+    if (origin === window.location.origin) return true
+    if (!document.referrer) return false
+    try {
+      return new URL(document.referrer).origin === origin
+    } catch {
+      return false
+    }
+  }
+
   /**
    * Parses a raw window MessageEvent into a VaMessage, or null when it is not a
    * recognisable DDC message (e.g. unrelated postMessage traffic).

client/src/app/shared/dc-validator/dc-validator.ts

@@ -92,14 +92,49 @@ export class DcValidator {
     this.rules.push({ ...EDIT_STATUS_COLUMN_RULE })
     this.hiddenColumns.push(this.rules.length - 1)
 
-    this.dqrules = dqRules
+    this.dqrules = [...dqRules]
     this.dqdata = dqData
     this.primaryKeys = sasparams.PK.split(' ')
 
+    // A primary key is NOT NULL by definition, so it must reject a blank and
+    // a special missing (".A"-".Z", "._") even when the target table carries
+    // no physical NOT NULL constraint and MPE_VALIDATIONS has no NOTNULL rule
+    // for it.  Synthesised here so every keyed table gets it, and so the grid,
+    // the edit-record modal and Excel upload validation all see the same rule.
+    this.addPrimaryKeyNotNullRules()
+
     this.updateDqData()
     this.setupValidations()
   }
 
+  /**
+   * Adds a NOTNULL rule for each primary key column that does not already
+   * have one.  A primary key identifies the row, so a blank or a special
+   * missing there is never valid.
+   */
+  private addPrimaryKeyNotNullRules(): void {
+    for (const pk of this.primaryKeys) {
+      if (!pk) continue
+
+      // A buskey can name a column the table no longer has - do not
+      // synthesise a rule for a column that is not in the grid.
+      if (!this.rules.some((rule) => rule.data === pk)) continue
+
+      const hasNotNull = this.dqrules.some(
+        (rule) => rule.BASE_COL === pk && rule.RULE_TYPE === 'NOTNULL'
+      )
+
+      if (!hasNotNull) {
+        this.dqrules.push({
+          BASE_COL: pk,
+          RULE_TYPE: 'NOTNULL',
+          RULE_VALUE: '',
+          X: 1
+        })
+      }
+    }
+  }
+
   registerCustomEditors() {
     Handsontable.editors.registerEditor(
       'autocomplete.custom',
@@ -283,7 +318,14 @@ export class DcValidator {
              * So we will convert it before pushing to array.
              */
             if (rule.type && rule.type === 'numeric') {
-              details.push(Number(data['RULE_DATA']))
+              // A special missing reaches us as a bare letter ("A", "_"), and
+              // the regular missing as "." - Number() would turn either into
+              // NaN, so a strict (HARDSELECT) dropdown could never accept a
+              // value the column actually holds.  Keep them as they are.
+              const rawValue = data['RULE_DATA']
+              details.push(
+                isSpecialMissing(rawValue) ? rawValue : Number(rawValue)
+              )
             } else {
               details.push(data['RULE_DATA'])
             }

client/src/app/shared/dc-validator/tests/dc-validator.spec.ts

@@ -118,7 +118,12 @@ describe('DC Validator', () => {
     expect(dcValidator.getRule('SOME_TIME')).toBeUndefined()
 
     // Test data quality functions
-    expect(dcValidator.getDqDetails()).toHaveSize(dqRules.length)
+    // dqRules + 2 synthesised rules: the SOFTSELECT rule updateDqData()
+    // derives for SOME_DROPDOWN out of dqdata, and the NOTNULL rule
+    // addPrimaryKeyNotNullRules() gives the primary key column (example_dqRules
+    // has none for it).  Note the constructor copies dqRules rather than
+    // aliasing it, so this array is no longer mutated by construction.
+    expect(dcValidator.getDqDetails()).toHaveSize(dqRules.length + 2)
     expect(dcValidator.getDqDetails('non_existant')).toHaveSize(0)
     expect(dcValidator.getDqDetails('SOME_NUM')).toHaveSize(2)
     expect(dcValidator.isDqCol('SOME_NUM')).toBeTrue()
@@ -133,6 +138,91 @@ describe('DC Validator', () => {
     ])
   })
 
+  it('treats the primary key column as NOT NULL, even with no NOTNULL rule configured', () => {
+    const sasparams: SASParam = example_sasparams
+    const cols: Col[] = example_cols
+    const dqRules: DQRule[] = example_dqRules // no NOTNULL for PRIMARY_KEY_FIELD
+    const dqData: DQData[] = example_dqData
+    const $dataFormats: $DataFormats = example_dataformats
+
+    const dcValidator: DcValidator = new DcValidator(
+      sasparams,
+      $dataFormats,
+      cols,
+      dqRules,
+      dqData
+    )
+
+    const pkRules = dcValidator.getDqDetails('PRIMARY_KEY_FIELD')
+    expect(pkRules.some((rule) => rule.RULE_TYPE === 'NOTNULL')).toBeTrue()
+
+    const pkRule = dcValidator.getRule('PRIMARY_KEY_FIELD')
+
+    // A primary key identifies the row, so neither a blank nor a special
+    // missing can satisfy it.
+    dcValidator.executeHotValidator(pkRule!, null, (valid: boolean) => {
+      expect(valid).toBeFalse()
+    })
+    dcValidator.executeHotValidator(pkRule!, 'A', (valid: boolean) => {
+      expect(valid).toBeFalse()
+    })
+    dcValidator.executeHotValidator(pkRule!, 5, (valid: boolean) => {
+      expect(valid).toBeTrue()
+    })
+  })
+
+  it('keeps a special missing in a numeric dropdown source, so a strict rule can match it', () => {
+    const dqData: DQData[] = [
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: 1,
+        SELECTBOX_ORDER: 1
+      },
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: 'A',
+        SELECTBOX_ORDER: 2
+      },
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: '_',
+        SELECTBOX_ORDER: 3
+      },
+      {
+        BASE_COL: 'SOME_NUM',
+        RULE_VALUE: 'SOME_NUM',
+        RULE_DATA: '.',
+        SELECTBOX_ORDER: 4
+      }
+    ] as DQData[]
+
+    const dcValidator: DcValidator = new DcValidator(
+      example_sasparams,
+      example_dataformats,
+      example_cols,
+      example_dqRules,
+      dqData
+    )
+
+    // SOME_NUM is numeric and carries a HARDSELECT_HOOK, so its dropdown
+    // source comes from dqdata.  Number() would have NaN'd the special
+    // missings, leaving the strict membership test unable to match a value
+    // the column actually holds.
+    const source = dcValidator.getDqDropdownSource(
+      dcValidator.getRule('SOME_NUM')!
+    )
+
+    expect(source[0]).toEqual(1)
+    expect(source[1]).toEqual('A')
+    expect(source[2]).toEqual('_')
+    // the regular missing is a dropdown option too, not a NaN
+    expect(source[3]).toEqual('.')
+    expect(source.some((value) => Number.isNaN(value as number))).toBeFalse()
+  })
+
   it('should test hot validator', () => {
     const sasparams: SASParam = example_sasparams
     const cols: Col[] = example_cols
@@ -163,21 +253,22 @@ describe('DC Validator', () => {
     dcValidator.executeHotValidator(someNumRule!, 'ss', (valid: boolean) => {
       expect(valid).toBeFalse()
     })
-    //Special missings
+    // Special missings - a SAS NOT NULL (or primary key) constraint rejects
+    // these, so the rule must reject them too: they are NULL, not values.
     dcValidator.executeHotValidator(someNumRule!, 's', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '.s', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '.', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '..', (valid: boolean) => {
       expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(someNumRule!, '._', (valid: boolean) => {
-      expect(valid).toBeTrue()
+      expect(valid).toBeFalse()
     })
 
     // MINVAL, MAXVAL Validation
@@ -193,7 +284,7 @@ describe('DC Validator', () => {
       expect(valid).toBeFalse()
     })
     dcValidator.executeHotValidator(shortNumRule!, 's', (valid: boolean) => {
-      expect(valid).toBeFalse() // Special missings are lowest numbers, if any MINVAL is set, special missing is always lower
+      expect(valid).toBeFalse() // Special missings are the lowest numbers, so any MINVAL is above them
     })
 
     // CASE validation
@@ -462,11 +553,27 @@ describe('DC Validator', () => {
         example_dqData
       )
 
+    // HOT 18.1 tightened `ColumnSettings['validator']` to the real
+    // `string | RegExp | function` union (18.0 erased it behind an index
+    // signature), so narrow it the way the production code does before
+    // invoking it.
+    const validatorOf = (rule: ReturnType<DcValidator['getRule']>) => {
+      const validator = rule?.validator
+      if (typeof validator !== 'function') {
+        throw new Error('expected the rule to carry a function validator')
+      }
+      return validator as (
+        this: any,
+        value: any,
+        callback: (valid: boolean) => void
+      ) => void
+    }
+
     it('exempts an invalid non-PK cell on a delete-marked row', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(true),
         'not a number',
         (valid: boolean) => {
@@ -479,7 +586,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(true),
         null,
         (valid: boolean) => {
@@ -492,7 +599,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(false),
         'not a number',
         (valid: boolean) => {
@@ -505,7 +612,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const someNumRule = dcValidator.getRule('SOME_NUM')
 
-      someNumRule!.validator!.call(
+      validatorOf(someNumRule).call(
         fakeCellProps(false),
         2,
         (valid: boolean) => {
@@ -518,7 +625,7 @@ describe('DC Validator', () => {
       const dcValidator = buildValidator()
       const pkRule = dcValidator.getRule('PRIMARY_KEY_FIELD')
 
-      pkRule!.validator!.call(
+      validatorOf(pkRule).call(
         fakeCellProps(true),
         'not a number',
         (valid: boolean) => {

client/src/app/shared/dc-validator/tests/dqValidation.spec.ts

@@ -24,7 +24,44 @@ describe('DC Validator - dq validation', () => {
     expect(dqValidate(dqRules, invalidValue)).toBeFalse()
     expect(dqValidate(dqRules, invalidStringValue)).toBeFalse()
     expect(dqValidate(dqRules, numericStringValue)).toBeTrue()
+    // A missing sorts below every number, so it is below the floor.
     expect(dqValidate(dqRules, numericSpecialMissingValue)).toBeFalse()
+    expect(dqValidate(dqRules, null)).toBeFalse()
+    expect(dqValidate(dqRules, undefined)).toBeFalse()
+  })
+
+  it('should order the missing values in a range rule as SAS does', () => {
+    const missingRange: DQRule[] = [
+      { BASE_COL: 'test', RULE_TYPE: 'MINVAL', RULE_VALUE: '.A', X: 0 },
+      { BASE_COL: 'test', RULE_TYPE: 'MAXVAL', RULE_VALUE: '.C', X: 0 }
+    ]
+
+    // Inside the range of missings
+    expect(dqValidate(missingRange, '.A')).toBeTrue()
+    expect(dqValidate(missingRange, '.B')).toBeTrue()
+    expect(dqValidate(missingRange, '.C')).toBeTrue()
+    expect(dqValidate(missingRange, 'b')).toBeTrue()
+
+    // Outside it - above the ceiling
+    expect(dqValidate(missingRange, '.D')).toBeFalse()
+    expect(dqValidate(missingRange, 'z')).toBeFalse()
+
+    // Outside it - below the floor.  The regular missing sits between ._ and .A
+    expect(dqValidate(missingRange, '._')).toBeFalse()
+    expect(dqValidate(missingRange, null)).toBeFalse()
+
+    // Every number sorts above every missing, so it is above the ceiling
+    expect(dqValidate(missingRange, 0)).toBeFalse()
+    expect(dqValidate(missingRange, 5)).toBeFalse()
+
+    // A floor of .A alone still lets the numbers through: they are above it
+    const missingFloor: DQRule[] = [
+      { BASE_COL: 'test', RULE_TYPE: 'MINVAL', RULE_VALUE: '.A', X: 0 }
+    ]
+    expect(dqValidate(missingFloor, '.A')).toBeTrue()
+    expect(dqValidate(missingFloor, '.Z')).toBeTrue()
+    expect(dqValidate(missingFloor, 5)).toBeTrue()
+    expect(dqValidate(missingFloor, '._')).toBeFalse()
   })
 
   it('should validate MAXVAL value', () => {
@@ -49,7 +86,10 @@ describe('DC Validator - dq validation', () => {
     expect(dqValidate(dqRules, invalidValue)).toBeFalse()
     expect(dqValidate(dqRules, invalidStringValue)).toBeFalse()
     expect(dqValidate(dqRules, numericStringValue)).toBeTrue()
+    // A missing sorts below every number, so it is below the ceiling
     expect(dqValidate(dqRules, numericSpecialMissingValue)).toBeTrue()
+    expect(dqValidate(dqRules, null)).toBeTrue()
+    expect(dqValidate(dqRules, undefined)).toBeTrue()
   })
 
   it('should validate UPCASE value', () => {
@@ -111,6 +151,43 @@ describe('DC Validator - dq validation', () => {
     expect(dqValidate(dqRules, invalidValue2)).toBeFalse()
   })
 
+  it('should reject a special missing on a numeric column (a SAS NOT NULL constraint does)', () => {
+    const dqRules: DQRule[] = [
+      {
+        BASE_COL: 'test',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: ' ',
+        X: 0
+      }
+    ]
+
+    // The values a real SAS service delivers for a numeric column.
+    expect(dqValidate(dqRules, 'A', true)).toBeFalse()
+    expect(dqValidate(dqRules, '_', true)).toBeFalse()
+    expect(dqValidate(dqRules, '.', true)).toBeFalse()
+    // Ordinary numbers and numeric strings still pass.
+    expect(dqValidate(dqRules, 5, true)).toBeTrue()
+    expect(dqValidate(dqRules, '5', true)).toBeTrue()
+  })
+
+  it('should not reject a single letter on a character column', () => {
+    const dqRules: DQRule[] = [
+      {
+        BASE_COL: 'test',
+        RULE_TYPE: 'NOTNULL',
+        RULE_VALUE: ' ',
+        X: 0
+      }
+    ]
+
+    // There is no special-missing concept on a character column - a lone
+    // letter is ordinary data.
+    expect(dqValidate(dqRules, 'A')).toBeTrue()
+    expect(dqValidate(dqRules, '_')).toBeTrue()
+    expect(dqValidate(dqRules, '.')).toBeTrue()
+    expect(dqValidate(dqRules, '', false)).toBeFalse()
+  })
+
   it('should return true if rule not found', () => {
     const validValue = 5
 

client/src/app/shared/dc-validator/validations/dq-validation.ts

@@ -1,8 +1,49 @@
 import { DQRule } from '../models/dq-rules.model'
-import { specialMissingNumericValidator } from './hot-custom-validators'
+import { isSpecialMissing } from '@sasjs/utils/input/validators'
 import { isRegexRuleExempt } from '../utils/isRegexRuleExempt'
 import { parseRegexRule } from '../utils/parseRegexRule'
 
+/**
+ * A SAS numeric variable's values have a total order, and its missing values sit
+ * below every non-missing value.  The missing values are themselves ordered:
+ * `._` is the lowest, then the regular missing, then `.A` through `.Z`.
+ *
+ * A range rule compares in that order.  That is what makes a range of missings
+ * meaningful - with `MINVAL .A` and `MAXVAL .C`, `.B` is inside the range and `.D`
+ * is outside it - and it is also why a special missing fails a numeric floor: it
+ * sorts below every number.
+ *
+ * The key is a pair: the class (0 for a missing, 1 for a number, so that every
+ * number sorts above every missing) and the position within that class.  A value
+ * that is neither a number nor a missing has no place in the order and gets null,
+ * which no rule accepts.
+ */
+const sasNumericOrderKey = (value: any): [number, number] | null => {
+  if (value === undefined || value === null || value === '') return [0, 1] // regular missing
+
+  if (typeof value === 'string') {
+    const upper = value.trim().toUpperCase()
+
+    if (upper === '.' || upper === '') return [0, 1] // regular missing
+    if (upper === '._' || upper === '_') return [0, 0] // the lowest missing
+    if (/^\.?[A-Z]$/.test(upper))
+      return [0, 2 + (upper.charCodeAt(upper.length - 1) - 65)] // .A .. .Z
+
+    const numValue = parseFloat(upper)
+
+    return isNaN(numValue) ? null : [1, numValue]
+  }
+
+  const numValue = Number(value)
+
+  return isNaN(numValue) ? null : [1, numValue]
+}
+
+const compareSasNumericOrder = (
+  a: [number, number],
+  b: [number, number]
+): number => (a[0] !== b[0] ? a[0] - b[0] : a[1] - b[1])
+
 const dqValidation: {
   [key: string]: (
     value: any,
@@ -33,25 +74,39 @@ const dqValidation: {
     return true
   },
   MINVAL: (value: any, ruleValue: string | number): boolean => {
-    const isValidNumeric = specialMissingNumericValidator(value)
-    const numValue = parseFloat(value)
+    const valueKey = sasNumericOrderKey(value)
+    const ruleKey = sasNumericOrderKey(ruleValue)
 
-    // If it's validNumeric and it is NaN it means it is special numeric, and those are always less then any
-    // min value set
-    if (isValidNumeric && isNaN(numValue)) return false
+    // A value that is neither a number nor a missing has no place in the order,
+    // so nothing satisfies the rule.
+    if (!valueKey || !ruleKey) return false
 
-    return numValue >= Number(ruleValue.toString())
+    return compareSasNumericOrder(valueKey, ruleKey) >= 0
   },
   MAXVAL: (value: any, ruleValue: string | number): boolean => {
-    const isValidNumeric = specialMissingNumericValidator(value)
-    const numValue = parseFloat(value)
+    const valueKey = sasNumericOrderKey(value)
+    const ruleKey = sasNumericOrderKey(ruleValue)
 
-    if (isValidNumeric && isNaN(numValue)) return true
+    if (!valueKey || !ruleKey) return false
 
-    return numValue <= Number(ruleValue.toString())
+    return compareSasNumericOrder(valueKey, ruleKey) <= 0
   },
-  NOTNULL: (value: any, ruleValue: string | number): boolean => {
-    return value !== undefined && value !== null && value.toString().length > 0
+  NOTNULL: (
+    value: any,
+    ruleValue: string | number,
+    isNumeric: boolean = false
+  ): boolean => {
+    if (value === undefined || value === null) return false
+
+    // A special missing (.A-.Z, ._) is NULL as far as a SAS NOT NULL (or
+    // primary key) constraint is concerned - an insert carrying one is
+    // rejected with an integrity constraint error - so the rule must reject
+    // it too, or the editor would accept a value the target table refuses.
+    // Numeric columns only: a lone letter is ordinary data on a character
+    // column.
+    if (isNumeric && isSpecialMissing(value)) return false
+
+    return value.toString().length > 0
   },
   // Pattern is used as authored, not auto-anchored — a rule author who
   // wants a full-value match must write ^...$ themselves.

client/src/app/shared/sidebar/sidebar.component.spec.ts

@@ -0,0 +1,34 @@
+import { SidebarComponent } from './sidebar.component'
+
+describe('SidebarComponent', () => {
+  // getSubPage is a pure read of the router's current URL, and the only other
+  // dependencies are the two SasService calls in the constructor - so the
+  // class can be constructed directly, without a TestBed.
+  const buildComponent = (routerUrl: string) =>
+    new SidebarComponent(
+      { url: routerUrl } as any,
+      {} as any,
+      {
+        getSasjsConfig: () => ({}),
+        getServerType: () => 'SASVIYA'
+      } as any
+    )
+
+  describe('getSubPage', () => {
+    it('returns the sub-page segment of a plain route', () => {
+      expect(buildComponent('/home/tables').getSubPage()).toEqual('tables')
+    })
+
+    it('excludes the query string - a VA report embed always adds one', () => {
+      expect(buildComponent('/home/tables?embed=va').getSubPage()).toEqual(
+        'tables'
+      )
+    })
+
+    it('excludes the query string on the viewer route', () => {
+      expect(buildComponent('/view/data?labels=true').getSubPage()).toEqual(
+        'data'
+      )
+    })
+  })
+})

client/src/app/shared/sidebar/sidebar.component.ts

@@ -67,7 +67,10 @@ export class SidebarComponent implements OnInit {
   }
 
   public getSubPage() {
-    let url = this._router.url.split('/')
+    // `Router.url` carries the query string, so take the path segment only -
+    // otherwise any route with a parameter renders it as part of the label
+    // (a VA report embed always adds one, e.g. `?embed=va`).
+    let url = this._router.url.split('?')[0].split('/')
 
     return url[2]
   }

client/src/app/shared/utils/col-info-html.spec.ts

@@ -149,3 +149,75 @@ describe('buildColInfoHtml', () => {
     )
   })
 })
+
+/**
+ * DOM-injection reproduction for the column-info dropdown.
+ * buildColInfoHtml interpolates server/DB-controlled values (column label,
+ * format, and DQ RULE_VALUE regex/formula strings) into a string that the
+ * viewer/editor assign to raw DOM `elem.innerHTML` - so a value containing
+ * markup (e.g. a HARDREGEX RULE_VALUE of `<img src=x onerror=alert(1)>`) is
+ * parsed and executed in the browser of whoever opens the info dropdown.
+ * These tests fail on the vulnerable implementation and pass once each field
+ * is escaped.
+ */
+describe('buildColInfoHtml escapes rather than injecting raw HTML', () => {
+  const malicious = '<img src=x onerror=alert(1)>'
+
+  const info: DataFormat = {
+    format: malicious,
+    label: malicious,
+    length: '8',
+    type: 'N'
+  }
+
+  // Parse the returned string the same way the callers do (innerHTML on a
+  // real element) and assert no scriptable element survived.
+  const parseInto = (html: string): HTMLElement => {
+    const host = document.createElement('div')
+    host.innerHTML = html
+    return host
+  }
+
+  const assertNoInjectedElement = (html: string) => {
+    const host = parseInto(html)
+    expect(host.querySelector('img[onerror]')).toBeNull()
+    host.remove()
+  }
+
+  it('is inert for a colInfo whose label and format carry markup', () => {
+    assertNoInjectedElement(buildColInfoHtml('SOMECHAR', info))
+  })
+
+  it('escapes the column NAME', () => {
+    const html = buildColInfoHtml(malicious, {
+      format: '$8.',
+      label: 'safe',
+      length: '8',
+      type: 'C'
+    })
+    // < and > must not survive as markup in the NAME position
+    expect(html).not.toContain(malicious)
+    assertNoInjectedElement(html)
+  })
+
+  it('escapes a HARDREGEX RULE_VALUE', () => {
+    assertNoInjectedElement(
+      buildColInfoHtml('SOMECHAR', info, malicious, undefined, undefined)
+    )
+  })
+
+  it('escapes a SOFTREGEX RULE_VALUE', () => {
+    assertNoInjectedElement(
+      buildColInfoHtml('SOMECHAR', info, undefined, malicious, undefined)
+    )
+  })
+
+  it('escapes a formula RULE_VALUE (with and without a leading =)', () => {
+    assertNoInjectedElement(
+      buildColInfoHtml('SOMECHAR', info, undefined, undefined, malicious)
+    )
+    assertNoInjectedElement(
+      buildColInfoHtml('SOMECHAR', info, undefined, undefined, `=${malicious}`)
+    )
+  })
+})

client/src/app/shared/utils/col-info-html.ts

@@ -1,9 +1,35 @@
 import { DataFormat } from '../../models/sas/common/DateFormat'
 
+/**
+ * Returns string-safe text of any value so it can be concatenated into a
+ * string that is later assigned to raw DOM innerHTML.  The column metadata
+ * (label/format) and DQ RULE_VALUE strings (regex/formula) are DB-controlled -
+ * a validation-rule author can store markup such as
+ * `<img src=x onerror=...>` in a HARDREGEX value or a column label - so they
+ * must never be parsed as HTML by the browser.  Escaping turns any embedded
+ * markup into inert text.
+ */
+const escapeHtml = (value: any): string =>
+  String(value ?? '').replace(/[&<>"']/g, (char) => {
+    const entities: Record<string, string> = {
+      '&': '&amp;',
+      '<': '&lt;',
+      '>': '&gt;',
+      '"': '&quot;',
+      "'": '&#39;'
+    }
+    return entities[char]
+  })
+
 /**
  * Builds the HTML shown in a column-header "info" dropdown item (viewer and
  * editor). NAME is listed first so it's visible regardless of whether
  * headers are currently displayed as NAME or LABEL.
+ *
+ * The returned string is assigned to raw DOM `elem.innerHTML` by both callers
+ * (viewer.component.ts / editor.component.ts) - every field interpolated below
+ * is therefore escaped via escapeHtml, since no Angular sanitizer runs on a
+ * raw innerHTML assignment.
  */
 export function buildColInfoHtml(
   colName: string,
@@ -14,16 +40,16 @@ export function buildColInfoHtml(
 ): string {
   if (!colInfo) return 'No info found'
 
-  let html = `NAME: ${colName}<br>LABEL: ${colInfo.label}<br>TYPE: ${colInfo.type}<br>LENGTH: ${colInfo.length}<br>FORMAT: ${colInfo.format}`
+  let html = `NAME: ${escapeHtml(colName)}<br>LABEL: ${escapeHtml(colInfo.label)}<br>TYPE: ${escapeHtml(colInfo.type)}<br>LENGTH: ${escapeHtml(colInfo.length)}<br>FORMAT: ${escapeHtml(colInfo.format)}`
 
   // Only ever one REGEX rule is applied per column: when both HARDREGEX
   // and SOFTREGEX exist, SOFTREGEX is ignored entirely (same precedence as
   // makeRegexWarningRenderer / DcValidator.failsSoftRegex).  Show only the
   // rule that is applied.
   if (hardRegexValue) {
-    html += `<br>HARDREGEX: ${hardRegexValue}`
+    html += `<br>HARDREGEX: ${escapeHtml(hardRegexValue)}`
   } else if (softRegexValue) {
-    html += `<br>SOFTREGEX: ${softRegexValue}`
+    html += `<br>SOFTREGEX: ${escapeHtml(softRegexValue)}`
   }
 
   // '√x=' stands in for a text label here - HARDFORMULA vs SOFTFORMULA is
@@ -36,7 +62,7 @@ export function buildColInfoHtml(
     const formula = formulaValue.startsWith('=')
       ? formulaValue.slice(1)
       : formulaValue
-    html += `<br>√x=${formula}`
+    html += `<br>√x=${escapeHtml(formula)}`
   }
 
   return html

client/src/app/shared/utils/parse-user-agent.spec.ts

@@ -0,0 +1,119 @@
+import { parseUserAgent } from './parse-user-agent'
+
+/**
+ * Table-driven: the point of the parser is the token ORDER, and the only way
+ * to pin an order is to feed it real user agent strings that carry several
+ * tokens at once. Every case below is a verbatim UA from the product named
+ * (the iOS ones are the ones that used to fall through to Safari).
+ */
+describe('parseUserAgent', () => {
+  const cases: Array<{
+    label: string
+    ua: string
+    browser: string
+    version: string
+    platform: string
+  }> = [
+    {
+      label: 'desktop Edge (Edg)',
+      ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.2592.87',
+      browser: 'Edge',
+      version: '126.0.2592.87',
+      platform: 'Windows'
+    },
+    {
+      label: 'desktop Edge (legacy EdgeHTML)',
+      ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763',
+      browser: 'Edge',
+      version: '18.17763',
+      platform: 'Windows'
+    },
+    {
+      label: 'desktop Opera (OPR)',
+      ua: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 OPR/111.0.0.0',
+      browser: 'Opera',
+      version: '111.0.0.0',
+      platform: 'Linux'
+    },
+    {
+      label: 'desktop Firefox',
+      ua: 'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0',
+      browser: 'Firefox',
+      version: '140.0',
+      platform: 'Linux'
+    },
+    {
+      label: 'desktop Chrome',
+      ua: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+      browser: 'Chrome',
+      version: '126.0.0.0',
+      platform: 'Linux'
+    },
+    {
+      label: 'desktop Safari',
+      ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15',
+      browser: 'Safari',
+      version: '17.5',
+      platform: 'macOS'
+    },
+    {
+      label: 'Android Chrome',
+      ua: 'Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36',
+      browser: 'Chrome',
+      version: '126.0.0.0',
+      platform: 'Android'
+    },
+    {
+      label: 'iOS Chrome (CriOS) - not Safari',
+      ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/126.0.6478.153 Mobile/15E148 Safari/604.1',
+      browser: 'Chrome',
+      version: '126.0.6478.153',
+      platform: 'iOS'
+    },
+    {
+      label: 'iOS Firefox (FxiOS) - not Safari',
+      ua: 'Mozilla/5.0 (iPad; CPU OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/127.0 Mobile/15E148 Safari/605.1.15',
+      browser: 'Firefox',
+      version: '127.0',
+      platform: 'iOS'
+    },
+    {
+      label: 'iOS Edge (EdgiOS) - not Safari',
+      ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) EdgiOS/126.0.2592.86 Version/17.0 Mobile/15E148 Safari/604.1',
+      browser: 'Edge',
+      version: '126.0.2592.86',
+      platform: 'iOS'
+    },
+    {
+      label: 'iOS Safari',
+      ua: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1',
+      browser: 'Safari',
+      version: '17.5',
+      platform: 'iOS'
+    },
+    {
+      label: 'a non-browser client',
+      ua: 'curl/8.5.0',
+      browser: 'Unknown',
+      version: '',
+      platform: 'Unknown'
+    },
+    {
+      label: 'an empty user agent',
+      ua: '',
+      browser: 'Unknown',
+      version: '',
+      platform: 'Unknown'
+    }
+  ]
+
+  cases.forEach((c) => {
+    it(`reports ${c.label} as ${c.browser} ${c.version || '(no version)'} on ${c.platform}`, () => {
+      expect(parseUserAgent(c.ua)).toEqual({
+        browser: c.browser,
+        browserVersion: c.version,
+        platform: c.platform
+      })
+    })
+  })
+})

client/src/app/shared/utils/parse-user-agent.ts

@@ -0,0 +1,77 @@
+/**
+ * Summarises `navigator.userAgent` into the three things worth logging about a
+ * browser: its family, its version, and the platform it is running on.
+ *
+ * The raw user agent is kept alongside these - it is what a support ticket
+ * actually needs - but a hook is far more likely to want to branch on a name
+ * than to write its own user agent parsing.
+ *
+ * Order matters: Edge and Opera both claim to be Chrome, and Chrome claims to
+ * be Safari, so the most specific token has to be tested first. The mobile
+ * tokens are the sharpest case - an iOS browser carries its own token
+ * (`CriOS`, `FxiOS`, `EdgiOS`) *and* `Safari/`, so without the mobile token
+ * being tested first every iPhone and iPad client was reported as Safari with
+ * an empty version.
+ */
+export interface UserAgentSummary {
+  browser: string
+  browserVersion: string
+  platform: string
+}
+
+/**
+ * Most specific token first. Each entry carries the version pattern for the
+ * same token, so the family and its version can never be read from different
+ * products.
+ */
+const BROWSER_TOKENS: Array<{ name: string; token: RegExp; version: RegExp }> =
+  [
+    {
+      name: 'Edge',
+      token: /(?:EdgiOS|Edg[A-Z]?|Edge)\//,
+      version: /(?:EdgiOS|Edg[A-Z]?|Edge)\/([\d.]+)/
+    },
+    {
+      name: 'Opera',
+      token: /(?:OPiOS|OPR)\//,
+      version: /(?:OPiOS|OPR)\/([\d.]+)/
+    },
+    {
+      name: 'Firefox',
+      token: /(?:FxiOS|Firefox)\//,
+      version: /(?:FxiOS|Firefox)\/([\d.]+)/
+    },
+    {
+      name: 'Chrome',
+      token: /(?:CriOS|Chrome)\//,
+      version: /(?:CriOS|Chrome)\/([\d.]+)/
+    },
+    // Safari is the fallback token: everything else that reaches here is
+    // WebKit wearing another product's name.
+    { name: 'Safari', token: /Safari\//, version: /Version\/([\d.]+)/ }
+  ]
+
+export function parseUserAgent(userAgent: string): UserAgentSummary {
+  const ua = userAgent || ''
+  const match = (re: RegExp) => (ua.match(re) || [])[1] || ''
+
+  let browser = 'Unknown'
+  let browserVersion = ''
+
+  for (const entry of BROWSER_TOKENS) {
+    if (entry.token.test(ua)) {
+      browser = entry.name
+      browserVersion = match(entry.version)
+      break
+    }
+  }
+
+  let platform = 'Unknown'
+  if (/Android/.test(ua)) platform = 'Android'
+  else if (/iPhone|iPad|iPod/.test(ua)) platform = 'iOS'
+  else if (/Windows/.test(ua)) platform = 'Windows'
+  else if (/Mac OS X/.test(ua)) platform = 'macOS'
+  else if (/Linux/.test(ua)) platform = 'Linux'
+
+  return { browser, browserVersion, platform }
+}

client/src/app/shared/viewboxes/viewboxes.component.ts

@@ -1404,10 +1404,17 @@ export class ViewboxesComponent implements OnInit, AfterViewInit, OnDestroy {
     this.filter = true
     this.cdf.detectChanges()
 
-    this.sasStoreService.setQueryVariables(
-      this.filterLibds,
-      viewboxTable.hotTable.cols
-    )
+    // Table columns when we have them (the normal, cheapest source); when the
+    // view was aborted by the row limit there is no loaded table to read them
+    // from, so the store falls back to public/getcols.
+    const cols =
+      viewboxTable && viewboxTable.hotTable ? viewboxTable.hotTable.cols : []
+
+    this.sasStoreService
+      .setQueryVariablesFromTable(this.filterLibds, cols)
+      .catch((err: any) =>
+        this.eventService.catchResponseError('public/getcols', err)
+      )
   }
 
   /**

client/src/app/system/system.component.html

@@ -286,6 +286,12 @@
             UPDATE
           </button>
         </div>
+        <div class="admin-action">
+          Export DC Library DDL
+          <button (click)="openDdlExport()" class="btn btn-info btn-sm">
+            EXPORT
+          </button>
+        </div>
       }
     </div>
   </div>
@@ -306,3 +312,40 @@
     </button>
   </div>
 </clr-modal>
+
+<clr-modal [(clrModalOpen)]="ddlModal">
+  <h3 class="modal-title">Export DC Library DDL</h3>
+  <div class="modal-body">
+    <clr-select-container>
+      <label>Flavour</label>
+      <select [(ngModel)]="ddlFlavour" clrSelect>
+        <option value="SAS">SAS</option>
+        <option value="PGSQL">PGSQL</option>
+        <option value="TSQL">TSQL</option>
+      </select>
+    </clr-select-container>
+    @if (ddlFlavour !== 'SAS') {
+      <clr-input-container>
+        <label>Schema (optional)</label>
+        <input
+          clrInput
+          type="text"
+          [(ngModel)]="ddlSchema"
+          placeholder="e.g. DC"
+        />
+      </clr-input-container>
+    }
+  </div>
+  <div class="modal-footer">
+    <button
+      type="button"
+      class="btn btn-sm btn-outline"
+      (click)="ddlModal = false"
+    >
+      Cancel
+    </button>
+    <button type="button" class="btn btn-sm btn-primary" (click)="exportDdl()">
+      Export
+    </button>
+  </div>
+</clr-modal>

client/src/app/system/system.component.ts

@@ -42,6 +42,9 @@ export class SystemComponent implements OnInit {
   Infinity = Infinity
   dcLib: string = globals.dcLib
   targetLibref: string = globals.dcLib
+  ddlModal: boolean = false
+  ddlFlavour: string = 'SAS'
+  ddlSchema: string = ''
 
   licenceState = this.licenceService.licenceState
   settings: AppSettings
@@ -92,6 +95,32 @@ export class SystemComponent implements OnInit {
     window.open(downUrl)
   }
 
+  openDdlExport() {
+    this.ddlFlavour = 'SAS'
+    this.ddlSchema = ''
+    this.ddlModal = true
+  }
+
+  exportDdl() {
+    let sasjsConfig = this.sasService.getSasjsConfig()
+    let storage = sasjsConfig.serverUrl
+    let metaData = sasjsConfig.appLoc
+    let path = this.sasService.getExecutionPath()
+    let downUrl =
+      storage +
+      path +
+      '/?_program=' +
+      metaData +
+      '/services/admin/exportdb&flavour=' +
+      encodeURIComponent(this.ddlFlavour)
+    let schema = this.ddlSchema.trim()
+    if (this.ddlFlavour !== 'SAS' && schema) {
+      downUrl += '&schema=' + encodeURIComponent(schema)
+    }
+    window.open(downUrl)
+    this.ddlModal = false
+  }
+
   refreshDataCatalog() {
     this.refreshingDataCatalog = true
 

client/src/app/viewer/viewer.component.html

@@ -499,10 +499,29 @@
             !['', ' '].includes(queryText) &&
             !abortActive
           ) {
-            <div class="clr-col-md-12 infoBar">
-              <span
+            <div
+              #infoBar
+              class="clr-col-md-12 infoBar"
+              [class.expanded]="filterExpanded"
+            >
+              <span class="infoBar-text"
                 >FILTER : <b>{{ queryText }}</b></span
               >
+              @if (filterOverflows) {
+                <button
+                  type="button"
+                  class="infoBar-toggle"
+                  [attr.aria-expanded]="filterExpanded"
+                  [attr.aria-label]="
+                    filterExpanded
+                      ? 'Collapse the filter clause'
+                      : 'Expand the filter clause'
+                  "
+                  (click)="toggleFilterPanel()"
+                >
+                  <clr-icon aria-hidden="true" shape="caret down"></clr-icon>
+                </button>
+              }
             </div>
           }
         </div>

client/src/app/viewer/viewer.component.ts

@@ -3,6 +3,8 @@ import {
   AfterContentInit,
   ChangeDetectorRef,
   AfterViewInit,
+  AfterViewChecked,
+  ElementRef,
   OnDestroy,
   ViewChildren,
   QueryList,
@@ -58,7 +60,7 @@ import { buildColInfoHtml } from '../shared/utils/col-info-html'
   standalone: false
 })
 export class ViewerComponent
-  implements AfterContentInit, AfterViewInit, OnDestroy
+  implements AfterContentInit, AfterViewInit, AfterViewChecked, OnDestroy
 {
   @ViewChildren('queryFilter')
   queryFilterCompList: QueryList<QueryComponent> = new QueryList()
@@ -96,6 +98,19 @@ export class ViewerComponent
   public libTab!: string
   public queryText: string = ''
   public webQueryText: string = ''
+
+  /**
+   * The applied-filter panel is collapsed to a single line by default. The
+   * chevron that expands it is only useful when the clause does not fit that
+   * line, which depends on the rendered width - so it is measured from the DOM
+   * rather than guessed from the length of the text.
+   */
+  public filterExpanded = false
+  public filterOverflows = false
+
+  @ViewChild('infoBar') infoBar?: ElementRef<HTMLElement>
+
+  private filterMeasuredKey = ''
   public submitLoading!: boolean
   public queryErr: boolean = false
   public queryErrMessage!: string
@@ -1445,6 +1460,48 @@ export class ViewerComponent
     }
   }
 
+  ngAfterViewChecked() {
+    this.scheduleFilterOverflowCheck()
+  }
+
+  /**
+   * Measures whether the applied-filter clause fits the collapsed single line,
+   * and shows or hides the chevron accordingly. The measurement is deferred
+   * because `filterOverflows` and `filterExpanded` are bound in this view -
+   * setting them inside the change-detection cycle would raise
+   * ExpressionChangedAfterItHasBeenChecked.
+   */
+  private scheduleFilterOverflowCheck() {
+    const el = this.infoBar?.nativeElement
+    if (!el) {
+      return
+    }
+
+    const text = el.querySelector('.infoBar-text') as HTMLElement | null
+    if (!text) {
+      return
+    }
+
+    // re-measure only when the clause, the available width, or the state changes
+    const key = `${this.queryText}|${text.clientWidth}|${this.filterExpanded}`
+    if (key === this.filterMeasuredKey) {
+      return
+    }
+    this.filterMeasuredKey = key
+
+    setTimeout(() => {
+      // while expanded the clause wraps, so there is nothing to measure
+      if (this.filterExpanded) {
+        return
+      }
+      this.filterOverflows = text.scrollWidth > text.clientWidth
+    })
+  }
+
+  public toggleFilterPanel() {
+    this.filterExpanded = !this.filterExpanded
+  }
+
   ngAfterViewInit() {
     // Fix ARIA accessibility issues after table initialization
     setTimeout(() => {

client/src/index.html

@@ -59,7 +59,6 @@
     runAsTask="true"
     contextName="SAS Job Execution compute context"
     adminGroup="SASAdministrators"
-    dcPath="/tmp/dc"
     hotLicenceKey="non-commercial-and-evaluation"
   >
   </sasjs>

client/src/styles.scss

@@ -120,30 +120,6 @@ app-editor {
 
   .infoBar {
     margin-top: 14px;
-    background: #495967;
-    color: white;
-    text-align: center;
-    padding: 3px;
-    font-size: 16px;
-
-    height: 30px;
-
-    text-overflow: ellipsis;
-    overflow: hidden;
-    white-space: nowrap;
-
-    span {
-      width: 80%;
-    }
-
-    &:hover {
-      height: unset;
-      white-space: normal;
-
-      span {
-        width: unset;
-      }
-    }
   }
 
   .pkHeader {
@@ -1228,11 +1204,6 @@ app-viewer {
 
   .infoBar {
     margin-top: 10px;
-    background: #495967;
-    color: white;
-    text-align: center;
-    padding: 3px;
-    font-size: 16px;
   }
 
   .filterSide {
@@ -5448,3 +5419,58 @@ body[cds-theme='dark'] {
   cursor: pointer;
   text-decoration: underline;
 }
+
+/* The applied-filter panel. Collapsed to a single line by default so that a
+   long filter does not take over the header; the chevron - rendered only when
+   the clause does not fit that line - expands the panel to show it in full. */
+.infoBar {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 8px;
+  background: #495967;
+  color: white;
+  text-align: center;
+  padding: 3px 8px;
+  font-size: 16px;
+  /* the panel shows a query, so set it in the monospace face the app already
+     uses for code-like content (SAS logs, cell text) */
+  font-family: 'Lucida Console', Monaco, monospace;
+
+  .infoBar-text {
+    flex: 1;
+    min-width: 0;
+    overflow: hidden;
+    white-space: nowrap;
+    text-overflow: ellipsis;
+  }
+
+  &.expanded {
+    align-items: flex-start;
+
+    .infoBar-text {
+      overflow: visible;
+      white-space: normal;
+      text-overflow: clip;
+      overflow-wrap: anywhere;
+    }
+  }
+
+  .infoBar-toggle {
+    flex: none;
+    background: transparent;
+    border: 0;
+    color: inherit;
+    cursor: pointer;
+    padding: 0 2px;
+    line-height: 1;
+
+    clr-icon {
+      transition: transform 0.15s ease-in;
+    }
+  }
+
+  &.expanded .infoBar-toggle clr-icon {
+    transform: rotate(180deg);
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "7.14.1",
+  "version": "7.16.0",
   "description": "Data Controller",
   "devDependencies": {
     "@nogoo9/gitleaks": "8.30.1-post.2",

sas/mocks/sasjs/sasjsconfig.json

@@ -14,7 +14,7 @@
       "streamConfig": {
         "streamWeb": true,
         "streamWebFolder": "web9",
-        "webSourcePath": "`../../../../client/dist",
+        "webSourcePath": "../../client/dist",
         "assetPaths": [],
         "streamServiceName": "clickme"
       }

sas/mocks/sasjs/services/admin/exportdb.js

@@ -0,0 +1,160 @@
+const nodePath = require('path')
+
+let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
+const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
+const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
+
+eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
+
+/**
+ * Mock of services/admin/exportdb.sas - exports the DC library as DDL.
+ *
+ * The real service streams mp_ds2ddl output for every table in &DC_LIBREF, in
+ * the requested flavour, and mp_lib2inserts appends the rows for the SAS and
+ * PGSQL flavours.  This mock emits the DDL only - a basic, readable rendering
+ * per flavour, enough to exercise the System screen's Export DC Library DDL
+ * action end to end (the download headers, the filename, the flavour and the
+ * optional schema).  It deliberately does not append INSERTs: the real service
+ * does, but for a mock that would mean serialising the whole library's rows
+ * (MPE_X_WIDE alone is 200,000 cells) for no extra test value.
+ *
+ * Params arrive from the URL the client opens:
+ *   <serverUrl>/SASjsApi/stp/execute/?_program=<appLoc>/services/admin/exportdb
+ *     &flavour=SAS|PGSQL|TSQL[&schema=<name>]
+ * The schema box only applies to the DB flavours, and the client omits the
+ * param entirely when it is blank, so both must be read defensively - an
+ * unset URL param is an undeclared const here, not an empty string.
+ */
+
+const ddlFlavour = (typeof flavour === 'undefined' || !flavour)
+    ? 'SAS'
+    : String(flavour).toUpperCase()
+
+const dcConf = loadDcSettings(driveRoot, appLoc)
+const libref = dcConf.dcLibref
+const dataDir = dcConf.dataDir
+
+// if no schema param, the DC libref is used (as in exportdb.sas)
+const ddlSchema = (typeof schema === 'undefined' || !schema)
+    ? libref
+    : String(schema)
+
+// ─── Column types, per flavour ───────────────────────────────────────────────
+
+function colType(col, flavour) {
+    const isChar = String(col.type).toUpperCase() === 'C'
+    const fmt = String(col.format || '').toUpperCase()
+    const len = Number(col.length) || (isChar ? 1 : 8)
+
+    if (flavour === 'SAS') {
+        return isChar ? `char(${len})` : 'num'
+    }
+
+    // PGSQL and TSQL share the mapping; only the quoting differs.  The one type
+    // that must not be shared is the datetime: T-SQL's TIMESTAMP is a deprecated
+    // synonym for rowversion - a binary row version, not a datetime - so it maps
+    // to DATETIME2 there.  DOUBLE PRECISION is a T-SQL synonym for float, so it
+    // stays valid on both.
+    if (isChar) return `VARCHAR(${len})`
+    if (fmt.startsWith('DATETIME'))
+        return flavour === 'TSQL' ? 'DATETIME2' : 'TIMESTAMP'
+    if (fmt.startsWith('DATE')) return 'DATE'
+    if (fmt.startsWith('TIME')) return 'TIME'
+    return 'DOUBLE PRECISION'
+}
+
+function colRef(name, flavour) {
+    if (flavour === 'PGSQL') return `"${name}"`
+    if (flavour === 'TSQL') return `[${name}]`
+    return name
+}
+
+function tableRef(table, flavour) {
+    if (flavour === 'TSQL') return `[${ddlSchema}].[${table}]`
+    return `${ddlSchema}.${table}`
+}
+
+// ─── Buskeys (for the unique index), from MPE_TABLES ─────────────────────────
+
+const buskeys = {}
+try {
+    const reg = makeTableLoader(dataDir)('MPE_TABLES')
+    for (const row of ((reg && reg.rows) || [])) {
+        const dsn = String(colVal(row, 'DSN') || '').trim().toUpperCase()
+        const buskey = String(colVal(row, 'BUSKEY') || '').trim()
+        if (dsn && buskey) buskeys[dsn] = buskey
+    }
+} catch(err) { /* no MPE_TABLES - the DDL is still emitted, without indexes */ }
+
+// ─── Build the DDL ───────────────────────────────────────────────────────────
+
+const lines = []
+const stamp = new Date().toISOString().replace('T', ':').substring(0, 19)
+lines.push(`/* DDL generated by the Data Controller mock on ${stamp} */`)
+
+if (ddlFlavour === 'PGSQL') {
+    lines.push(`CREATE SCHEMA ${ddlSchema};`)
+}
+
+let tables = []
+try {
+    tables = fs.readdirSync(dataDir)
+        .filter(f => f.endsWith('.json'))
+        .map(f => {
+            try {
+                return JSON.parse(fs.readFileSync(nodePath.resolve(dataDir, f), {encoding:'utf8'}).toString())
+            } catch(e) { return null }
+        })
+        .filter(t => t && t.name && Array.isArray(t.columns))
+        .sort((a, b) => String(a.name).localeCompare(String(b.name)))
+} catch(err) { /* nothing to export */ }
+
+for (const table of tables) {
+    const tableName = String(table.name).toUpperCase()
+    const cols = table.columns.filter(c => String(c.name || '').trim() !== '')
+    if (cols.length === 0) continue
+
+    if (ddlFlavour === 'SAS') {
+        lines.push(`/* SAS flavour DDL for ${libref}.${tableName} */`)
+        lines.push(`create table ${tableRef(tableName, ddlFlavour)}(`)
+    } else if (ddlFlavour === 'PGSQL') {
+        lines.push(`/* Postgres Flavour DDL for ${libref}.${tableName} */`)
+        lines.push(`CREATE TABLE ${tableRef(tableName, ddlFlavour)} (`)
+    } else {
+        lines.push(`/* TSQL Flavour DDL for ${libref}.${tableName} */`)
+        lines.push(`create table ${tableRef(tableName, ddlFlavour)}(`)
+    }
+
+    cols.forEach((col, idx) => {
+        const name = colRef(String(col.name).toUpperCase(), ddlFlavour)
+        const prefix = idx === 0 ? '   ' : '  ,'
+        lines.push(`${prefix}${name} ${colType(col, ddlFlavour)}`)
+    })
+    lines.push(');')
+
+    // a unique index on the buskey, where MPE_TABLES declares one - SAS has no
+    // equivalent in mp_ds2ddl output, so it is skipped for that flavour
+    const buskey = buskeys[tableName]
+    if (buskey && ddlFlavour !== 'SAS') {
+        const keys = buskey.split(/[\s,]+/).filter(Boolean)
+            .map(k => colRef(k.toUpperCase(), ddlFlavour))
+        lines.push(
+            `CREATE UNIQUE INDEX ${colRef(tableName + '_pk', ddlFlavour)} ON ` +
+            `${tableRef(tableName, ddlFlavour)}(\n  ${keys.join('\n ,')}\n);`
+        )
+    }
+    lines.push('')
+}
+
+// ─── Stream it back as a download, as mp_streamfile does ─────────────────────
+
+const filename = `${libref}.ddl`
+fs.writeFile(
+    _SASJS_WEBOUT_HEADERS,
+    `Content-type: text/plain \n Content-Disposition: attachment;filename="${filename}"`,
+    function (err) {
+        if (err) throw err
+    }
+)
+
+_webout = lines.join('\n') + '\n'

sas/mocks/sasjs/services/auditors/postdata.js

@@ -152,10 +152,19 @@ for (const col of diffCols) colDdtypes[col.name] = getDdType(col)
 // Staged values arrive from CSV as strings ("42"), base values are native JSON
 // (42). Compare numerics by value and strings case-sensitively.
 
+// A SAS special missing (._ , .A-.Z) reaches us as a string. Special missings
+// are numeric-only, and real DC writes the DIFF with `missing=STRING` - whose
+// format maps ._ and .a-.z to a string but leaves a bare `.` as null (see the
+// `bart` format in mp_jsonout.sas). Coercing a special missing with Number()
+// would yield NaN and blank the cell out of the DIFF.
+const SPECIAL_MISSING_RE = /^\.(_|[a-z])$/i
+
 function normVal(value, colName) {
   const col = diffCols.find((c) => c.name === colName)
   if (col && col.type === 'N') {
     if (value === null || value === undefined || value === '') return null
+    const str = String(value).trim()
+    if (SPECIAL_MISSING_RE.test(str)) return str
     const num = Number(value)
     return isNaN(num) ? null : num
   }
@@ -543,7 +552,7 @@ if (action === 'SHOW_DIFFS') {
   const stageFolder = getStageFolder(loadRef)
 
   // check: has this user already approved? (mirrors prev_upload_check in postdata.sas)
-  const reviewData = loadTableData('MPE_REVIEW') || { rows: [] }
+  const reviewData = mpeLoadTableData('MPE_REVIEW') || { rows: [] }
   const alreadyApproved = reviewData.rows.some(
     (r) =>
       r.TABLE_ID === loadRef &&

sas/mocks/sasjs/services/dcMockUtils.js

@@ -5,6 +5,7 @@
  * Provides:
  *   - fetchTable / fetchRaw / parseCsv : emulate SAS %webout(FETCH)
  *   - webOutOpen / webOutObj / webOutClose : emulate SAS %webout(OPEN/OBJ/CLOSE)
+ *   - mpeinit                         : emulate the %mpeinit debug dump
  *   - formatSasValue                  : apply a SAS format to a numeric value
  *   - parseFormattedToSas             : convert an ISO string back to a SAS numeric
  *   - getDdType                       : derive DATE/DATETIME/TIME/N/C from a column format
@@ -147,14 +148,63 @@ function _parseCsvLine(line) {
 // ─── SAS format <-> ISO conversions ──────────────────────────────────────────
 
 /**
- * Derives a DDTYPE (DATE, DATETIME, TIME, N, C) from a SAS column format string.
+ * Column metadata, mirroring %mp_getcols.sas.
+ *
+ * The SAS macro runs proc contents and derives, per column: NAME (upcased),
+ * TYPE (C/N), LENGTH, LABEL (defaulting to the name), VARNUM, FMTNAME, FORMAT
+ * and DDTYPE - CHARACTER, NUMERIC, DATETIME, DATE or TIME.  The date list is
+ * the macro's (YYMMDD, MMDDYY, DDMMYY, MONYY, E8601DA, B8601DA, E8601DT,
+ * NLDATM, NLDATE), so a column types the same way here as it does in the
+ * service that shares this code.
+ *
+ * @param {Array} schemaCols the table's columns as the mock data stores them
+ *   ({name, type, length, format, label}), the equivalent of the proc contents
+ *   output the macro reads
+ * @returns {Array} one entry per column, in the macro's shape and order
+ */
+function mpGetcols(schemaCols) {
+    return (schemaCols || []).map((col, index) => {
+        const name = String(col.name || '').toUpperCase()
+        const isChar = String(col.type || '').toUpperCase() === 'C'
+        const fmtname = String(col.format || '')
+            .replace(/[\d.]+$/, '')
+            .toUpperCase()
+
+        let ddtype
+        if (isChar) ddtype = 'CHARACTER'
+        else if (fmtname.startsWith('DATETIME') || fmtname.startsWith('E8601DT')
+            || fmtname.startsWith('NLDATM')) ddtype = 'DATETIME'
+        else if (fmtname.startsWith('DATE') || fmtname.startsWith('DDMMYY')
+            || fmtname.startsWith('MMDDYY') || fmtname.startsWith('YYMMDD')
+            || fmtname.startsWith('E8601DA') || fmtname.startsWith('B8601DA')
+            || fmtname.startsWith('MONYY') || fmtname.startsWith('NLDATE')) ddtype = 'DATE'
+        else if (fmtname.startsWith('TIME')) ddtype = 'TIME'
+        else ddtype = 'NUMERIC'
+
+        return {
+            NAME: name,
+            TYPE: isChar ? 'C' : 'N',
+            LENGTH: Number(col.length) || 0,
+            LABEL: col.label || col.name,
+            VARNUM: index + 1,
+            FORMAT: String(col.format || ''),
+            FMTNAME: fmtname,
+            DDTYPE: ddtype
+        }
+    })
+}
+
+/**
+ * Derives a DDTYPE (DATE, DATETIME, TIME, N, C) from a SAS column format
+ * string, in the shape the client expects.  Delegates to mpGetcols so the
+ * inference is the macro's, rather than a second format list that can drift
+ * away from the service's.
  */
 function getDdType(col) {
-    const fmt = (col.format || '').toLowerCase()
-    if (fmt.startsWith('datetime') || fmt.startsWith('e8601dt')) return 'DATETIME'
-    if (fmt.startsWith('date') || fmt.startsWith('yymmdd') || fmt.startsWith('ddmmyy') || fmt.startsWith('mmddyy')) return 'DATE'
-    if (fmt.startsWith('time')) return 'TIME'
-    return col.type === 'N' ? 'N' : 'C'
+    const ddtype = mpGetcols([col])[0].DDTYPE
+    if (ddtype === 'CHARACTER') return 'C'
+    if (ddtype === 'NUMERIC') return 'N'
+    return ddtype
 }
 
 /**
@@ -336,6 +386,44 @@ function loadTableSchema(dataDir, tableName) {
     return (data && data.columns) ? data.columns : []
 }
 
+/**
+ * Resolves a hook program name to the source of its mock implementation.
+ *
+ * MPE_TABLES.pre_edit_hook / post_edit_hook (and the SOFTSELECT_HOOK /
+ * HARDSELECT_HOOK rule_value in MPE_VALIDATIONS) hold a hook program: either a
+ * physical .sas path, or a path relative to the appLoc (e.g.
+ * 'services/hooks/mytable_postedit').  The real backend %includes the program
+ * into the running service, so the hook shares the service's macro variables
+ * and work datasets and can modify them.
+ *
+ * The mock mirrors that: this returns the source of the hook's .js counterpart,
+ * which the calling service eval()s in its OWN scope so the hook can read and
+ * modify the service's variables.  Returns null when the hook has no mock
+ * implementation, so callers can fall back to an inline emulation or ignore it.
+ *
+ * Requires nodePath, driveRoot and appLoc in the caller's scope.
+ */
+function mockHookSource(hookValue) {
+    if (!hookValue) return null
+    let hook = String(hookValue).trim()
+    if (!hook) return null
+    // Reduce to the services-relative form, dropping any drive path or appLoc
+    // prefix the config may carry.
+    const idx = hook.indexOf('services/')
+    if (idx > -1) hook = hook.slice(idx)
+    // A physical hook is a .sas program on the server; its mock is a .js file
+    // sitting in the same place on the Drive.
+    hook = hook.replace(/\.sas$/i, '')
+    const hookFile = nodePath.resolve(driveRoot, 'files', appLoc, hook + '.js')
+    // The hook value is hand-edited mock config, and a value carrying '..'
+    // segments would otherwise resolve outside the Drive and be eval()'d
+    // in-service. Keep it inside the Drive's files tree.
+    const filesRoot = nodePath.resolve(driveRoot, 'files')
+    if (!hookFile.startsWith(filesRoot + nodePath.sep)) return null
+    if (!fs.existsSync(hookFile)) return null
+    return fs.readFileSync(hookFile, 'utf8')
+}
+
 /**
  * Synthesises NOTNULL dqrules from the table schema, mirroring the
  * dictionary.columns union in getdata.sas:
@@ -1063,6 +1151,61 @@ function loadDcSettings(driveRoot, appLoc) {
     return result
 }
 
+/**
+ * Reads the DC-scope options out of MPE_CONFIG, mirroring the loop in the real
+ * backend's generated settings.sas: only rows that are active (var_active=1)
+ * and current (now < tx_to) count, and the latest tx_from wins.  Returns a map
+ * of var_name -> trimmed var_value.
+ *
+ * An option that is absent, or switched off, is simply missing from the map so
+ * the caller's own default applies - which is what the real service's
+ * %symexist fallback does (a switched-off option means "use the code default",
+ * not "no limit").
+ *
+ * Requires nodePath, fs and the caller's driveRoot/appLoc/dataDir.
+ */
+function loadDcConfig(driveRoot, appLoc, dataDir) {
+    const out = {}
+    try {
+        const configFile = nodePath.resolve(
+            dataDir || nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
+            'mpe_config.json'
+        )
+        const config = JSON.parse(fs.readFileSync(configFile, {encoding:'utf8'}).toString())
+        const now = Math.floor(Date.now() / 1000)
+        const latest = {}
+        for (const row of (config.rows || [])) {
+            // colVal, not row.var_name: a row that has been through the app's
+            // own edit and approve flow comes back from the adapter with
+            // upper-case keys, and a direct property read would skip it - which
+            // silently drops the option back to its default.
+            if (colVal(row, 'VAR_SCOPE') !== 'DC') continue
+            if (Number(colVal(row, 'VAR_ACTIVE')) !== 1) continue
+            if (Number(colVal(row, 'TX_TO')) <= now) continue
+            const name = String(colVal(row, 'VAR_NAME') || '').trim()
+            if (!name) continue
+            const from = Number(colVal(row, 'TX_FROM')) || 0
+            if (latest[name] === undefined || from > latest[name]) {
+                const val = colVal(row, 'VAR_VALUE')
+                latest[name] = from
+                out[name] = String(val == null ? '' : val).trim()
+            }
+        }
+    } catch(err) { /* no MPE_CONFIG - every option falls back to its default */ }
+    return out
+}
+
+/**
+ * A numeric DC option out of a loadDcConfig() map, or the service's own default
+ * when the option is absent, blank, or not a number.
+ */
+function dcConfigNumber(cfg, name, fallback) {
+    const raw = cfg ? cfg[name] : undefined
+    if (raw === undefined || raw === '') return fallback
+    const num = Number(raw)
+    return isNaN(num) ? fallback : num
+}
+
 // ─── %webout(OPEN/OBJ/CLOSE) emulation ───────────────────────────────────────
 
 /**
@@ -1176,3 +1319,73 @@ function webOutSend(tables) {
     }
     webOutClose()
 }
+
+// ─── %mpeinit() emulation ────────────────────────────────────────────────────
+
+/**
+ * The `_debug` values that mean "debug on", mirroring the test in mpeinit.sas.
+ *
+ * 131 is what the adapter sends - SASjsApiClient.executeJob() puts it on every
+ * server-mode request, and on the multipart form when the payload goes out as
+ * FormData.  128 is what arrives on the Viya WEB JES path when runAsTask is
+ * enabled, and 2477 / 'fields,log,trace' are the SASjs debug values.
+ */
+const MPEINIT_DEBUG_VALUES = ['2477', 'fields,log,trace', '131', '128']
+
+/* Mirrors the &mpeinit=1 guard in the SAS macro: dump once per execution. */
+let _mpeinitDone = false
+
+/**
+ * Dumps the browser_url_vars and browser_info input tables, mirroring the
+ * debug block of %mpeinit (sas/sasjs/macros/mpeinit.sas).
+ *
+ * The SAS macro reads work.browser_url_vars (name/value pairs taken from the
+ * query string) and work.browser_info (the browser fingerprint).  Both arrive
+ * here as input tables - see fetchRaw() - and are dumped with console.log,
+ * which SASjs Server returns in the request's log (after its
+ * SASJS_LOGS_SEPARATOR marker), so the dump shows up in Data Controller's
+ * SAS Log tab exactly as the SAS version does.
+ *
+ * Called at the bottom of this file, so the services that eval() it dump the
+ * tables at startup - the same place %mpeinit() runs in the SAS services.
+ * That covers every diagnostics-carrying service (startup, editors, auditors);
+ * the mocks that do not load this file (lineage, metanav, usernav, a few
+ * public ones) have no equivalent startup step in the SAS app either.
+ *
+ * Difference from the SAS version: the header line is printed once per table,
+ * where SAS repeats it for every row (its PUTLOG sits inside the SET loop).
+ */
+function mpeinit() {
+    if (_mpeinitDone) return
+
+    const debugValue = typeof _debug === 'undefined' ? '' : _debug
+    if (MPEINIT_DEBUG_VALUES.indexOf(String(debugValue)) < 0) return
+
+    _mpeinitDone = true
+
+    const urlVars = fetchRaw('browser_url_vars')
+    if (urlVars === null) {
+        console.log('NOTE: mpeinit: no work.browser_url_vars on this request')
+    } else {
+        console.log('NOTE: mpeinit: work.browser_url_vars:')
+        parseCsv(urlVars).forEach(row => {
+            console.log(
+                'name=' + colVal(row, 'name') + ' value=' + colVal(row, 'value')
+            )
+        })
+    }
+
+    const browserInfo = fetchRaw('browser_info')
+    if (browserInfo === null) {
+        console.log('NOTE: mpeinit: no work.browser_info on this request')
+    } else {
+        console.log('NOTE: mpeinit: work.browser_info:')
+        parseCsv(browserInfo).forEach(row => {
+            console.log(Object.keys(row).map(col => col + '=' + row[col]).join(' '))
+        })
+    }
+}
+
+/* Every service that eval()s this file runs %mpeinit() at startup, the same
+   place the SAS services call it. */
+mpeinit()

sas/mocks/sasjs/services/editors/getdata.js

@@ -8,6 +8,28 @@ const dcLibref = 'DC_JSLIB'
 // Load shared DC mock utilities
 eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
 
+// Mirrors SAS cats() for the values a dropdown source can hold: a special
+// missing stored in its period form (".a") becomes the bare uppercase letter
+// ("A") that cats() returns, so the mock's dropdown list matches the real one.
+function cats(value) {
+    if (typeof value === 'string') {
+        const m = /^\.(_|[a-z])$/i.exec(value.trim())
+        if (m) return m[1].toUpperCase()
+        return value.trim()
+    }
+    return String(value)
+}
+
+/**
+ * SAS sort position of a special missing, or null when the value is ordinary.
+ * `._` sorts first, then `.a` to `.z` - all of them below every number.
+ */
+function missingRank(value) {
+    const m = /^\.(_|[a-z])$/i.exec(String(value).trim())
+    if (!m) return null
+    return m[1] === '_' ? 0 : m[1].toUpperCase().charCodeAt(0) - 64
+}
+
 // ─── Parse input ──────────────────────────────────────────────────────────────
 
 const _sctRow = fetchTable('SASControlTable')[0] || {}
@@ -130,7 +152,7 @@ if (tableData && tableData.rows && tableData.columns) {
         const colName = parts[parts.length - 1]
         const lcName = colName.toLowerCase()
         const seen = new Set()
-        let order = 1
+        const values = []
         for (const row of tableData.rows) {
             let val = row[colName]
             if (val === undefined) val = row[lcName]
@@ -139,19 +161,39 @@ if (tableData && tableData.rows && tableData.columns) {
                 if (key) val = row[key]
             }
             if (val !== undefined && val !== null) {
-                const strVal = String(val)
+                const strVal = cats(val)
                 if (!seen.has(strVal)) {
                     seen.add(strVal)
-                    dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: strVal, SELECTBOX_ORDER: order++ })
+                    values.push({ raw: val, str: strVal })
                 }
             }
         }
+        // getdata.sas orders the source by the column itself, and a special
+        // missing sorts below every number - so the list reads `._`, `.a`-`.z`,
+        // then the numbers ascending.
+        values.sort((a, b) => {
+            const ra = missingRank(a.raw)
+            const rb = missingRank(b.raw)
+            if (ra !== null && rb !== null) return ra - rb
+            if (ra !== null) return -1
+            if (rb !== null) return 1
+            const na = Number(a.str)
+            const nb = Number(b.str)
+            if (!isNaN(na) && !isNaN(nb)) return na - nb
+            return a.str < b.str ? -1 : a.str > b.str ? 1 : 0
+        })
+        values.forEach((v, i) => {
+            dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: v.str, SELECTBOX_ORDER: i + 1 })
+        })
     }
 }
 
 // ─── Build response ──────────────────────────────────────────────────────────
 
 let response
+// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
+// getdata.sas, and is emitted as sasjsAbort below.
+let abortMsg = ''
 
 if (tableData && tableData.columns && tableData.rows) {
     // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
@@ -165,8 +207,8 @@ if (tableData && tableData.columns && tableData.rows) {
         if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
     }
 
-    const visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
-    const visibleRows = tableData.rows
+    let visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
+    let visibleRows = tableData.rows
         .filter(r => filterPredicate(r))
         .map(r => {
         const copy = {}
@@ -176,6 +218,56 @@ if (tableData && tableData.columns && tableData.rows) {
         return copy
     })
 
+    // ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
+    // Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
+    // been applied and the rows sorted, with the data in work.OUT, and may
+    // replace it.  The mock eval()s the hook's .js counterpart in this scope, so
+    // the hook may reassign visibleRows / visibleColumns - e.g. to show the live
+    // rows of another table behind an empty mirror.
+    if (tableReg && tableReg.pre_edit_hook) {
+        const hookSrc = mockHookSource(tableReg.pre_edit_hook)
+        if (hookSrc) {
+            try {
+                eval(hookSrc)
+            } catch (err) {
+                console.log('Error running pre_edit_hook ' + tableReg.pre_edit_hook + ': ' + err.message)
+            }
+        }
+    }
+
+    // ─── Row and cell limits ─────────────────────────────────────────────────
+    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
+    // cells (rows x columns), whichever is reached first.  Both are read from
+    // MPE_CONFIG, the way the real service picks them up from the settings
+    // program it generates - so editing the option in DC changes what the mock
+    // allows, and an option that is absent (or switched off, var_active=0)
+    // falls back to the shipped default.  The check runs after the
+    // PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
+    // replace the data with a larger table.  The cell count includes the
+    // housekeeping delete column, which is what getdata.sas counts.
+    // MPE_CONFIG lives in the control library, not in the table's own library,
+    // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
+    // uses.  Passing the requested table's dataDir made the read fail for every
+    // non-control libref, and the catch then fell back to the shipped defaults.
+    const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
+    const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
+    const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
+    const editColCount = visibleColumns.length + 1
+    const cellCount = visibleRows.length * editColCount
+    if (visibleRows.length > maxObsWebEdit) {
+        abortMsg =
+            'Table is too big (' +
+            visibleRows.length +
+            ' rows) - please filter and try again!'
+    } else if (cellCount > maxCellsWebEdit) {
+        abortMsg =
+            'Selection is too wide (' +
+            visibleRows.length +
+            ' rows x ' +
+            editColCount +
+            ' cols) - please filter and try again!'
+    }
+
     // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
     const colLookup = {}
     for (const col of visibleColumns) {
@@ -208,22 +300,25 @@ if (tableData && tableData.columns && tableData.rows) {
         else if (ddtype === 'TIME') tmVars.push(col.name)
     }
 
-    // Build cols array
-    const cols = visibleColumns.map(col => {
-        const ddtype = getDdType(col)
-        const fmtname = col.format
-            ? col.format.replace(/[\d.]+$/, '').replace('datetime', 'DATETIME').replace('date', 'DATE').replace('time', 'TIME').replace('best', 'BEST').replace('E8601DT', 'DATETIME')
-            : ' '
+    // Build cols array.  Every column of the table, as getdata.sas emits them:
+    // the temporal columns are dropped from the data (the rows below) but not
+    // from the cols payload, so a filter can still be built on them.  mpGetcols
+    // is the JS counterpart of %mp_getcols.sas - the macro getdata.sas itself
+    // calls to type these columns - so the editor sees the same DDTYPE the
+    // service would give it.
+    const cols = mpGetcols(tableData.columns).map((c, i) => {
+        const raw = tableData.columns[i]
+        const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
         let coltype
-        if (ddtype === 'DATE') coltype = `{"data":"${col.name}","type":"date"}`
-        else if (ddtype === 'DATETIME') coltype = `{"data":"${col.name}","type":"datetime"}`
-        else if (ddtype === 'TIME') coltype = `{"data":"${col.name}","type":"time"}`
-        else if (ddtype === 'N') coltype = `{"data":"${col.name}","type":"numeric","format":"0"}`
-        else coltype = `{"data":"${col.name}"}`
+        if (ddtype === 'DATE') coltype = `{"data":"${raw.name}","type":"date"}`
+        else if (ddtype === 'DATETIME') coltype = `{"data":"${raw.name}","type":"datetime"}`
+        else if (ddtype === 'TIME') coltype = `{"data":"${raw.name}","type":"time"}`
+        else if (ddtype === 'N') coltype = `{"data":"${raw.name}","type":"numeric","format":"0"}`
+        else coltype = `{"data":"${raw.name}"}`
         return {
-            NAME: col.name,
-            LABEL: col.label || col.name,
-            FMTNAME: fmtname || ' ',
+            NAME: c.NAME,
+            LABEL: c.LABEL,
+            FMTNAME: c.FMTNAME || ' ',
             DDTYPE: ddtype,
             CLS_RULE: 'READ',
             MEMLABEL: ' ',
@@ -298,15 +393,20 @@ if (tableData && tableData.columns && tableData.rows) {
 }
 
 webOutOpen()
-webOutObj(response.approvers, 'approvers')
-webOutObj(response.cols, 'cols')
-webOutObj(response.dqdata, 'dqdata')
-webOutObj(response.dqrules, 'dqrules')
-webOutObj(response.dsmeta, 'dsmeta')
-webOutObj(response.maxvarlengths, 'maxvarlengths')
-webOutObj(response.query, 'query')
-webOutObj(response.sasdata, 'sasdata', response.$sasdata)
-webOutObj(response.sasparams, 'sasparams')
-webOutObj(response.versions, 'versions')
-webOutObj(response.xl_rules, 'xl_rules')
+if (abortMsg) {
+    webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
+    webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
+} else {
+    webOutObj(response.approvers, 'approvers')
+    webOutObj(response.cols, 'cols')
+    webOutObj(response.dqdata, 'dqdata')
+    webOutObj(response.dqrules, 'dqrules')
+    webOutObj(response.dsmeta, 'dsmeta')
+    webOutObj(response.maxvarlengths, 'maxvarlengths')
+    webOutObj(response.query, 'query')
+    webOutObj(response.sasdata, 'sasdata', response.$sasdata)
+    webOutObj(response.sasparams, 'sasparams')
+    webOutObj(response.versions, 'versions')
+    webOutObj(response.xl_rules, 'xl_rules')
+}
 webOutClose()

sas/mocks/sasjs/services/editors/loadfile.js

@@ -59,8 +59,8 @@ for (let i = 1; i <= 10; i++) {
 
 let libds = ''
 if (typeof table !== 'undefined' && table) libds = String(table).toUpperCase()
-const libref = libds.split('.')[0] || dcLibref
-const dsn = libds.split('.')[1] || ''
+let libref = libds.split('.')[0] || dcLibref
+let dsn = libds.split('.')[1] || ''
 
 // ─── Validate (mirrors the mp_abort checks in loadfile.sas) ─────────────────
 
@@ -167,6 +167,29 @@ if (!msg) {
     }
 
     if (!msg) {
+        // ─── POST_EDIT_HOOK ──────────────────────────────────────────────────
+        // Mirrors mpe_loader.sas: the hook runs before the MPE_SUBMIT record is
+        // written, while LIBREF / DS are still ordinary macro variables, so it
+        // can re-point the changeset at a different table.  The mock eval()s the
+        // hook's .js counterpart in this scope so it can reassign libref / dsn.
+        const hookTables = mpeLoadTableData('MPE_TABLES')
+        const hookReg = (hookTables && hookTables.rows)
+            ? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
+            : null
+        if (hookReg && hookReg.post_edit_hook) {
+            const hookSrc = mockHookSource(hookReg.post_edit_hook)
+            if (hookSrc) {
+                try {
+                    eval(hookSrc)
+                } catch (err) {
+                    console.log(
+                        'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
+                    )
+                }
+                libds = libref + '.' + dsn
+            }
+        }
+
         tableId = makeTableId()
         stageSubmission({
             dataDir: libDataDir(libref),

sas/mocks/sasjs/services/editors/stagedata.js

@@ -20,8 +20,8 @@ const _sctRow = fetchTable('SASControlTable')[0] || {}
 let action = _sctRow.ACTION || 'LOAD'
 let message = _sctRow.MESSAGE || ''
 let libds = _sctRow.LIBDS || ''
-const libref = libds.split('.')[0] || dcLibref
-const dsn = libds.split('.')[1] || ''
+let libref = libds.split('.')[0] || dcLibref
+let dsn = libds.split('.')[1] || ''
 
 // The base table lives in its own library (DC_JSLIB for control tables,
 // TESTDATA for demo user tables); the staging dir stays in the control lib.
@@ -56,6 +56,31 @@ if (schema.length > 0) {
   })
 }
 
+// ─── POST_EDIT_HOOK ──────────────────────────────────────────────────────────
+// Mirrors mpe_loader.sas: %mpe_runhook(POST_EDIT_HOOK) runs before the MPE_SUBMIT
+// record is written, while LIBREF / DS are still ordinary macro variables - which
+// is what lets a hook re-point a changeset at a different table.  The mock
+// eval()s the hook's .js counterpart in this scope so it can reassign
+// libref / dsn / libds; the submit record then names the new base table, exactly
+// as the SAS hook does via call symputx.
+const hookTables = makeTableLoader(mpeDataDir)('MPE_TABLES')
+const hookReg = (hookTables && hookTables.rows)
+    ? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
+    : null
+if (hookReg && hookReg.post_edit_hook) {
+    const hookSrc = mockHookSource(hookReg.post_edit_hook)
+    if (hookSrc) {
+        try {
+            eval(hookSrc)
+        } catch (err) {
+            console.log(
+                'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
+            )
+        }
+        libds = libref + '.' + dsn
+    }
+}
+
 // ─── Stage the data ──────────────────────────────────────────────────────────
 // One subfolder per submission (DSID) under the staging directory, so that the
 // staged data can sit alongside related artifacts (submit/approval logs,

sas/mocks/sasjs/services/hooks/demo_mirror_postedit.js

@@ -0,0 +1,31 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// Mock of a POST_EDIT_HOOK program
+//
+// Registered against TESTDATA.DEMO_MIRROR (MPE_TABLES.post_edit_hook).
+//
+// This file is eval'd by editors/stagedata.js and editors/loadfile.js
+// (mirroring how mpe_loader.sas %includes the hook program via %mpe_runhook),
+// so it shares the caller's scope: libref, dsn, libds and every dcMockUtils
+// function are available directly.
+//
+// The real hook runs in mpe_loader BEFORE the submit record is written, while
+// LIBREF / DS are still ordinary macro variables - which is what lets it
+// re-point a changeset at a different table:
+//
+//     data _null_;
+//       call symputx('libref','TESTDATA');
+//       call symputx('ds','DEMO_ORDERS');
+//     run;
+//
+// (call symputx reaches the calling service's variable because LIBREF/DS are
+// not declared %local in mpe_loader; a %let creates a new variable in the
+// hook's own scope and is discarded.)
+//
+// The mock's equivalent is to reassign libref / dsn, so the MPE_SUBMIT record
+// names the real table and the approval is raised against it rather than
+// against the empty mirror.
+// ─────────────────────────────────────────────────────────────────────────────
+
+libref = 'TESTDATA'
+dsn = 'DEMO_ORDERS'
+console.log('[HOOK DEBUG] demo_mirror_postedit: routing changeset to ' + libref + '.' + dsn)

sas/mocks/sasjs/services/hooks/demo_mirror_preedit.js

@@ -0,0 +1,35 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// Mock of a PRE_EDIT_HOOK program
+//
+// Registered against TESTDATA.DEMO_MIRROR (MPE_TABLES.pre_edit_hook), which is
+// an EMPTY mirror of TESTDATA.DEMO_ORDERS.
+//
+// This file is eval'd by editors/getdata.js (mirroring how getdata.sas
+// %includes the hook program via %mpe_runhook), so it shares the caller's
+// scope: visibleRows, visibleColumns, tableReg, requestedLibref and every
+// dcMockUtils function (loadTableAnyLib, makeTableLoader, libDataDir, ...) are
+// available directly.
+//
+// The real hook is a .sas program which runs after the filter has been applied
+// and the table sorted, with the editor data in work.OUT:
+//
+//     data work.out;
+//       set TESTDATA.DEMO_ORDERS;
+//     run;
+//
+// The mock's equivalent is to replace visibleRows / visibleColumns with the
+// rows and columns of the real table, so an empty mirror still shows live data.
+// (A real hook that must honour the user's filter has to re-apply it - the
+// filter has already been applied to the empty mirror by the time it runs.)
+// ─────────────────────────────────────────────────────────────────────────────
+
+const realTable = loadTableAnyLib('DEMO_ORDERS')
+if (realTable && realTable.data) {
+    visibleRows = realTable.data.rows.map((row) => ({ ...row }))
+    visibleColumns = realTable.data.columns
+    console.log(
+        '[HOOK DEBUG] demo_mirror_preedit: loaded ' +
+            visibleRows.length +
+            ' rows from DEMO_ORDERS'
+    )
+}

sas/mocks/sasjs/services/public/getcols.js

@@ -0,0 +1,79 @@
+const nodePath = require('path')
+
+let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
+const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
+const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
+const dcLibref = 'DC_JSLIB'
+
+// Load shared DC mock utilities
+eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
+
+// ─── Parse input ──────────────────────────────────────────────────────────────
+// getcols.sas reads %mf_getvalue(work.iwant,libds). The adapter serialises the
+// input table as CSV, which fetchTable parses - so the row is `_iwantRow` (the
+// runtime already owns the name `iwant`).
+
+const _iwantRow = fetchTable('iwant')[0] || {}
+const libds = String(_iwantRow.libds || _iwantRow.LIBDS || '').trim()
+
+let libref = dcLibref
+let table = ''
+if (libds) {
+    const parts = libds.split('.')
+    if (parts.length >= 2) {
+        libref = parts[0].trim()
+        table = parts[1].trim()
+    }
+}
+
+// ─── Load the table schema ────────────────────────────────────────────────────
+// Same resolution as getdata.js: the requested libref first, then any libref
+// (a LIBDS that doesn't match the data folder still resolves).
+
+let tableData = table ? makeTableLoader(libDataDir(libref))(table) : null
+if (!tableData) {
+    // Fall back to searching every libref if the table isn't in the requested
+    // libref (e.g. a LIBDS that doesn't match the actual data folder), and take
+    // the libref that actually held it for the registry lookup below.
+    const found = loadTableAnyLib(table)
+    if (found) {
+        tableData = found.data
+        libref = found.libref
+    }
+}
+
+// ─── Build the column list ────────────────────────────────────────────────────
+// Every column, as getdata.sas emits them: the temporal columns are dropped from
+// the data, not from the cols payload, so a filter can still be built on them -
+// which is exactly what "please filter and try again" wants to do.
+
+let cols = []
+
+if (tableData && tableData.columns) {
+    // mpGetcols, then the mapping getcols.sas applies on top of %mp_getcols:
+    // DDTYPE in the client's vocabulary, and TYPE too - on the normal path
+    // mergeColsRules() sets TYPE from the loaded table's formats, and the picker
+    // keys its operator set and its value quoting on it.  This list exists
+    // because that load did not happen, so it has to carry it.
+    cols = mpGetcols(tableData.columns)
+        .map(c => {
+            const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
+            return {
+                NAME: c.NAME,
+                LABEL: c.LABEL,
+                FMTNAME: c.FMTNAME || ' ',
+                DDTYPE: ddtype,
+                TYPE: ['DATE', 'DATETIME', 'TIME', 'N'].includes(ddtype) ? 'num' : 'char',
+                CLS_RULE: 'READ',
+                MEMLABEL: ' ',
+                DESC: ' ',
+                LONGDESC: ' ',
+                COLTYPE: ' '
+            }
+        })
+        .sort((a, b) => a.NAME.localeCompare(b.NAME))
+}
+
+webOutOpen()
+webOutObj(cols, 'cols')
+webOutClose()

sas/mocks/sasjs/services/public/viewdata.js

@@ -8,9 +8,15 @@ const mpeDataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', dcLibref
 
 eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
 
-// Row cap for the viewer - mirrors DC_MAXOBS_WEBVIEW in viewdata.sas, and the
+// Row cap for the viewer - mirrors DC_MAXOBS_WEBVIEW in viewdata.sas, read from
+// MPE_CONFIG the way the real service reads it from the settings program (an
+// absent or switched-off option falls back to the shipped default).  Also the
 // MAXROWS value reported in sasparams.
-const maxObsWebView = 500
+const maxObsWebView = dcConfigNumber(
+    loadDcConfig(driveRoot, appLoc, mpeDataDir),
+    'DC_MAXOBS_WEBVIEW',
+    2000
+)
 
 // ─── Parse input ──────────────────────────────────────────────────────────────
 
@@ -95,16 +101,15 @@ let vars = {}
 let nobs = 0
 
 if (tableData && tableData.columns) {
-    cols = tableData.columns.map(col => {
-        const ddtype = getDdType(col)
-        const fmtname = col.format
-            ? col.format.replace(/[\d.]+$/, '').replace('datetime', 'DATETIME').replace('date', 'DATE').replace('time', 'TIME').replace('best', 'BEST').replace('E8601DT', 'DATETIME')
-            : ' '
-        const formatStr = col.type === 'N' ? (col.format || '8.') : ('$' + col.length + '.')
+    // mpGetcols, the JS counterpart of %mp_getcols.sas - the macro viewdata.sas
+    // calls for its own column list.
+    cols = mpGetcols(tableData.columns).map(c => {
+        const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
+        const formatStr = c.TYPE === 'N' ? (c.FORMAT || '8.') : ('$' + c.LENGTH + '.')
         return {
-            NAME: col.name, TYPE: col.type === 'N' ? 'N' : 'C', LENGTH: col.length,
-            FORMAT: formatStr, FMTNAME: fmtname || ' ', DDTYPE: ddtype,
-            LABEL: col.label || col.name
+            NAME: c.NAME, TYPE: c.TYPE, LENGTH: c.LENGTH,
+            FORMAT: formatStr, FMTNAME: c.FMTNAME || ' ', DDTYPE: ddtype,
+            LABEL: c.LABEL
         }
     })
 

sas/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "name": "dc-sas",
       "dependencies": {
-        "@sasjs/cli": "4.20.4",
+        "@sasjs/cli": "4.20.5",
         "@sasjs/core": "5.2.8"
       }
     },
@@ -153,49 +153,14 @@
         "lodash.isnil": "^4.0.0"
       }
     },
-    "node_modules/@nodelib/fs.scandir": {
-      "version": "2.1.5",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
-      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
-      "license": "MIT",
-      "dependencies": {
-        "@nodelib/fs.stat": "2.0.5",
-        "run-parallel": "^1.1.9"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/@nodelib/fs.stat": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
-      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/@nodelib/fs.walk": {
-      "version": "1.2.8",
-      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
-      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
-      "license": "MIT",
-      "dependencies": {
-        "@nodelib/fs.scandir": "2.1.5",
-        "fastq": "^1.6.0"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/@sasjs/adapter": {
-      "version": "4.19.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.19.0.tgz",
-      "integrity": "sha512-AhSldduDFPAuhRsXdzeiwMqEm0lm6dnh2yVxDSUmxxw8V9YShmeGPmEzvU8n2Cl4VAyrhc+Ietc5YWFYC03G+Q==",
+      "version": "4.19.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.19.1.tgz",
+      "integrity": "sha512-IpN6UPSdqVb5cAWEBt+jG/i0rhgCubdyp5kxdouuX3ifL313oJW2ZPVUNUvs0+ZGzsXWcGDTNoEzzTrGKiqJlQ==",
       "license": "ISC",
       "dependencies": {
         "@sasjs/utils": "^3.6.0",
-        "axios": "1.18.1",
+        "axios": "1.20.0",
         "axios-cookiejar-support": "5.0.5",
         "form-data": "4.0.6",
         "https": "1.0.0",
@@ -203,14 +168,14 @@
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "4.20.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.20.4.tgz",
-      "integrity": "sha512-hKE002Dm9AzWU62qpRyYpjQv8BIXG951Z0ToNdG9J1wvvJNYH7h6rGv7kdQMbgvXff/KOQvVdnGJZ023SrG+Jw==",
+      "version": "4.20.5",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.20.5.tgz",
+      "integrity": "sha512-/HSpueCFS+SvXas9FwfP8qoD9skhdTDE5r7dT9Xlrs7HGCBjxDYvSiw6tW0GIjfw3/49Yf8TCVDFBsISang1hg==",
       "license": "ISC",
       "dependencies": {
-        "@sasjs/adapter": "^4.19.0",
-        "@sasjs/core": "5.2.9",
-        "@sasjs/lint": "2.5.0",
+        "@sasjs/adapter": "4.19.1",
+        "@sasjs/core": "5.2.10",
+        "@sasjs/lint": "4.1.0",
         "@sasjs/utils": "3.6.2",
         "chalk": "4.1.2",
         "dotenv": "17.4.2",
@@ -236,11 +201,28 @@
       }
     },
     "node_modules/@sasjs/cli/node_modules/@sasjs/core": {
-      "version": "5.2.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.9.tgz",
-      "integrity": "sha512-R5wtJF0ANHchaURqTF2t1vRXrhUJ+uDfj21ewzdOq4Lp14CWW22+yYeRQooY4QOpyHUyFDjReA5q6SoJt2Tz1g==",
+      "version": "5.2.10",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.10.tgz",
+      "integrity": "sha512-Lk/ENHVuXaeAl8M0+Mjh1rcBSdFt4tIM6OPxPPmk78ZD0gYFqAOUZreqIoV9EN+VwPDqYsUPDqXzh0crW6j8OA==",
       "license": "MIT"
     },
+    "node_modules/@sasjs/cli/node_modules/shelljs": {
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "glob": "^7.0.0",
+        "interpret": "^1.0.0",
+        "rechoir": "^0.6.2"
+      },
+      "bin": {
+        "shjs": "bin/shjs"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/@sasjs/core": {
       "version": "5.2.8",
       "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.8.tgz",
@@ -248,14 +230,14 @@
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-2.5.0.tgz",
-      "integrity": "sha512-bVUtYnKhCigXylCCvQ6dsAIywpPKkX3BXatMqOWjDVw9R30jfj4M3/qn05fcR45L42Y7e5Tuzlt+LYs+9k95Yg==",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-4.1.0.tgz",
+      "integrity": "sha512-Vpg1s0/zdPMFJ0hRXNXk53HhE3kfRr5gFRS0kvrse+bxgcbqih1qqVW7yZ2krhSCYylp9z13eBXLhXbRrj70VQ==",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
         "@sasjs/utils": "3.6.2",
-        "ignore": "7.0.8"
+        "ignore": "7.0.11"
       }
     },
     "node_modules/@sasjs/utils": {
@@ -390,13 +372,13 @@
       "license": "MIT"
     },
     "node_modules/axios": {
-      "version": "1.18.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz",
-      "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==",
+      "version": "1.20.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz",
+      "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==",
       "license": "MIT",
       "dependencies": {
         "follow-redirects": "^1.16.0",
-        "form-data": "^4.0.5",
+        "form-data": "^4.0.6",
         "https-proxy-agent": "^5.0.1",
         "proxy-from-env": "^2.1.0"
       }
@@ -420,6 +402,12 @@
         "tough-cookie": ">=4.0.0"
       }
     },
+    "node_modules/balanced-match": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+      "license": "MIT"
+    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -451,16 +439,14 @@
         "readable-stream": "^3.4.0"
       }
     },
-    "node_modules/braces": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
-      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
+    "node_modules/brace-expansion": {
+      "version": "1.1.21",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz",
+      "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==",
       "license": "MIT",
       "dependencies": {
-        "fill-range": "^7.1.1"
-      },
-      "engines": {
-        "node": ">=8"
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
       }
     },
     "node_modules/buffer": {
@@ -646,26 +632,18 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+      "license": "MIT"
+    },
     "node_modules/consola": {
       "version": "2.15.0",
       "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.0.tgz",
       "integrity": "sha512-vlcSGgdYS26mPf7qNi+dCisbhiyDnrN1zaRbw3CSuc2wGOMEGGPsp46PdRG5gqXwgtJfjxDkxRNAgRPr1B77vQ==",
       "license": "MIT"
     },
-    "node_modules/cross-spawn": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
-      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
-      "license": "MIT",
-      "dependencies": {
-        "path-key": "^3.1.0",
-        "shebang-command": "^2.0.0",
-        "which": "^2.0.1"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/cssstyle": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz",
@@ -840,66 +818,6 @@
       "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
       "license": "MIT"
     },
-    "node_modules/execa": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
-      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
-      "license": "MIT",
-      "dependencies": {
-        "cross-spawn": "^7.0.3",
-        "get-stream": "^6.0.0",
-        "human-signals": "^2.1.0",
-        "is-stream": "^2.0.0",
-        "merge-stream": "^2.0.0",
-        "npm-run-path": "^4.0.1",
-        "onetime": "^5.1.2",
-        "signal-exit": "^3.0.3",
-        "strip-final-newline": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/execa?sponsor=1"
-      }
-    },
-    "node_modules/fast-glob": {
-      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
-      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
-      "license": "MIT",
-      "dependencies": {
-        "@nodelib/fs.stat": "^2.0.2",
-        "@nodelib/fs.walk": "^1.2.3",
-        "glob-parent": "^5.1.2",
-        "merge2": "^1.3.0",
-        "micromatch": "^4.0.8"
-      },
-      "engines": {
-        "node": ">=8.6.0"
-      }
-    },
-    "node_modules/fastq": {
-      "version": "1.20.3",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
-      "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
-      "license": "ISC",
-      "dependencies": {
-        "reusify": "^1.0.4"
-      }
-    },
-    "node_modules/fill-range": {
-      "version": "7.1.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
-      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
-      "license": "MIT",
-      "dependencies": {
-        "to-regex-range": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/find": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
@@ -910,9 +828,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
-      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
+      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
       "funding": [
         {
           "type": "individual",
@@ -959,6 +877,12 @@
         "node": ">=14.14"
       }
     },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+      "license": "ISC"
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -1014,28 +938,25 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/get-stream": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/glob-parent": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
-      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+    "node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
       "license": "ISC",
       "dependencies": {
-        "is-glob": "^4.0.1"
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
       },
       "engines": {
-        "node": ">= 6"
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
     "node_modules/gopd": {
@@ -1184,15 +1105,6 @@
         "node": ">= 6.0.0"
       }
     },
-    "node_modules/human-signals": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
-      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
-      "license": "Apache-2.0",
-      "engines": {
-        "node": ">=10.17.0"
-      }
-    },
     "node_modules/iconv-lite": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
@@ -1226,20 +1138,55 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/ignore": {
-      "version": "7.0.8",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.8.tgz",
-      "integrity": "sha512-YYNsSlXBjMk92SKnkwvB5LOVSa6OznlFUGcsvrFgNJbJCd0M1XKeFVRc8ZByeCqz32FivYNHJVooLmdqrmvp/Q==",
+      "version": "7.0.11",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.11.tgz",
+      "integrity": "sha512-YChdK5txDjwGUvgR7oCJcLGkwi3LDh8Zx8tS7ndYciLEg/oOCL26VUBSUEtgS7EjiybcBqFxh5j0rAdDX7/bbg==",
       "license": "MIT",
       "engines": {
         "node": ">= 4"
       }
     },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
+      "license": "ISC",
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "license": "ISC"
     },
+    "node_modules/interpret": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
+      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
+    "node_modules/is-core-module": {
+      "version": "2.17.0",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz",
+      "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==",
+      "license": "MIT",
+      "dependencies": {
+        "hasown": "^2.0.4"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/is-docker": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
@@ -1255,15 +1202,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/is-extglob": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -1273,18 +1211,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/is-glob": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
-      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
-      "license": "MIT",
-      "dependencies": {
-        "is-extglob": "^2.1.1"
-      },
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/is-interactive": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
@@ -1294,33 +1220,12 @@
         "node": ">=8"
       }
     },
-    "node_modules/is-number": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
-      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.12.0"
-      }
-    },
     "node_modules/is-potential-custom-element-name": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
       "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==",
       "license": "MIT"
     },
-    "node_modules/is-stream": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/is-unicode-supported": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz",
@@ -1345,12 +1250,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-      "license": "ISC"
-    },
     "node_modules/js-base64": {
       "version": "3.9.3",
       "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.3.tgz",
@@ -1532,34 +1431,6 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/merge-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
-      "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
-      "license": "MIT"
-    },
-    "node_modules/merge2": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
-      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 8"
-      }
-    },
-    "node_modules/micromatch": {
-      "version": "4.0.8",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
-      "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
-      "license": "MIT",
-      "dependencies": {
-        "braces": "^3.0.3",
-        "picomatch": "^2.3.1"
-      },
-      "engines": {
-        "node": ">=8.6"
-      }
-    },
     "node_modules/mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
@@ -1590,6 +1461,18 @@
         "node": ">=6"
       }
     },
+    "node_modules/minimatch": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
+      "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -1630,24 +1513,21 @@
         "is-wsl": "^2.2.0"
       }
     },
-    "node_modules/npm-run-path": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
-      "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
-      "license": "MIT",
-      "dependencies": {
-        "path-key": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/nwsapi": {
       "version": "2.2.27",
       "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.27.tgz",
       "integrity": "sha512-gQPNF78qebCQ6tvVFBYrvJdBNOrYZm90ZlXgpIFm06p6qHDHq/XC4TnJftN6OMbxVE0UTBAoRgcsDeJBBooITw==",
       "license": "MIT"
     },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "license": "ISC",
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
     "node_modules/onetime": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
@@ -1744,33 +1624,27 @@
         "url": "https://github.com/inikulin/parse5?sponsor=1"
       }
     },
-    "node_modules/path-key": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
       "license": "MIT",
       "engines": {
-        "node": ">=8"
+        "node": ">=0.10.0"
       }
     },
+    "node_modules/path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "license": "MIT"
+    },
     "node_modules/pend": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
       "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
       "license": "MIT"
     },
-    "node_modules/picomatch": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
-      "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/jonschlinkert"
-      }
-    },
     "node_modules/prompts": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
@@ -1820,26 +1694,6 @@
       "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
       "license": "MIT"
     },
-    "node_modules/queue-microtask": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
-      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "license": "MIT"
-    },
     "node_modules/readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -1854,6 +1708,17 @@
         "node": ">= 6"
       }
     },
+    "node_modules/rechoir": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
+      "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
+      "dependencies": {
+        "resolve": "^1.1.6"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -1869,6 +1734,27 @@
       "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
       "license": "MIT"
     },
+    "node_modules/resolve": {
+      "version": "1.22.12",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
+      "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "is-core-module": "^2.16.1",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/restore-cursor": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz",
@@ -1882,45 +1768,12 @@
         "node": ">=8"
       }
     },
-    "node_modules/reusify": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
-      "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
-      "license": "MIT",
-      "engines": {
-        "iojs": ">=1.0.0",
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/rrweb-cssom": {
       "version": "0.8.0",
       "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz",
       "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==",
       "license": "MIT"
     },
-    "node_modules/run-parallel": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
-      "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "license": "MIT",
-      "dependencies": {
-        "queue-microtask": "^1.2.2"
-      }
-    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1959,40 +1812,6 @@
         "node": ">=v12.22.7"
       }
     },
-    "node_modules/shebang-command": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-      "license": "MIT",
-      "dependencies": {
-        "shebang-regex": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/shebang-regex": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/shelljs": {
-      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.10.0.tgz",
-      "integrity": "sha512-Jex+xw5Mg2qMZL3qnzXIfaxEtBaC4n7xifqaqtrZDdlheR70OGkydrPJWT0V1cA1k3nanC86x9FwAmQl6w3Klw==",
-      "license": "BSD-3-Clause",
-      "dependencies": {
-        "execa": "^5.1.1",
-        "fast-glob": "^3.3.2"
-      },
-      "engines": {
-        "node": ">=18"
-      }
-    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -2049,15 +1868,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/strip-final-newline": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
-      "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/supports-color": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -2070,6 +1880,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/symbol-tree": {
       "version": "3.2.4",
       "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
@@ -2094,18 +1916,6 @@
       "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==",
       "license": "MIT"
     },
-    "node_modules/to-regex-range": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
-      "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-      "license": "MIT",
-      "dependencies": {
-        "is-number": "^7.0.0"
-      },
-      "engines": {
-        "node": ">=8.0"
-      }
-    },
     "node_modules/tough-cookie": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
@@ -2249,21 +2059,6 @@
         "node": ">=18"
       }
     },
-    "node_modules/which": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-      "license": "ISC",
-      "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/node-which"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@@ -2281,6 +2076,12 @@
         "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "license": "ISC"
+    },
     "node_modules/ws": {
       "version": "8.21.3",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",

sas/package.json

@@ -30,10 +30,11 @@
   },
   "private": true,
   "dependencies": {
-    "@sasjs/cli": "4.20.4",
+    "@sasjs/cli": "4.20.5",
     "@sasjs/core": "5.2.8"
   },
   "overrides": {
-    "nanoid": "3.3.18"
+    "nanoid": "3.3.18",
+    "shelljs": "0.8.5"
   }
 }

sas/sasjs/db/migrations/20261003_row_and_cell_limits.sas

@@ -0,0 +1,107 @@
+/**
+  @file
+  @brief migration script - raise the VIEW row limit, add the EDIT cell limit
+
+  OPTIONAL CHANGE - updates three options in MPE_CONFIG:
+
+  |option|old|new|
+  |---|---|---|
+  |DC_MAXOBS_WEBVIEW|500|2000|
+  |DC_MAXOBS_WEBEDIT|100|250|
+  |DC_MAXCELLS_WEBEDIT|(new)|200000|
+
+  MPE_CONFIG is bitemporal, so the current row for each option is closed and a
+  new row written with the new value.  A site that has already chosen its own
+  values for these options will have them replaced - edit the values below
+  first if that is not what you want.
+
+  An option that is switched off - a current row with var_active=0 - is left
+  alone, and a note is written to the log.  Writing an active row for it would
+  silently switch it back on.
+
+  Without this script the code defaults still apply (the same numbers), but
+  only for options that are not present at all - an existing install carries
+  its own rows, so it keeps 500 / 100 until this is run.
+
+**/
+
+%let dclib=YOURDCLIB;
+%let now=%sysfunc(datetime());
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+/* which of the three options the site has switched off */
+proc sql noprint;
+  select count(*) into :off_view trimmed
+    from &dclib..mpe_config
+    where var_name='DC_MAXOBS_WEBVIEW' and var_active=0 and &now lt tx_to;
+  select count(*) into :off_edit trimmed
+    from &dclib..mpe_config
+    where var_name='DC_MAXOBS_WEBEDIT' and var_active=0 and &now lt tx_to;
+  select count(*) into :off_cells trimmed
+    from &dclib..mpe_config
+    where var_name='DC_MAXCELLS_WEBEDIT' and var_active=0 and &now lt tx_to;
+quit;
+
+%if &off_view=0 %then %do;
+  proc sql noprint;
+    update &dclib..mpe_config
+      set tx_to=&now
+      where var_name='DC_MAXOBS_WEBVIEW' and var_active=1 and &now lt tx_to;
+
+    insert into &dclib..mpe_config
+      set tx_from=&now
+        ,tx_to='31DEC9999:23:59:59'dt
+        ,var_scope="DC"
+        ,var_name="DC_MAXOBS_WEBVIEW"
+        ,var_value='2000'
+        ,var_active=1
+        ,var_desc='This sets the maximum number of observations that can be'
+          !!' loaded into the browser in the VIEW screen.';
+  quit;
+%end;
+%else %put NOTE: DC_MAXOBS_WEBVIEW is switched off (var_active=0) - not changed;
+
+%if &off_edit=0 %then %do;
+  proc sql noprint;
+    update &dclib..mpe_config
+      set tx_to=&now
+      where var_name='DC_MAXOBS_WEBEDIT' and var_active=1 and &now lt tx_to;
+
+    insert into &dclib..mpe_config
+      set tx_from=&now
+        ,tx_to='31DEC9999:23:59:59'dt
+        ,var_scope="DC"
+        ,var_name="DC_MAXOBS_WEBEDIT"
+        ,var_value='250'
+        ,var_active=1
+        ,var_desc='This sets the maximum number of observations that can be'
+          !!' loaded into the browser for editing in the EDIT screen.  The'
+          !!' DC_MAXCELLS_WEBEDIT limit is applied at the same time, whichever'
+          !!' is reached first.';
+  quit;
+%end;
+%else %put NOTE: DC_MAXOBS_WEBEDIT is switched off (var_active=0) - not changed;
+
+%if &off_cells=0 %then %do;
+  proc sql noprint;
+    update &dclib..mpe_config
+      set tx_to=&now
+      where var_name='DC_MAXCELLS_WEBEDIT' and var_active=1 and &now lt tx_to;
+
+    insert into &dclib..mpe_config
+      set tx_from=&now
+        ,tx_to='31DEC9999:23:59:59'dt
+        ,var_scope="DC"
+        ,var_name="DC_MAXCELLS_WEBEDIT"
+        ,var_value='200000'
+        ,var_active=1
+        ,var_desc='This sets the maximum number of CELLS (rows multiplied by'
+          !!' columns) that can be loaded into the browser for editing in the'
+          !!' EDIT screen.  It exists because a wide table is expensive long'
+          !!' before it is tall - 250 rows of a 1000 column table is a quarter'
+          !!' of a million cells.  Whichever of this and DC_MAXOBS_WEBEDIT is'
+          !!' reached first applies.';
+  quit;
+%end;
+%else %put NOTE: DC_MAXCELLS_WEBEDIT is switched off (var_active=0) - not changed;

sas/sasjs/macros/mpe_accesscheck.sas

@@ -23,7 +23,7 @@
   @li mp_abort.sas
   @li mf_getuniquename.sas
   @li mf_getuser.sas
-  @li mf_verifymacvars.sas
+  @li mpe_validatecol.sas
   @li mpe_getgroups.sas
 
   <h4> Related Macros </h4>
@@ -52,10 +52,52 @@
     ,msg=%str(outds should be a WORK table)
   )
 
+  /**
+    * Validate inputs before they reach executable code.  base_table is
+    * interpolated into a SQL where clause (and callers may pass raw request
+    * input), so it must be a well-formed LIBREF.DATASET (or the
+    * LIBREF.CATALOGNAME-FC form of a format catalog) and access_level must
+    * be one of the known levels - anything else aborts before the query is
+    * built.  Values are read with symget (never re-resolved) and scanned in
+    * a data step so no macro content in the input can execute.
+    */
+  %local is_libds is_level;
+  %let is_libds=0;
+  %let is_level=0;
+  data _null_;
+    length _bt $64 _lvl $16;
+    _bt=symget('base_table');
+    _lvl=upcase(symget('access_level'));
+    %mpe_validatecol(_bt,LIBDS,is_libds)
+    if is_libds=0 then do;
+      call symputx('is_libds',0,'l');
+      putlog 'ERR' 'OR: Invalid base_table:' _bt;
+      stop;
+    end;
+    if _lvl not in ('EDIT','APPROVE','VIEW','SIGNOFF','AUDIT') then do;
+      call symputx('is_level',0,'l');
+      putlog 'ERR' 'OR: Invalid access_level:' _lvl;
+      stop;
+    end;
+    /* escape any embedded quotes so the value cannot break out of the
+      * double-quoted SQL literals below (defence in depth - the LIBDS
+      * check above already rejects quotes) */
+    _bt=tranwrd(_bt,'"','');
+    call symputx('base_table',_bt,'l');
+    call symputx('access_level',_lvl,'l');
+    call symputx('is_libds',is_libds,'l');
+    call symputx('is_level',1,'l');
+  run;
+
   %mp_abort(
-    iftrue=(%mf_verifymacvars(base_table user access_level)=0)
+    iftrue=(&is_libds ne 1)
     ,mac=mpe_accesscheck
-    ,msg=%str(Missing base_table/user access_level variables)
+    ,msg=%str(Invalid base_table)
+  )
+  %mp_abort(
+    iftrue=(&is_level ne 1)
+    ,mac=mpe_accesscheck
+    ,msg=%str(Invalid access_level)
   )
 
   /* make unique temp table vars */

sas/sasjs/macros/mpe_loader.sas

@@ -462,6 +462,32 @@ run;
   %return;
 %end;
 
+/* The post edit hook may have re-pointed the changeset at a different table -
+  * that is what lets an empty mirror stand in for a real one.  The target must
+  * itself be registered in MPE_TABLES: the approval screen resolves the table's
+  * audit settings from that row, and a changeset whose base table has no row
+  * cannot be reviewed.  Fail here, while the submitter is still watching. */
+%local target_chk;
+proc sql noprint;
+select count(*) into: target_chk
+  from &mpelib..mpe_tables
+  where tx_from le &now and &now lt tx_to
+    and upcase(libref)="%upcase(&libref)"
+    and upcase(dsn)="%upcase(&ds)";
+%if &target_chk=0 %then %do;
+  %let msg=%str(ERR)OR: target table &libref..&ds is not registered in%trim(
+    ) &mpelib..mpe_tables - a post edit hook may only route a changeset to a%trim(
+    ) registered table;
+  %mpe_loadfail(
+      status=FAILED - TARGET NOT REGISTERED
+    ,now=&now
+    ,mperef=&mperef
+    ,reason_txt=%quote(&msg)
+    ,dc_dttmtfmt=&dc_dttmtfmt.
+  )
+  %return;
+%end;
+
 
 /**
   * send to approve process

sas/sasjs/macros/mpe_makedata.sas

@@ -72,12 +72,35 @@ insert into &lib..mpe_config set
     ,tx_to='31DEC9999:23:59:59'dt
     ,var_scope="DC"
     ,var_name="DC_MAXOBS_WEBEDIT"
-    ,var_value="100"
+    ,var_value="250"
     ,var_active=1
     ,var_desc='This sets the maximum number of observations that can be loaded'
       !!' into the browser for editing in the EDIT screen.  A higher number'
       !!' will require a decent browser (ie, not IE) and more memory on the'
-      !!' client side.';
+      !!' client side.  The DC_MAXCELLS_WEBEDIT limit is applied at the same'
+      !!' time, whichever is reached first.';
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC"
+    ,var_name="DC_MAXCELLS_WEBEDIT"
+    ,var_value="200000"
+    ,var_active=1
+    ,var_desc='This sets the maximum number of CELLS (rows multiplied by'
+      !!' columns) that can be loaded into the browser for editing in the EDIT'
+      !!' screen.  It exists because a wide table is expensive long before it'
+      !!' is tall - 250 rows of a 1000 column table is a quarter of a million'
+      !!' cells.  Whichever of this and DC_MAXOBS_WEBEDIT is reached first'
+      !!' applies.';
+insert into &lib..mpe_config set
+    tx_from=0
+    ,tx_to='31DEC9999:23:59:59'dt
+    ,var_scope="DC"
+    ,var_name="DC_MAXOBS_WEBVIEW"
+    ,var_value="2000"
+    ,var_active=1
+    ,var_desc='This sets the maximum number of observations that can be'
+      !!' loaded into the browser in the VIEW screen.';
 insert into &lib..mpe_config set
     tx_from=0
     ,tx_to='31DEC9999:23:59:59'dt

sas/sasjs/macros/mpe_validatecol.sas

@@ -0,0 +1,47 @@
+/**
+  @file
+  @brief Validates a column of values, including the Data Controller
+    format-catalog form of a libds reference
+  @details Wrapper around mp_validatecol() that adds the Data Controller
+    convention of addressing a format catalog as `LIBREF.CATALOGNAME-FC`
+    to the LIBDS rule.
+
+    The `-FC` suffix is matched exactly and the remainder is validated as
+    a strict LIBREF.DATASET, so the *whole* value is covered by the
+    validation - a bare scan on the dash would leave anything after it
+    unvalidated.  The input column itself is never modified, so a caller
+    that needs the catalog reference downstream (to match MPE_SECURITY,
+    for instance) still receives it.
+
+  @param [in] incol Input column (a data step variable)
+  @param [in] rule Validation rule, as per mp_validatecol()
+  @param [out] outcol 1 when the value is valid, else 0
+
+  <h4> SAS Macros </h4>
+  @li mp_validatecol.sas
+  @li mf_getuniquename.sas
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+**/
+
+%macro mpe_validatecol(incol,rule,outcol);
+
+%if &rule=LIBDS %then %do;
+  /* tempcol is given a unique name with every invocation */
+  %local tempcol;
+  %let tempcol=%mf_getuniquename(prefix=cat);
+  &tempcol=strip(&incol);
+  /* permit the format-catalog form: LIBREF.CATALOGNAME-FC, exactly */
+  if length(&tempcol)>3
+    and upcase(substr(&tempcol,length(&tempcol)-2,3))='-FC'
+  then &tempcol=substr(&tempcol,1,length(&tempcol)-3);
+  %mp_validatecol(&tempcol,LIBDS,&outcol)
+  drop &tempcol;
+%end;
+%else %mp_validatecol(&incol,&rule,&outcol);
+
+%mend mpe_validatecol;

sas/sasjs/macros/mpe_validatecol.test.sas

@@ -0,0 +1,72 @@
+/**
+  @file
+  @brief testing the mpe_validatecol macro (LIBDS rule)
+  @details The LIBDS rule must accept the Data Controller format-catalog
+  reference (LIBREF.CATALOGNAME-FC) and reject everything else - including
+  a value that merely *starts* with a valid libds and continues past the
+  dash, which a bare scan on the dash would let through.
+
+  <h4> SAS Macros </h4>
+  @li mpe_validatecol.sas
+  @li mp_assertdsobs.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+data work.check;
+  length val $64;
+  /* valid: a plain libds */
+  exp=1; val='WORK.CLASS';              output;
+  exp=1; val='DC.MPE_TABLES';           output;
+  exp=1; val='_A._B';                   output;
+  /* valid: the format-catalog form, however the suffix is cased */
+  exp=1; val='DCTEST.DCFMTS-FC';        output;
+  exp=1; val='dctest.dcfmts-fc';        output;
+  exp=1; val='WORK.CLASS-FC';           output;
+  /* valid: padded values are trimmed before the check */
+  exp=1; val='WORK.CLASS   ';           output;
+  /* invalid: not a libds at all */
+  exp=0; val='WORK';                    output;
+  exp=0; val='WORK.CLASS.NOPE';         output;
+  exp=0; val='';                        output;
+  exp=0; val='-FC';                     output;
+  exp=0; val='../secprobe';             output;
+  /* invalid: the dash suffix is not exactly -FC */
+  exp=0; val='WORK.CLASS-FCX';          output;
+  exp=0; val='WORK.CLASS-C';            output;
+  exp=0; val='WORK.CLASS-FC X';         output;
+  /* invalid: content smuggled past a valid libds prefix */
+  exp=0; val="WORK.CLASS-FC'";          output;
+  exp=0; val='WORK.CLASS-FC;proc sql;'; output;
+  exp=0; val='WORK.CLASS-FC) or 1=1';   output;
+  exp=0; val='WORK.CLASS-fc-';          output;
+run;
+
+data work.check;
+  set work.check;
+  is_libds=0;
+  %mpe_validatecol(val,LIBDS,is_libds)
+  got=is_libds;
+  if got ne exp then putlog 'ERR' 'OR: unexpected result for [' val +(-1) ']';
+run;
+
+data work.mismatch;
+  set work.check;
+  where got ne exp;
+run;
+
+%mp_assertdsobs(work.mismatch,
+  desc=Every LIBDS value validated as expected (catalog form permitted, nothing smuggled past the dash),
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* dump for offline inspection */
+data _null_;
+  set work.check;
+  putlog 'TEST_RESULT_LINE: [' val +(-1) '] exp=' exp 'got=' got;
+run;

sas/sasjs/macros/mpeinit.sas

@@ -115,9 +115,34 @@ run;
   ,msg=%str(Problem during compilation or with STP precode (&syswarningtext))
 )
 
+/* `_debug` is 131 by default, but the adapter sends 128 on the Viya WEB JES
+  * path when runAsTask is enabled (see WebJobExecutor.getRequestParams), and
+  * 128 also arrives when a `log` debug value is requested.  All of them mean
+  * "debug on" - so all of them enable the extra logging below. */
 %if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131"
+  or "&_debug"="128"
 %then %do;
   %let sasjs_mdebug=1;
 %end;
 
+%if "&sasjs_mdebug"="1" %then %do;
+  %if %sysfunc(exist(work.browser_url_vars)) %then %do;
+    data _null_;
+      length name value $1024;
+      set work.browser_url_vars;
+      putlog "NOTE: &sysmacroname: work.browser_url_vars:";
+      putlog name= value=;
+    run;
+  %end;
+  %else %put NOTE: &sysmacroname: no work.browser_url_vars on this request;
+  %if %sysfunc(exist(work.browser_info)) %then %do;
+    data _null_;
+      set work.browser_info;
+      putlog "NOTE: &sysmacroname: work.browser_info:";
+      putlog (_all_)(=);
+    run;
+  %end;
+  %else %put NOTE: &sysmacroname: no work.browser_info on this request;
+%end;
+
 %mend mpeinit;

sas/sasjs/services/admin/dirlist.sas

@@ -5,8 +5,13 @@
 
     @li &parent= (parent path)
 
+  Requires membership of the DC administrators group.
+
   <h4> SAS Macros </h4>
+  @li mf_getuser.sas
+  @li mp_abort.sas
   @li mp_dirlist.sas
+  @li mpe_getgroups.sas
 
   @version 9.2
   @author 4GL Apps Ltd
@@ -17,8 +22,37 @@
 **/
 
 %global parent;
-/* if no flavour is specified, default to root */
-%let parent=%sysfunc(coalescec(&parent,/));
+%mpeinit()
+
+/* check user is in admin group */
+%let cnt=0;
+%mpe_getgroups(user=%mf_getuser(),outds=work.usergroups)
+proc sql noprint;
+select count(*) into:cnt
+  from usergroups
+  where groupname="&mpeadmins";
+%mp_abort(iftrue= (&cnt=0)
+  ,mac=&_program
+  ,msg=%str(This service is only available to &mpeadmins members)
+)
+
+/* if no parent is specified, default to root - read with symget
+  * (never re-resolved) so macro content in the param cannot execute */
+%let is_bad=0;
+data _null_;
+length _parent $512;
+_parent=coalescec(symget('parent'),'/');
+  if index(_parent,'%')>0 or index(_parent,'&')>0 then do;
+    putlog 'ERR' 'OR: Invalid parent:' _parent;
+    call symputx('is_bad',1,'l');
+  end;
+  else call symputx('parent',_parent,'g');
+run;
+
+%mp_abort(iftrue=(&is_bad=1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid parent)
+)
 
 %mp_dirlist(path=&parent,outds=dirlist, maxdepth=2)
 

sas/sasjs/services/admin/dirlist.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief testing admin dirlist service - admin gate (security)
+  @details The service requires membership of the DC administrators
+  group.  The test suite runs as a member of that group, so the gate
+  passes and the directory listing is returned.  (A non-admin negative
+  test would need a second user, which this suite does not have.)
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+
+**/
+
+%let _program=&appLoc/services/admin/dirlist;
+
+data work.params;
+  length name $32 value $1000;
+  name='parent';value='/tmp';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params,
+  outlib=web1
+)
+
+%let nobs=0;
+proc sql noprint;
+select count(*) into: nobs from web1.dirlist;
+quit;
+
+%mp_assert(
+  iftrue=(&nobs>0),
+  desc=Admin user gets a directory listing,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/admin/exportconfig.sas

@@ -15,6 +15,7 @@
   @li mp_ds2csv.sas
   @li mp_streamfile.sas
   @li mp_validatecol.sas
+  @li mpe_getgroups.sas
 
   @author 4GL Apps Ltd
   @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
@@ -26,6 +27,18 @@
 %global dclib islib newlib;
 %mpeinit()
 
+/* check user is in admin group */
+%let cnt=0;
+%mpe_getgroups(user=%mf_getuser(),outds=work.usergroups)
+proc sql noprint;
+select count(*) into:cnt
+  from usergroups
+  where groupname="&mpeadmins";
+%mp_abort(iftrue= (&cnt=0)
+  ,mac=&_program
+  ,msg=%str(The DC configuration can only be exported by &mpeadmins members)
+)
+
 data _null_;
   newlib=coalescec(symget('dclib'),"&mpelib");
   %mp_validatecol(newlib,ISLIB,islib)

sas/sasjs/services/admin/exportdb.sas

@@ -4,9 +4,12 @@
   @details If user is in the administrator group, they can call this
   service directly adding the following URL params:
 
-    @li &flavour= (only PGSQL supported at this time)
+    @li &flavour= (SAS, PGSQL or TSQL - defaults to SAS)
     @li &schema= (optional, if target schema is needed)
 
+  Data inserts are generated for the SAS and PGSQL flavours only; the TSQL
+  export contains DDL without inserts.
+
   <h4> SAS Macros </h4>
   @li mf_getuser.sas
   @li mp_abort.sas

sas/sasjs/services/admin/exportdb.test.sas

@@ -72,3 +72,34 @@ run;
   test=EQUALS 1,
   outds=work.test_results
 )
+
+/* test 3 - TSQL flavour (DDL only, no inserts) */
+data work.params;
+  length name $32 value $1000;
+  name='flavour';value='TSQL';output;
+  name='schema';value='public';output;
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params,
+  outref=web3,
+  viyaresult=WEBOUT_TXT,
+  mdebug=&sasjs_mdebug
+)
+
+data work.results3;
+  infile web3;
+  input;
+  putlog _infile_;
+  if index(upcase(_infile_),'CREATE TABLE') then do;
+    putlog 'test passed';
+    output;
+    stop;
+  end;
+run;
+
+%mp_assertdsobs(work.results3,
+  desc=TSQL flavour DDL file is successfully returned,
+  test=EQUALS 1,
+  outds=work.test_results
+)

sas/sasjs/services/admin/refreshcatalog.sas

@@ -2,10 +2,15 @@
   @file refreshcatalog.sas
   @brief Refreshes the library data catalog
   @details  A library may be passed in a LIBREF url param.
+  Requires membership of the DC administrators group.
 
   <h4> SAS Macros </h4>
   @li mpeinit.sas
   @li dc_refreshcatalog.sas
+  @li mf_getuser.sas
+  @li mpe_getgroups.sas
+  @li mp_abort.sas
+  @li mp_validatecol.sas
   @li mpeterm.sas
 
   @version 9.3
@@ -18,6 +23,44 @@
 %global libref;
 %mpeinit()
 
+/**
+  * libref is a request input used in dc_assignlib and catalog queries -
+  * it must be a well-formed libref before use.  Read with symget (never
+  * re-resolved) and validated in a data step.
+  */
+%let is_lib=0;
+data _null_;
+  length _libref $8;
+  _libref=coalescec(symget('libref'),'');
+  /* an absent libref is a valid, full catalog refresh */
+  if missing(_libref) then do;
+    call symputx('is_lib',1,'l');
+    call symputx('libref','','g');
+    stop;
+  end;
+  %mp_validatecol(_libref,ISLIB,is_lib)
+  if is_lib=0 then putlog 'ERR' 'OR: Invalid libref:' _libref;
+  call symputx('is_lib',is_lib,'l');
+  if is_lib=1 then call symputx('libref',upcase(_libref),'g');
+run;
+
+%mp_abort(iftrue= (&is_lib ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid libref)
+)
+
+/* check user is in admin group */
+%let cnt=0;
+%mpe_getgroups(user=%mf_getuser(),outds=work.usergroups)
+proc sql noprint;
+select count(*) into:cnt
+  from usergroups
+  where groupname="&mpeadmins";
+%mp_abort(iftrue= (&cnt=0)
+  ,mac=&_program
+  ,msg=%str(This service is only available to &mpeadmins members)
+)
+
 %dc_refreshcatalog(&libref)
 
 

sas/sasjs/services/admin/refreshcatalog.test.1.sas

@@ -0,0 +1,78 @@
+/**
+  @file
+  @brief testing admin refreshcatalog service - admin gate + libref validation (security)
+  @details The service requires membership of the DC administrators
+  group and a well-formed libref (or none).  An invalid libref aborts
+  the service, which shows up as a canceled child job (an aborted
+  service registers no webout).  The test suite runs as a member of
+  the admin group, so the gate passes here.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+
+**/
+
+%let _program=&appLoc/services/admin/refreshcatalog;
+
+/**
+  * Test 1 - an invalid libref must abort the service
+  */
+data work.params1;
+  length name $32 value $1000;
+  name='libref';value='A.%sysevalf(3+4)B';output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params1,
+  outref=web1,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=Macro content in libref aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - a valid libref still refreshes the catalog (admin user)
+  */
+data work.params2;
+  length name $32 value $1000;
+  name='libref';value='DCTEST';output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params2,
+  outlib=web2
+)
+
+%let msgcheck=0;
+data _null_;
+  set web2.sasparams;
+  putlog (_all_)(=);
+  if index(msg,'Catalog Refresh Complete') then call symputx('msgcheck',1);
+run;
+
+%mp_assert(
+  iftrue=(&msgcheck=1),
+  desc=Valid libref refresh completes for admin user,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/admin/refreshlibs.sas

@@ -1,11 +1,14 @@
 /**
   @file refreshlibs.sas
   @brief Refreshes the library data catalog
-  @details
+  @details Requires membership of the DC administrators group.
 
   <h4> SAS Macros </h4>
   @li mpeinit.sas
   @li mpe_refreshlibs.sas
+  @li mf_getuser.sas
+  @li mpe_getgroups.sas
+  @li mp_abort.sas
 
   @version 9.3
   @author 4GL Apps Ltd
@@ -17,4 +20,16 @@
 
 %mpeinit()
 
+/* check user is in admin group */
+%let cnt=0;
+%mpe_getgroups(user=%mf_getuser(),outds=work.usergroups)
+proc sql noprint;
+select count(*) into:cnt
+  from usergroups
+  where groupname="&mpeadmins";
+%mp_abort(iftrue= (&cnt=0)
+  ,mac=&_program
+  ,msg=%str(This service is only available to &mpeadmins members)
+)
+
 %mpe_refreshlibs()

sas/sasjs/services/auditors/getdiffs.sas

@@ -7,9 +7,11 @@
   @li mpe_getvars.sas
   @li mpe_accesscheck.sas
   @li mf_getattrn.sas
+  @li mf_getuser.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_streamfile.sas
+  @li mp_validatecol.sas
 
   @version 9.2
   @author 4GL Apps Ltd
@@ -22,6 +24,52 @@
 %mpeinit()
 %mpe_getvars(BrowserParams, BrowserParams);
 
+/**
+  * Validate inputs before they reach executable code.  libds is passed to
+  * the access check (which interpolates it into SQL) and table is used in
+  * the staging file path, so both must be well formed before use.  Values
+  * are re-read with symget (never re-resolved) and validated in a data
+  * step so no macro content in the request can execute.
+  */
+%let is_libds=0;
+%let is_table=0;
+%let is_csv=0;
+data _null_;
+  length _libds $64 _table $64 _csv $128;
+  _libds=symget('libds');
+  _table=symget('table');
+  _csv=coalescec(symget('stp_diffs_csv'),'tempDiffs.csv');
+  %mp_validatecol(_libds,LIBDS,is_libds)
+  %mp_validatecol(_table,ISNAME,is_table)
+  /* the diffs csv filename must stay inside the staging directory */
+  if findc(_csv,'/\')>0 or index(_csv,'..')>0 then do;
+    is_csv=0;
+    putlog 'ERR' 'OR: Invalid stp_diffs_csv:' _csv;
+  end;
+  else is_csv=1;
+  if is_libds=0 then putlog 'ERR' 'OR: Invalid libds:' _libds;
+  if is_table=0 then putlog 'ERR' 'OR: Invalid table:' _table;
+  call symputx('is_libds',is_libds,'l');
+  call symputx('is_table',is_table,'l');
+  call symputx('is_csv',is_csv,'l');
+  if is_libds=1 then call symputx('libds',_libds,'g');
+  if is_table=1 then call symputx('table',_table,'g');
+  if is_csv=1 then call symputx('stp_diffs_csv',_csv,'g');
+run;
+
+%mp_abort(iftrue= (&is_libds ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid libds)
+)
+%mp_abort(iftrue= (&is_table ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid table)
+)
+%mp_abort(iftrue= (&is_csv ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid stp_diffs_csv)
+)
+
 /* security checks */
 %let user=%mf_getuser();
 %mpe_accesscheck(&libds,outds=authEDIT,user=&user,access_level=EDIT)
@@ -51,5 +99,4 @@
 %mpestp_diffs()
 
 
-
 %mpeterm()

sas/sasjs/services/auditors/getdiffs.test.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief testing getdiffs service - input validation (security)
+  @details The libds, table and stp_diffs_csv request params must be
+  well-formed before they reach the access check and the staging file
+  path.  An invalid value aborts the service, which shows up as a
+  canceled child job (an aborted service registers no webout).
+
+  A real load is staged first (stagedata) and a diffs csv written into
+  the staging directory, so every payload below resolves to that REAL
+  file when executed - on a vulnerable service the job completes, and
+  only the validating service cancels it.  The assertions cannot pass
+  against a service that does not validate.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%let _program=&appLoc/services/auditors/getdiffs;
+
+/**
+  * Stage a real load so a real staging directory exists
+  */
+data work.sascontroltable;
+  action='LOAD';
+  message="getdiffs test prep";
+  libds="&dclib..MPE_X_TEST";
+  output;
+  stop;
+run;
+
+proc sql noprint;
+select max(primary_key_field) into: maxpk
+  from &dclib..mpe_x_test;
+quit;
+
+data work.jsdata;
+  set &dclib..mpe_x_test(rename=(
+    some_date=dt2 SOME_DATETIME=dttm2 some_time=tm2)
+  );
+  some_date=put(dt2,date9.);
+  SOME_DATETIME=put(dttm2,datetime19.);
+  some_time=put(tm2,time.);
+  drop dt2 dttm2 tm2;
+  if _n_=1 then do;
+    _____DELETE__THIS__RECORD_____='No';
+    some_char='getdiffs security test';
+    some_num=&maxpk+1;
+  end;
+  else stop;
+run;
+
+%mx_execute(&appLoc/services/editors/stagedata,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.jsdata work.sascontroltable,
+  outlib=webstage,
+  mdebug=&sasjs_mdebug
+)
+
+%let stagetest=0;
+data _null_;
+  set webstage.sasparams;
+  putlog (_all_)(=);
+  if status='SUCCESS' then call symputx('stagetest',1);
+  call symputx('loadref',dsid);
+run;
+
+%mp_assert(
+  iftrue=(&stagetest=1 and &syscc=0),
+  desc=stagedata succeeded in getdiffs prep,
+  outds=work.test_results
+)
+
+/**
+  * Write the diffs csv into the real staging directory
+  */
+%let diffscsv=tempDiffs_secrev.csv;
+data _null_;
+  file "&dc_staging_area/&loadref./&diffscsv";
+  put 'SOME_CHAR,_____STATUS_____';
+  put 'getdiffs security test,UPDATED';
+run;
+
+/**
+  * Test 1 - the mpe_accesscheck SQL injection payload in libds must
+  * abort the service (validation fires before the authz query, so
+  * the authz bypass cannot happen).  The payload is sent through the
+  * sasjs table channel (BrowserParams) like the frontend does - the
+  * raw-quote form is masked in plain URL params on this platform.
+  */
+%let fb1=%mf_getuniquefileref();
+data _null_;
+  file &fb1 termstr=crlf;
+  length _row $400.;
+  put 'TABLE:$41. STP_DIFFS_CSV:$100. libds:$41.';
+  _row=cats(symget('loadref'),',',symget('diffscsv'),',',
+    'SOMELIB.SOMEDS',"'22'x"," or ","'22'x",'1',"'22'x",' ne ',"'22'x",'2');
+  put _row;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&fb1:BrowserParams,
+  outref=web1,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=SQL injection payload in libds aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - path traversal in table must abort the service (the
+  * payload resolves to the real staged file through a .. detour)
+  */
+data _null_;
+  length _dir $512;
+  _dir=scan(symget('dc_staging_area'),-1,'/');
+  call symputx('travtable',cats('../',_dir,'/','&loadref'));
+run;
+
+%let fb2=%mf_getuniquefileref();
+data _null_;
+  file &fb2 termstr=crlf;
+  put 'TABLE:$41. STP_DIFFS_CSV:$100. libds:$41.';
+  put "&travtable.,&diffscsv.,&dclib..MPE_X_TEST";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&fb2:BrowserParams,
+  outref=web2,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort2=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort2',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort2=1),
+  desc=Path traversal in table aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - path traversal in stp_diffs_csv must abort the service
+  */
+%let fb3=%mf_getuniquefileref();
+data _null_;
+  file &fb3 termstr=crlf;
+  put 'TABLE:$41. STP_DIFFS_CSV:$100. libds:$41.';
+  put "&loadref.,../&loadref./&diffscsv.,&dclib..MPE_X_TEST";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&fb3:BrowserParams,
+  outref=web3,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort3=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort3',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort3=1),
+  desc=Path traversal in stp_diffs_csv aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/editors/getdata.sas

@@ -205,7 +205,14 @@ select upcase(loadtype)
     %put NOTE- Please add to &mpelib..MPE_CONFIG table;
     %put NOTE-;%put NOTE-;
     %global DC_MAXOBS_WEBEDIT;
-    %let DC_MAXOBS_WEBEDIT=500;
+    %let DC_MAXOBS_WEBEDIT=250;
+  %end;
+  %if not %symexist(DC_MAXCELLS_WEBEDIT) %then %do;
+    %put NOTE:;%put NOTE- DC_MAXCELLS_WEBEDIT not found!;
+    %put NOTE- Please add to &mpelib..MPE_CONFIG table;
+    %put NOTE-;%put NOTE-;
+    %global DC_MAXCELLS_WEBEDIT;
+    %let DC_MAXCELLS_WEBEDIT=200000;
   %end;
   /* for tables which use RKs/SKs then we just expose the business key to
       users - this lets uploads be sent to multiple environments (with
@@ -254,18 +261,31 @@ select upcase(loadtype)
     ,msg=%str(Issue with filtering (line 165) )
   )
 
-  options obs=&DC_MAXOBS_WEBEDIT;
+  /* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
+    then run the hook, then check the row and cell limits.  Checking once,
+    after both, means the counts cover everything that could reach the
+    browser: the rows the selection matched, and anything the hook has added
+    to work.out.  A wide table exhausts the cell budget long before it
+    reaches the row limit - 250 rows of a 1000 column table is a quarter of
+    a million cells - so capping rows alone does not bound the work the
+    browser is asked to do.  The payload itself is capped further down, when
+    work.outdata is built. */
   %let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
   proc sort data=work.out; by &sortPK; run;
-  options obs=max;
 
   %mpe_runhook(PRE_EDIT_HOOK)
 
   %let obscnt=%mf_getattrn(work.out,NLOBS);
+  %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
+  %let cellcnt=%eval(&obscnt * &colcnt);
   %mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
     ,mac=&_program
     ,msg=Table is too big (&obscnt rows) - please filter and try again!
   )
+  %mp_abort(iftrue=(&cellcnt>&DC_MAXCELLS_WEBEDIT)
+    ,mac=&_program
+    ,msg=Selection is too wide (&obscnt rows x &colcnt cols) - please filter and try again!
+  )
 
 
   /* order delete var and pk fields at start of table */
@@ -274,6 +294,9 @@ select upcase(loadtype)
                             ,Str2= _____DELETE__THIS__RECORD_____ &pk
                           );
   %put sourcevars=&sourcevars;
+  /* the payload is capped here, at the row limit - the limits above are
+    checked against the whole selection, this is what goes to the browser */
+  options obs=&DC_MAXOBS_WEBEDIT;
   data outdata;
     /* delete & pk fields come first */
     attrib _____DELETE__THIS__RECORD_____ &pk label='';
@@ -288,6 +311,7 @@ select upcase(loadtype)
     %end;
     set work.out ;
   run;
+  options obs=max;
 
 
 

sas/sasjs/services/public/getcols.sas

@@ -5,8 +5,9 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
-  @li mf_getvalue.sas
   @li mp_abort.sas
+  @li mp_getcols.sas
+  @li mp_validatecol.sas
 
   @version 9.2
   @author 4GL Apps Ltd
@@ -18,35 +19,58 @@
 %mpeinit()
 
 
-%let ds=%mf_getvalue(work.iwant,libds);
+/**
+  * The libds is read from the IWANT input table.  Reading it with
+  * mf_getvalue would re-resolve any macro content in the value, so it is
+  * read with symget in a data step and validated (LIBREF.DATASET) before
+  * it is used in proc contents.
+  */
+%let is_libds=0;
+data _null_;
+  length _libds $64;
+  set work.iwant;
+  _libds=libds;
+  %mp_validatecol(_libds,LIBDS,is_libds)
+  if is_libds=0 then putlog 'ERR' 'OR: Invalid libds:' _libds;
+  call symputx('is_libds',is_libds,'l');
+  if is_libds=1 then call symputx('ds',upcase(_libds),'l');
+  stop;
+run;
+
+%mp_abort(iftrue= (&is_libds ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid libds)
+)
+
 %dc_assignlib(READ,%scan(&ds,1,.))
 
-proc contents noprint data=&ds
-  out=droplist1 (keep=name type length label varnum format:);
-run;
-data cols(keep=name type length varnum format label);
-  set droplist1(rename=(format=format2 type=type2));
-  name=upcase(name);
-  if type2=2 then do;
-    length format $49.;
-    if format2='' then format=cats('$',length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else format=cats(format2,formatl,'.');
-    type='C';
-    ddtype='C';
-  end;
-  else do;
-    if format2='' then format=cats(length,'.');
-    else if formatl=0 then format=cats(format2,'.');
-    else if formatd=0 then format=cats(format2,formatl,'.');
-    else format=cats(format2,formatl,'.',formatd);
-    type='N';
-    if format=:'DATETIME' then ddtype='DATETIME';
-    else if format=:'DATE' then ddtype='DATE';
-    else if format=:'TIME' then ddtype='TIME';
-    else ddtype='N';
-  end;
-  if label='' then label=name;
+/* Column metadata, typed the way the editor's own load types it: %mp_getcols
+  infers DATE and DATETIME across the whole format list (YYMMDD, MMDDYY,
+  E8601DA, B8601DA, NLDATE, NLDATM, E8601DT, ...), which a test against the
+  format prefix does not - a column formatted YYMMDD would otherwise reach the
+  picker as numeric.  getdata.sas makes the same call for the same reason. */
+%mp_getcols(&ds, outds=cols1)
+
+data cols(keep=name type length varnum fmtname ddtype);
+  /* type arrives from %mp_getcols as a $1 holding 'C' or 'N', so it has to be
+    widened before the client vocabulary below is assigned to it: a $1
+    variable truncates 'num' and 'char' to 'n' and 'c', and the picker
+    compares them exactly (query.component for the operator set, sas-store
+    for the quoting).  ddtype needs no statement - it is created by the macro
+    with 'CHARACTER', so it is already $9. */
+  length type $4;
+  set cols1;
+  /* DDTYPE in the client's vocabulary, as getdata.sas maps it */
+  if ddtype='CHARACTER' then ddtype='C';
+  else if ddtype='NUMERIC' then ddtype='N';
+  /* TYPE in the client's vocabulary too.  On the normal path mergeColsRules()
+    sets it from the loaded table's formats - 'num' for the date and time
+    types, the column's own type otherwise - and the picker keys both its
+    operator set (query.component) and its value quoting (sas-store) on it.
+    The fallback list never reaches that pass, because the load that would
+    have fed it is the one that failed, so the service has to carry it. */
+  if ddtype in ('DATE','DATETIME','TIME') or ddtype='N' then type='num';
+  else type='char';
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)

sas/sasjs/services/public/getcols.test.sas

@@ -0,0 +1,103 @@
+/**
+  @file
+  @brief testing getcols service - input validation (security)
+  @details The libds in the IWANT input table must be a well-formed
+  LIBREF.DATASET.  An invalid value aborts the service before it
+  reaches proc contents.  The abort shows up as a canceled child job
+  (an aborted service registers no webout).
+
+  The payload in test 1 resolves to a REAL table when the request
+  content is executed as macro code, so on a vulnerable service the
+  job completes, and only the validating service cancels it - the
+  assertion cannot pass against a service that does not validate.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%let _program=&appLoc/services/public/getcols;
+
+/**
+  * Test 1 - macro content in libds must abort the service
+  */
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'LIBDS:$41.';
+  put '%sysfunc(coalescec(&dclib..MPE_X_TEST,))';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=web1,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=Macro content in libds aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - valid libds still returns columns
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put 'LIBDS:$41.';
+  put "&dclib..MPE_X_TEST";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:iwant,
+  outlib=web2
+)
+
+%let nobs=0;
+%let nbad=0;
+proc sql noprint;
+select count(*) into: nobs from web2.cols;
+/**
+  * The vocabulary the picker matches on, and the mapping it is derived from.
+  * TYPE is assigned in a step that SETs the %mp_getcols output, where type is
+  * a $1 holding 'C'/'N' - without the LENGTH statement that widens it, the
+  * values truncate to 'n'/'c' and the client's exact comparisons (query
+  * component for the operator set, sas-store for the quoting) stop matching.
+  * The mock derives TYPE in JS, so only this suite can catch that.
+  */
+select count(*) into: nbad from web2.cols
+  where (ddtype='C' and type ne 'char')
+  or (ddtype in ('N','DATE','DATETIME','TIME') and type ne 'num');
+quit;
+
+%mp_assert(
+  iftrue=(&nobs>0),
+  desc=Valid libds returns columns,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&nbad=0),
+  desc=TYPE vocabulary intact - every column is num or char, per its ddtype,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/public/getcolvals.sas

@@ -35,7 +35,6 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
-  @li mf_getvalue.sas
   @li mf_verifymacvars.sas
   @li dc_assignlib.sas
   @li mf_getvarformat.sas
@@ -43,6 +42,8 @@
   @li mp_cntlout.sas
   @li mp_filtercheck.sas
   @li mp_filtergenerate.sas
+  @li mp_validatecol.sas
+  @li mpe_validatecol.sas
 
   @version 9.2
   @author 4GL Apps Ltd.
@@ -77,18 +78,48 @@ data _null_;
   put (_all_)(=);
 run;
 
-%let libds=%mf_getvalue(work.iwant,libds);
-%let col2=%mf_getvalue(work.iwant,col);
+/**
+  * libds and col are request inputs that flow into executable positions
+  * (set &libds, proc sql select &col2).  They are read from the IWANT
+  * table with symget in a data step (never re-resolved) and validated
+  * here before use - mf_getvalue would re-resolve any macro content in
+  * the value before this code ran.
+  */
+%let libds=;
+%let col2=;
+%let is_libds=0;
+%let is_col=0;
+data _null_;
+  length _libds $64 _col $32;
+  set work.iwant;
+  _libds=libds;
+  _col=col;
+  %mpe_validatecol(_libds,LIBDS,is_libds)
+  %mp_validatecol(_col,ISNAME,is_col)
+  if is_libds=0 then putlog 'ERR' 'OR: Invalid libds:' _libds;
+  if is_col=0 then putlog 'ERR' 'OR: Invalid col:' _col;
+  call symputx('is_libds',is_libds,'l');
+  call symputx('is_col',is_col,'l');
+  if is_libds=1 then call symputx('libds',upcase(_libds),'l');
+  if is_col=1 then call symputx('col2',upcase(_col),'l');
+  stop;
+run;
+
 %let is_fmt=0;
 %let startrow=1;
 %let rows=4000;
 
-%put &=libds;
-%put &=col2;
-
 %mp_abort(iftrue= (%mf_verifymacvars(libds col2)=0)
   ,mac=&_program..sas
-  ,msg=%str(Missing inputs from iwant. Libds=&libds col=&col2 )
+  ,msg=%str(Missing inputs from iwant)
+)
+%mp_abort(iftrue= (&is_libds ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid libds)
+)
+%mp_abort(iftrue= (&is_col ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid col)
 )
 
 %dc_assignlib(WRITE,%scan(&libds,1,.))

sas/sasjs/services/public/getcolvals.test.4.sas

@@ -0,0 +1,115 @@
+/**
+  @file
+  @brief testing getcolvals service - input validation (security)
+  @details The libds and col in the IWANT input table must be
+  well-formed (LIBREF.DATASET and SAS name).  An invalid value aborts
+  the service, which shows up as a canceled child job (an aborted
+  service registers no webout).
+
+  The payloads in tests 1-2 resolve to a REAL table / column when the
+  request content is executed as macro code, so on a vulnerable
+  service the job completes, and only the validating service cancels
+  it - the assertion cannot pass against a service that does not
+  validate.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%let _program=&appLoc/services/public/getcolvals;
+
+/**
+  * Test 1 - macro content in libds must abort the service
+  */
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'LIBDS:$19. COL:$9.';
+  put '%sysfunc(coalescec(&dclib..MPE_X_TEST,)),SOME_TIME';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=web1,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=Macro content in libds aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - macro content in col must abort the service
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put 'LIBDS:$19. COL:$9.';
+  put '&dclib..MPE_X_TEST,%sysfunc(coalescec(SOME_TIME,))';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:iwant,
+  outref=web2,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort2=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort2',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort2=1),
+  desc=Macro content in col aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - valid inputs still return values
+  */
+%let f3=%mf_getuniquefileref();
+data _null_;
+  file &f3 termstr=crlf;
+  put 'LIBDS:$19. COL:$9.';
+  put "&dclib..MPE_X_TEST,SOME_TIME";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f3:iwant,
+  outlib=web3
+)
+
+%let nobs=0;
+proc sql noprint;
+select count(*) into: nobs from web3.vals;
+quit;
+
+%mp_assert(
+  iftrue=(&nobs>0),
+  desc=Valid inputs return values,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/public/getrawdata.sas

@@ -11,16 +11,17 @@
   @li filter - the filter RK if used
 
   <h4> SAS Macros </h4>
-  @li mf_verifymacvars.sas
   @li mf_getuser.sas
   @li mf_existfeature.sas
   @li dc_assignlib.sas
-  @li mp_ds2cards.sas
-  @li mp_ds2csv.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_cntlout.sas
+  @li mp_ds2cards.sas
+  @li mp_ds2csv.sas
   @li mp_streamfile.sas
+  @li mp_validatecol.sas
+  @li mpe_validatecol.sas
   @li mpe_filtermaster.sas
 
 
@@ -37,9 +38,57 @@
 %let user=%mf_getuser();
 %let is_fmt=0;
 
-%mp_abort(iftrue= (%mf_verifymacvars(type table)=0)
+/**
+  * Validate inputs before they reach executable code.  table is used as a
+  * dataset reference, in the output file path, and in the download filename,
+  * so it must be a well-formed LIBREF.DATASET (the trailing -FC catalog
+  * suffix is permitted); filter must be an integer.  Values are read with
+  * symget (never re-resolved) and validated in a data step so no macro
+  * content in the input can execute.
+  */
+%let is_libds=0;
+%let is_int=0;
+%let is_type=0;
+data _null_;
+  length _table $64 _filter $16 _type $16;
+  _type=upcase(coalescec(symget('type'),''));
+  _table=coalescec(symget('table'),'');
+  _filter=coalescec(symget('filter'),'0');
+  if missing(_table) then do;
+    putlog 'ERR' 'OR: Missing table';
+    stop;
+  end;
+  %mpe_validatecol(_table,LIBDS,is_libds)
+  /* an absent filter is a valid, unfiltered download */
+  if missing(_filter) then _filter='0';
+  %mp_validatecol(_filter,ISINT,is_int)
+  /* type is validated against a fixed list of download formats */
+  length _types_ok 8;
+  _types_ok=0;
+  if _type in ('SAS','CSV','EXCEL','MARKDOWN','WEBCSV','WEBTAB')
+  then _types_ok=1;
+  else putlog 'ERR' 'OR: Invalid type:' _type;
+  if is_libds=0 then putlog 'ERR' 'OR: Invalid table:' _table;
+  if is_int=0 then putlog 'ERR' 'OR: Invalid filter:' _filter;
+  call symputx('is_libds',is_libds,'l');
+  call symputx('is_int',is_int,'l');
+  call symputx('is_type',_types_ok,'l');
+  call symputx('filter',_filter,'l');
+  if is_libds=1 then call symputx('table',upcase(_table),'l');
+  if _types_ok=1 then call symputx('type',_type,'l');
+run;
+
+%mp_abort(iftrue= (&is_libds ne 1)
   ,mac=&_program..sas
-  ,msg=%str(Invalid inputs: type table)
+  ,msg=%str(Invalid table)
+)
+%mp_abort(iftrue= (&is_int ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid filter)
+)
+%mp_abort(iftrue= (&is_type ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid type)
 )
 
 %let libds=%upcase(&table); /* actual source */

sas/sasjs/services/public/getrawdata.test.1.sas

@@ -0,0 +1,152 @@
+/**
+  @file
+  @brief testing getrawdata service - input validation (security)
+  @details table must be a well-formed LIBREF.DATASET (the trailing
+  format-catalog suffix is permitted), filter must be an integer and
+  type one of the supported download types.  An invalid value aborts
+  the service before any request content can execute.  The abort is
+  asserted from the child job state: the payloads either resolve to a
+  real table when executed as macro code, or write a file outside the
+  WORK directory (verified by live probe) - so on the vulnerable
+  service the job completes, and only the validating service cancels
+  it.  The assertion cannot pass against a service that does not
+  validate.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+
+**/
+
+%let _program=&appLoc/services/public/getrawdata;
+
+/**
+  * Test 1 - macro content in table must abort the service
+  * (the payload resolves to a real table when executed as macro code)
+  */
+data work.params1;
+  length name $32 value $1000;
+  name='type';value='CSV';output;
+  name='table';value='%sysfunc(coalescec(&mpelib..MPE_X_TEST,))';output;
+  name='filter';value='0';output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params1,
+  outref=web1,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=Macro content in table aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - sql injection in filter must abort the service
+  */
+data work.params2;
+  length name $32 value $1000;
+  name='type';value='CSV';output;
+  name='table';value="&dclib..MPE_X_TEST";output;
+  name='filter';value='0 or 1=1';output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params2,
+  outref=web2,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort2=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort2',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort2=1),
+  desc=SQL injection in filter aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - path traversal in table must abort the service
+  * (verified by live probe: on a vulnerable service this writes
+  * ../SECPROBE.csv outside the WORK directory and completes)
+  */
+data work.params3;
+  length name $32 value $1000;
+  name='type';value='CSV';output;
+  name='table';value='../secprobe';output;
+  name='filter';value='0';output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params3,
+  outref=web3,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort3=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort3',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort3=1),
+  desc=Path traversal in table aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - the valid request must still work (positive control)
+  */
+data work.params4;
+  length name $32 value $1000;
+  name='type';value='CSV';output;
+  name='table';value="&dclib..MPE_X_TEST";output;
+  name='filter';value='0';output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params4,
+  outref=web4,
+  viyaresult=WEBOUT_TXT
+)
+
+%let ok4=0;
+data _null_;
+  infile web4;
+  input;
+  if _infile_=:'PRIMARY_KEY_FIELD' then do;
+    call symputx('ok4',1);
+    stop;
+  end;
+run;
+
+%mp_assert(
+  iftrue=(&ok4=1),
+  desc=Valid table request still returns data,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/public/validatefilter.sas

@@ -26,8 +26,9 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
-  @li mf_getvalue.sas
+  @li mp_abort.sas
   @li mp_filterstore.sas
+  @li mpe_validatecol.sas
   @li removecolsfromwork.sas
 
   @version 9.2
@@ -40,7 +41,32 @@
 
 %mpeinit()
 
-%let ds=%upcase(%mf_getvalue(work.iwant,filter_table));
+/**
+  * filter_table is a request input that flows into executable positions
+  * (mp_filterstore libds=, which interpolates it into SQL).  It is read
+  * from the IWANT table in a data step (never re-resolved) and validated
+  * before use - mf_getvalue would re-resolve any macro content in the
+  * value before this code ran.  A format catalog is referenced as
+  * LIBREF.CATALOGNAME-FC, so that form is accepted too.
+  */
+%let ds=;
+%let is_libds=0;
+data _null_;
+  length _ds $64;
+  set work.iwant;
+  _ds=filter_table;
+  %mpe_validatecol(_ds,LIBDS,is_libds)
+  if is_libds=0 then putlog 'ERR' 'OR: Invalid filter_table:' _ds;
+  call symputx('is_libds',is_libds,'l');
+  if is_libds=1 then call symputx('ds',upcase(_ds),'l');
+  stop;
+run;
+
+%mp_abort(iftrue= (&is_libds ne 1)
+  ,mac=&_program..sas
+  ,msg=%str(Invalid filter_table)
+)
+
 %dc_assignlib(WRITE,%scan(&ds,1,.))
 
 %mp_filterstore(

sas/sasjs/services/public/validatefilter.test.1.sas

@@ -0,0 +1,97 @@
+/**
+  @file
+  @brief testing validatefilter service - input validation (security)
+  @details The filter_table in the IWANT input table must be a
+  well-formed LIBREF.DATASET.  An invalid value aborts the service
+  before it reaches mp_filterstore.  The abort shows up as a canceled
+  child job (an aborted service registers no webout).
+
+  The payload in test 1 resolves to a REAL table when the request
+  content is executed as macro code, so on a vulnerable service the
+  job completes, and only the validating service cancels it - the
+  assertion cannot pass against a service that does not validate.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+  @li mf_getuniquefileref.sas
+
+**/
+
+%let _program=&appLoc/services/public/validatefilter;
+
+/**
+  * Test 1 - macro content in filter_table must abort the service
+  */
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'FILTER_TABLE:$41.';
+  put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=web1,
+  viyaresult=WEBOUT_TXT
+)
+
+%let abort1=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort1',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort1=1),
+  desc=Macro content in filter_table aborts the service,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - valid filter_table still stores a filter
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put 'FILTER_TABLE:$41.';
+  put "&dclib..MPE_TABLES";
+run;
+%let f3=%mf_getuniquefileref();
+data _null_;
+  file &f3 termstr=crlf;
+  infile datalines4 dsd;
+  input;
+  put _infile_;
+datalines4;
+GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. OPERATOR_NM:$10. RAW_VALUE:$4000.
+AND,AND,1,LIBREF,CONTAINS,"'DC'"
+AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
+;;;;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:iwant &f3:filterquery,
+  outlib=web2
+)
+
+%let nobs=0;
+proc sql noprint;
+select count(*) into: nobs from web2.result;
+quit;
+
+%mp_assert(
+  iftrue=(&nobs>0),
+  desc=Valid filter_table returns a filter result,
+  outds=work.test_results
+)
+
+/**
+  * dump results to the log for offline inspection
+  */
+data _null_;
+  set work.test_results;
+  putlog 'TEST_RESULT_LINE: ' test_result ' - ' test_description;
+run;

sas/sasjs/services/public/viewdata.sas

@@ -181,7 +181,7 @@ run;
   %put NOTE- Please add to &mpelib..MPE_CONFIG table;
   %put NOTE-;%put NOTE-;
   %global DC_MAXOBS_WEBVIEW;
-  %let DC_MAXOBS_WEBVIEW=500;
+  %let DC_MAXOBS_WEBVIEW=2000;
 %end;
 %if &existds>0 %then %do;