Merge pull request 'fix(ci): publish the docs with surfer 7, and rebuild release assets on demand' (#382 ) from fix/ci-surfer-app-password into release/7.20

Reviewed-on: #382
fix(ci): rebuild and upload release assets on demand
2026-10-28 08:49:33 +00:00 · 2026-10-28 02:07:31 +00:00 · 2026-10-28 02:05:10 +00:00 · 2026-10-28 02:02:57 +00:00 · 2026-10-28 01:57:23 +00:00 · 2026-10-28 01:42:51 +00:00
66 changed files with 2167 additions and 778 deletions
@@ -142,6 +142,7 @@ jobs:
          echo NODE_PATH=node >> .env
          echo CORS=enable >> .env
          echo WHITELIST=http://localhost:4200 >> .env
+          echo HTTP_PORT=5000 >> .env
          # The server refuses to run as root unless RUN_AS names an account
          # (a security change in the recent releases). This job runs as root
          # in a throwaway container, so opt in explicitly.
@@ -0,0 +1,106 @@
+name: Publish code documentation
+run-name: Publishing typedoc and SAS docs
+
+# The code documentation sites are published here rather than inline in the
+# release job, so that they can be republished on their own - after a secret
+# rotation, or for a release whose artefact upload failed (the release job
+# cannot simply be re-run: semantic-release refuses once the tag exists).
+on:
+  workflow_dispatch:
+
+env:
+  NODE_VERSION: '24.5.0'
+
+jobs:
+  typedoc:
+    runs-on: ubuntu-latest
+    # One deploy per site at a time: `surfer deploy` swaps the site atomically,
+    # so two runs racing is not corruption, but it is a wasted build and the
+    # last swap wins.
+    concurrency:
+      group: surfer-webdoc
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Write .npmrc file
+        run: |
+          echo "$NPMRC" > client/.npmrc
+          echo "legacy-peer-deps=true" >> client/.npmrc
+        shell: bash
+        env:
+          NPMRC: ${{ secrets.NPMRC }}
+
+      - name: Install dependencies
+        run: |
+          cd client
+          # Decrypt and Install sheet
+          echo "$SHEET_PWD" | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
+          npm ci
+        shell: bash
+        env:
+          SHEET_PWD: ${{ secrets.SHEET_PWD }}
+
+      - name: Build typedoc
+        run: |
+          cd client
+          npm run compodoc:build
+
+      # Deploy tooling is pinned to an exact version: an unpinned `latest` would
+      # let a breaking or compromised upstream release publish straight to the
+      # live site.  `deploy` (v7) replaces the site atomically, and it takes the
+      # Cloudron username plus an app password - the v6 `put --token` flow this
+      # replaces is gone.
+      - name: Deploy typedoc
+        env:
+          SURFER_PASSWORD: ${{ secrets.WEBDOC_DATACONTROLLER_IO }}
+          REF_NAME: ${{ gitea.ref_name }}
+          COMMIT_SHA: ${{ gitea.sha }}
+        run: |
+          set -euo pipefail
+          npx --yes @cloudron/surfer@7.2.0 deploy client/documentation \
+            --server webdoc.datacontroller.io \
+            --username allan \
+            --password "$SURFER_PASSWORD" \
+            --message "Typedoc from ${REF_NAME} @ ${COMMIT_SHA:0:8}"
+
+  sas-docs:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: surfer-code-datacontroller
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          # `sasjs doc` shells out to doxygen
+          apt-get update
+          apt-get install -y doxygen
+          npm i -g @sasjs/cli
+          cd sas
+          npm i
+
+      - name: Build SAS docs
+        run: |
+          cd sas
+          sasjs doc
+
+      - name: Deploy SAS docs
+        env:
+          SURFER_PASSWORD: ${{ secrets.CODE_DATACONTROLLER_IO }}
+          REF_NAME: ${{ gitea.ref_name }}
+          COMMIT_SHA: ${{ gitea.sha }}
+        run: |
+          set -euo pipefail
+          npx --yes @cloudron/surfer@7.2.0 deploy sas/sasjsbuild/sasdocs \
+            --server code.datacontroller.io \
+            --username allan \
+            --password "$SURFER_PASSWORD" \
+            --message "SAS docs from ${REF_NAME} @ ${COMMIT_SHA:0:8}"
@@ -33,6 +33,7 @@ jobs:
          echo NODE_PATH=node >> .env
          echo CORS=enable >> .env
          echo WHITELIST=http://localhost:4200 >> .env
+          echo HTTP_PORT=5000 >> .env
          # The server refuses to run as root unless RUN_AS names an account
          # (a security change in the recent releases). This job runs as root
          # in a throwaway container, so opt in explicitly.
@@ -0,0 +1,195 @@
+name: Release assets
+run-name: Building and uploading the release assets
+
+# Builds the downloadable artefacts for an existing release and uploads them.
+#
+# The release job cuts the tag before it builds, so a failure anywhere between
+# the semantic-release step and the asset upload leaves a published release with
+# no files - and that job cannot simply be re-run, because semantic-release
+# refuses once the tag exists. This workflow rebuilds the artefacts from the tag
+# and attaches them, so the release can be completed without cutting a new one.
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag to build and upload (blank = the latest release)'
+        required: false
+        default: ''
+
+env:
+  NODE_VERSION: '24.5.0'
+
+permissions:
+  contents: write
+
+jobs:
+  assets:
+    runs-on: ubuntu-latest
+    # Two runs uploading the same asset names would race the delete-then-upload
+    # below.
+    concurrency:
+      group: release-assets
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Write .npmrc file
+        run: |
+          echo "$NPMRC" > client/.npmrc
+          echo "legacy-peer-deps=true" >> client/.npmrc
+        shell: bash
+        env:
+          NPMRC: ${{ secrets.NPMRC }}
+
+      - name: Resolve the release tag
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          REQUESTED_TAG: ${{ inputs.tag }}
+        run: |
+          set -euo pipefail
+          AUTH_HEADER="Authorization: token $GITEA_TOKEN"
+          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"
+
+          if [ -n "${REQUESTED_TAG}" ]; then
+            TAG="${REQUESTED_TAG}"
+            # Fail here rather than after a ten-minute build if the tag is not
+            # cut yet or has no release.
+            curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$BASE/releases/tags/${TAG}" > /dev/null
+          else
+            TAG=$(curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$BASE/releases/latest" | jq -r '.tag_name')
+          fi
+          echo "Building assets for ${TAG}"
+          echo "TAG=${TAG}" >> "$GITHUB_ENV"
+
+      - name: Check out the tag
+        run: |
+          set -euo pipefail
+          git fetch --force --tags origin "+refs/tags/${TAG}:refs/tags/${TAG}"
+          git checkout --force "refs/tags/${TAG}"
+
+      - name: Install system packages
+        run: |
+          apt-get update
+          apt-get install -y zip jq
+          npm i -g @sasjs/cli
+
+      - name: Frontend build
+        run: |
+          cd client
+          # Decrypt and Install sheet
+          echo "$SHEET_PWD" | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
+          npm ci
+          npm i webpack
+          npm run build
+        shell: bash
+        env:
+          SHEET_PWD: ${{ secrets.SHEET_PWD }}
+
+      - name: Build SAS9 EBI Release
+        run: |
+          cd sas
+          npm i
+          sasjs c -t sas9
+          rm -rf sasjsbuild/tests
+          sasjs b -t sas9
+          cp sasjsbuild/mysas9deploy.sas ./demostream_sas9.sas
+          # remove streamed component and rebuild SAS 9 services
+          rm -rf sasjsbuild/services/web9
+          rm sasjsbuild/services/clickme.sas
+          sasjs b -t sas9
+          cp sasjsbuild/mysas9deploy.sas ./sas9.sas
+
+      - name: Build SASjs Server Release
+        run: |
+          cd sas
+          cp sasjs/utils/favicon.ico ../client/dist/favicon.ico
+          sasjs c -t server
+          rm -rf sasjsbuild/tests
+          server_apploc="/Public/app/dc"
+          sed -i "s|apploc=\"[^\"]*\"|apploc=\"${server_apploc}\"|g" sasjsbuild/services/web/index.html
+          sasjs b -t server
+          cp sasjsbuild/server.json.zip ./sasjs_server.json.zip
+
+      - name: Build Viya Release
+        run: |
+          cd sas
+          sasjs c -t viya
+          rm -rf sasjsbuild/tests
+          sed -i -e 's/servertype="SASJS"/servertype="SASVIYA"/g' sasjsbuild/services/DC.html
+          sasjs b -t viya
+          cp sasjsbuild/viya.sas ./viya.sas
+          # compile Viya Full deploy (without web)
+          rm -rf sasjsbuild/services/web
+          rm sasjsbuild/services/DC.html
+          sasjs b -t viya
+          cp sasjsbuild/viya.sas ./viya_noweb.sas
+          cp sasjsbuild/viya.json ./viya_noweb.json
+
+      - name: Zip Frontend (including viya.json for full viya deploy)
+        run: |
+          cd sas
+          cp sasjsbuild/viya.json ../client/dist/viya.json
+          # Zip from *inside* dist so the archive holds the frontend files at its
+          # root rather than under a client/dist/ prefix (see #147).
+          cd ../client/dist
+          zip -r ../../frontend.zip .
+
+      - name: Upload assets to the release
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          set -euo pipefail
+          AUTH_HEADER="Authorization: token $GITEA_TOKEN"
+          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"
+
+          RELEASE_JSON=$(curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$BASE/releases/tags/${TAG}")
+          RELEASE_ID=$(echo "$RELEASE_JSON" | jq -r '.id')
+          RELEASE_BODY=$(echo "$RELEASE_JSON" | jq -r '.body')
+
+          ASSETS=(frontend.zip
+                  sas/demostream_sas9.sas
+                  sas/viya.sas
+                  sas/sasjs_server.json.zip
+                  sas/sas9.sas
+                  sas/viya_noweb.sas
+                  sas/viya_noweb.json)
+
+          : > SHA256SUMS
+          for f in "${ASSETS[@]}"; do
+            sum=$(sha256sum "$f" | cut -d' ' -f1)
+            printf '%s  %s\n' "$sum" "${f##*/}" >> SHA256SUMS
+          done
+          cat SHA256SUMS
+
+          # Replacing the assets means removing any this workflow uploaded
+          # before, and a second run must not append the verification section
+          # twice.
+          for asset_id in $(echo "$RELEASE_JSON" | jq -r '.assets[]?.id'); do
+            curl -k --fail-with-body -sS -X DELETE -H "$AUTH_HEADER" "$BASE/releases/${RELEASE_ID}/assets/${asset_id}"
+          done
+
+          if echo "$RELEASE_BODY" | grep -q "## Verifying the download"; then
+            echo "Release body already carries the verification section"
+          else
+            VERIFY_SECTION=$(./.gitea/scripts/verify-section.sh)
+            NEW_BODY=$(jq -nr --arg body "$RELEASE_BODY" --arg verify "$VERIFY_SECTION" \
+              '$body + "\n" + $verify + "\n\nFor installation instructions, please visit https://docs.datacontroller.io/"')
+            curl -k --fail-with-body -sS -X PATCH \
+              -H "$AUTH_HEADER" \
+              -H 'Content-Type: application/json' \
+              --data "$(jq -n --arg body "$NEW_BODY" '{draft:false, body:$body}')" \
+              "$BASE/releases/$RELEASE_ID"
+          fi
+
+          URL="$BASE/releases/${RELEASE_ID}/assets"
+          for f in "${ASSETS[@]}"; do
+            echo "Uploading $f ..."
+            curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@$f"
+          done
+          echo "Uploading SHA256SUMS ..."
+          curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@SHA256SUMS"
@@ -107,6 +107,16 @@ jobs:
          echo ${{ secrets.SHEET_PWD }} | gpg --batch --yes --passphrase-fd 0 ./libraries/sheet-crypto.tgz.gpg
          npm ci

+      - name: Build frontend for web streaming
+        # The server-ci target streams client/dist as the 'clickme' web app, so
+        # the production build has to exist before the mocked services are
+        # deployed. Without it `sasjs cbd` reports
+        # "webSourcePath: .../client/dist ... doesn't exist", streams no
+        # frontend and the deploy step fails.
+        run: |
+          cd client
+          npm run build
+
      - name: Setup and start SASjs server
        run: |
          npm i -g pm2
@@ -117,6 +127,7 @@ jobs:
          echo NODE_PATH=node >> .env
          echo CORS=enable >> .env
          echo WHITELIST=http://localhost:4200 >> .env
+          echo HTTP_PORT=5000 >> .env
          # The server refuses to run as root unless RUN_AS names an account
          # (a security change in the recent releases). This job runs as root
          # in a throwaway container, so opt in explicitly.
@@ -330,19 +341,6 @@ jobs:
          cd ../client/dist
          zip -r ../../frontend.zip .

-      - name: Release Typedoc
-        run: |
-          cd client
-          npm -g install cloudron-surfer
-          npm run compodoc:build
-          surfer put --token ${{ secrets.TSDOC_TOKEN }} --server webdoc.datacontroller.io documentation/* /
-
-      - name: Release code.datacontroller.io
-        run: |
-          cd sas
-          sasjs doc
-          surfer put --token ${{ secrets.CODE_DATACONTROLLER_IO }} --server code.datacontroller.io sasjsbuild/sasdocs/* /
-
      - name: Upload assets to release
        env:
          # Use the same ephemeral per-job Gitea Actions token that the
@@ -406,6 +404,33 @@ jobs:
          echo "Uploading SHA256SUMS ..."
          curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@SHA256SUMS"

+      - name: Publish the code documentation
+        # Typedoc (webdoc) and the SAS docs (code) are published by docs.yaml,
+        # not inline here. Two reasons: the v6 `surfer put --token` flow is gone
+        # in Surfer 7, which takes a Cloudron username plus an app password; and
+        # a dispatch means a stale or failed docs deploy can be republished
+        # without re-running this job, which semantic-release refuses once the
+        # tag exists.
+        #
+        # This runs AFTER the release assets are uploaded: the documentation is
+        # a side channel, and a docs problem must not cost a cut release its
+        # artefacts.
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          set -euo pipefail
+          AUTH_HEADER="Authorization: token $GITEA_TOKEN"
+          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"
+
+          TAG=$(curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$BASE/releases/latest" | jq -r '.tag_name')
+          echo "Dispatching code documentation for ${TAG}"
+
+          curl -k --fail-with-body -sS -X POST \
+            -H "$AUTH_HEADER" \
+            -H 'Content-Type: application/json' \
+            --data "$(jq -n --arg ref "refs/tags/$TAG" '{ref: $ref}')" \
+            "$BASE/actions/workflows/docs.yaml/dispatches"
+
      - name: Publish the demo site for this release
        # The tag is created by this job's semantic-release step, so it carries
        # the Actions user as its author - and the forge suppresses ref events
@@ -75,6 +75,7 @@ jobs:
          touch .env
          echo RUN_TIMES=js >> .env
          echo NODE_PATH=node >> .env
+          echo HTTP_PORT=5000 >> .env
          # The server refuses to run as root unless RUN_AS names an account
          echo RUN_AS=root >> .env
          nohup ./api-linux > server.log 2>&1 &
@@ -32,6 +32,8 @@ Never consider a change complete until the relevant linters pass on the files yo

 - **Client (TypeScript/HTML/etc.)**: run `npm run lint:check` from the `client/` directory (prettier). Fix any failures with `npm run lint:fix`.
 - **SAS**: after creating or modifying any `.sas` files, run `sasjs lint` from the `sas/` directory and ensure the files you touched have no lint warnings (the repo currently has pre-existing warnings in other files, which can be ignored).
+  `sas/.sasjslint` carries `ignoredMacros: ["webout", "mpeinit", "mpeterm"]`. `webout` is the linter's own default - `sasjs compile` injects `%webout` into every service - and an explicit list replaces that default, so it has to be restated there. `mpeinit` and `mpeterm` are listed because every service is compiled with a serviceInit program (`sasjs/utils/serviceinitsas9|viya|server.sas`) that declares both macros in its own header, so a service body may call them without declaring them. Add a macro to that list only when it reaches a file the same way - injected by the compiler, through serviceInit, testSetUp or another implicit provider - never to silence a genuinely missing declaration.
+  `noBannedStrings` flags the literal words `error ` and `warning ` anywhere in a file, comments included. Reword prose; where the text has to stay (a real `%put`, a doxygen `@warning` tag), keep the log output identical with `%put %str(WARN)ING:` rather than a per-file override, and use `@sasjslint {"noBannedStrings": false}` only when the word itself is the content.

 ## No external assets

@@ -1,3 +1,115 @@
+# [7.19.0](https://git.datacontroller.io/dc/dc/compare/v7.18.0...v7.19.0) (2026-10-28)
+
+
+### Bug Fixes
+
+* address the review findings on the harness and refresh changes ([36e4e88](https://git.datacontroller.io/dc/dc/commit/36e4e88eb134d3b48217d6dd185ea3c03f03fba5))
+* **ci:** dispatch the demo deploy with a tag ref and the actions scope ([698dea6](https://git.datacontroller.io/dc/dc/commit/698dea6d6b7a8635313f11acfcc7a4f39a02bc99))
+* **ci:** restore the gitleaks ignore entries for the mock licence pair ([0d58be9](https://git.datacontroller.io/dc/dc/commit/0d58be975d8eb2d7f1ab6a648f760151392c7973))
+* **client:** open the SAS log page with noopener ([072a133](https://git.datacontroller.io/dc/dc/commit/072a1337020970b5b4ff8aa31719cc5ffec7701e))
+* **client:** refuse a truncated table in the multi-table upload ([1fb4818](https://git.datacontroller.io/dc/dc/commit/1fb4818d215f538a0ac51772ab1f261aeb91f7e1))
+* **cypress:** apply the licence key when the estate has none ([be8f445](https://git.datacontroller.io/dc/dc/commit/be8f445ae1051e132fb6eb5a69b45264255326b5)), closes [input#file-upload](https://git.datacontroller.io/input/issues/file-upload)
+* **dc_execute:** use = comparisons, not IN, in the fail-fast guard ([fed43cb](https://git.datacontroller.io/dc/dc/commit/fed43cb07ef7f2afc685612a124dbc16d7618e21))
+* **dc_getlibs:** exclude the DC libref from the dictionary scan ([f5aade5](https://git.datacontroller.io/dc/dc/commit/f5aade59760cc376b0f96eb4c892c8bca82dde14))
+* **dc_refreshcatalog:** terminate the generated statements ([1e22476](https://git.datacontroller.io/dc/dc/commit/1e224763ba80500a112812a983801968e3751959))
+* **derived:** a fileref with nothing written is a FAIL, not an error ([9b3cf50](https://git.datacontroller.io/dc/dc/commit/9b3cf50b9c3889786605f0f5d9032d84c50f7107))
+* **derived:** address five review findings ([ccd468b](https://git.datacontroller.io/dc/dc/commit/ccd468b3358c1418ee6d8e198b088a0cf23b8c24))
+* **derived:** carry the placement sequence in mpe_derived ([be50cb5](https://git.datacontroller.io/dc/dc/commit/be50cb5d082e788c6ae106bc1d02e73a8c7b9646))
+* **derived:** close four gaps in the MPE_DERIVED hook ([9914790](https://git.datacontroller.io/dc/dc/commit/9914790777f92128c74ecce67e6c67ee48824b45))
+* **derived:** close the RLS and search gaps on derived columns ([9ef813d](https://git.datacontroller.io/dc/dc/commit/9ef813d439b48e7fefd35c426e62b99e5943002a))
+* **derived:** close the two gaps in the derived/RLS invariant ([422f766](https://git.datacontroller.io/dc/dc/commit/422f766cd43db4ce740e1890fd5df30790eaa7e3))
+* **derived:** compare the outref as text, prove the file is empty ([7830b7b](https://git.datacontroller.io/dc/dc/commit/7830b7bcfa9806dde840ce8dd3e6a1887d0b38be))
+* **derived:** do not drop a shadowed column from staging or downloads ([2d5de9a](https://git.datacontroller.io/dc/dc/commit/2d5de9a324a07a36edc0a4f74c8fad09fde07cbf))
+* **derived:** do not duplicate a shadowed column in the grid payload ([6a1128a](https://git.datacontroller.io/dc/dc/commit/6a1128a46352f77c95dd02df9309f0984a1db2d1))
+* **derived:** eight review findings ([561d7fc](https://git.datacontroller.io/dc/dc/commit/561d7fc3ed2a460835619962cf4dcb29d3dc87c7))
+* **derived:** enforce the ACTIVE picker pair in the extracted check ([1863076](https://git.datacontroller.io/dc/dc/commit/1863076bc956b4ff94bf7b85b3dde0d6e3b54d7a)), closes [#366](https://git.datacontroller.io/dc/dc/issues/366)
+* **derived:** enforce the documented limits, and handle derived filters in VIEW ([d3dec68](https://git.datacontroller.io/dc/dc/commit/d3dec682aafeb143fe983d6546337cb6490f5620))
+* **derived:** filtering a format catalog no longer aborts the service ([469229a](https://git.datacontroller.io/dc/dc/commit/469229aecbaad304a835a5f229df111c37ff7c0d))
+* **derived:** guard the bitemporal cut-off, coalesce the selectbox seed ([7fb981d](https://git.datacontroller.io/dc/dc/commit/7fb981daa91bce29c2d62c246185d01dcad08d6b))
+* **derived:** guard the empty physical column list in the dep check ([5f5d868](https://git.datacontroller.io/dc/dc/commit/5f5d8681b9d78280a5200309917415e135c3ce5b))
+* **derived:** hold a derived column read-only whatever the table's security ([5f443b0](https://git.datacontroller.io/dc/dc/commit/5f443b0b16a2abd3c11d96caa7a043911c29ef01))
+* **derived:** ignore a re-added name when collecting inactive rows ([6b5bf3e](https://git.datacontroller.io/dc/dc/commit/6b5bf3e6be89b22e5a633c34b9aecbc8c7d48ee2))
+* **derived:** ignore inactive rows and VIEW rules in the submit guards ([85c89ca](https://git.datacontroller.io/dc/dc/commit/85c89ca2a4e02112ba496ac36a222f33675544da))
+* **derived:** keep targetds for plain datasets, test the empty file ([f8f6568](https://git.datacontroller.io/dc/dc/commit/f8f65681e1489f4ce94d9127d5e4c0c82798e960))
+* **derived:** make the model enforce the required columns, and leave VAR_FMT optional ([7efbf31](https://git.datacontroller.io/dc/dc/commit/7efbf31ea5ac234bd8f924fdf89a95d101b3aff1))
+* **derived:** only the delete marker is a reserved name ([af31c36](https://git.datacontroller.io/dc/dc/commit/af31c36b670ec63a2a4ba056bf6c4b28f6096e08))
+* **derived:** open the statements file with a long record length ([ee31782](https://git.datacontroller.io/dc/dc/commit/ee31782fbff909b540162b1bdfca6f307208331b))
+* **derived:** pair the filterds %else with %do/%end ([88a38db](https://git.datacontroller.io/dc/dc/commit/88a38db877dba9b98ccf50f0cc7242674654586b))
+* **derived:** raise the VAR_ORDER ceiling to 10000 ([27ddcfb](https://git.datacontroller.io/dc/dc/commit/27ddcfb5759d6eb8f9880adc7c271a408e7741ad))
+* **derived:** refuse dependencies on inactive rows, and keep the RLS scope guard coherent ([7799a30](https://git.datacontroller.io/dc/dc/commit/7799a300df82339683eea21e4a89ce7a6471c575))
+* **derived:** reject a dependency on a row this submission deletes ([8806504](https://git.datacontroller.io/dc/dc/commit/88065049674c2cc0141eef22704b759f5b708b0f))
+* **derived:** repair the validation rows for MPE_DERIVED ([9a047ad](https://git.datacontroller.io/dc/dc/commit/9a047ad42ec86df7d1eaf5058c0edde56016e88f))
+* **derived:** review feedback - cover the empty response, assert the last row ([e0da9f1](https://git.datacontroller.io/dc/dc/commit/e0da9f1098db63185538398ca478696cc849da4b))
+* **derived:** stop the hooks aborting on their own macro variables ([688550a](https://git.datacontroller.io/dc/dc/commit/688550a28d9794ac6b878926a0e1e2ef8f6e9335))
+* **derived:** truncate the derived label on an even quote boundary ([1b5c4b3](https://git.datacontroller.io/dc/dc/commit/1b5c4b3ecd962a28e08fc5671b57dbf7ae0b8d9f))
+* **derived:** verify on stripped values, and label with the variable name ([4d9ad38](https://git.datacontroller.io/dc/dc/commit/4d9ad38df1fa89b0d5a12214d703d9c1fe5295f2))
+* **editor:** a blank truncation note is not a warning ([f182179](https://git.datacontroller.io/dc/dc/commit/f18217940365f27ae388fae087e6126931e1222c))
+* **editor:** keep decimal values when an uploaded file is edited manually ([04ce186](https://git.datacontroller.io/dc/dc/commit/04ce1861e7821063b1be42c22f92f62aed509602))
+* **getcolvals:** report an invalid value, not missing inputs ([4bcee70](https://git.datacontroller.io/dc/dc/commit/4bcee706ffaf1929bdd14bb2fb15111174ca3916))
+* **getdata.test:** pin the derived filter to a libref, and spell the table as it is ([45dce7b](https://git.datacontroller.io/dc/dc/commit/45dce7b11b7a4399c1d0dc9c3e7a283b1cf59df1))
+* **getdata:** clear the globals it declares before use ([359b570](https://git.datacontroller.io/dc/dc/commit/359b570b85aa2f66507e81ea2977243be4590f83))
+* **getddl.test:** ask for a table, not a whole library ([059cdf1](https://git.datacontroller.io/dc/dc/commit/059cdf176a7ec556b45ea1b21d880c048eba14ca))
+* **getrawdata:** declare mp_ds2md so the Markdown download can run ([661ee19](https://git.datacontroller.io/dc/dc/commit/661ee19cfa5f996bf01f2b11817039f38dbb9985))
+* **lint:** clear the sasjs lint warnings on the files 7.19 touched ([355c46a](https://git.datacontroller.io/dc/dc/commit/355c46a7b64994d0146a88657dc5635593249138))
+* **makedata:** read KEEPME from every input row, and keep it out of the Viya seed ([d626cb8](https://git.datacontroller.io/dc/dc/commit/d626cb8f2daf94fde7ff21d924b9a2ca4968d983))
+* **mocks:** address the audit-pack review findings ([489819b](https://git.datacontroller.io/dc/dc/commit/489819b05111323ae1e34edda22a5446cdbc2d87))
+* **mocks:** carry the zip flags into the central directory ([c4021a4](https://git.datacontroller.io/dc/dc/commit/c4021a42ffaed2af80c53aaa1455249190b339b8))
+* **mocks:** clear the licence for published sites, not for the estate ([62f55db](https://git.datacontroller.io/dc/dc/commit/62f55db170185900063f7a431cd8f375ebabf930))
+* **mocks:** copy the physical column list before deriving ([f411b7d](https://git.datacontroller.io/dc/dc/commit/f411b7d269381e3f7cab2d85b6d8f7d7d50ea4d1))
+* **mocks:** derive before filtering in the viewer, and drop the columns ([60ec470](https://git.datacontroller.io/dc/dc/commit/60ec470404e5c279b58ac6a01768dfc83413b197))
+* **mocks:** derive the retained key wherever rows are staged ([74060c0](https://git.datacontroller.io/dc/dc/commit/74060c0585fb8b217cb181ea747ad318f5238868))
+* **mocks:** drop derived variables before staging ([73d25b1](https://git.datacontroller.io/dc/dc/commit/73d25b1220d574ef2547fbd9e6e3cf1ceda375ee))
+* **mocks:** key rk_underlying tables on their business key ([347dbb5](https://git.datacontroller.io/dc/dc/commit/347dbb53b01c47bbd562de14d5ccbae7d96407c6))
+* **mocks:** register MPE_DERIVED and seed its field rules ([ee3c878](https://git.datacontroller.io/dc/dc/commit/ee3c8787efef9cd716bdd2bf80f6cb6751461db9))
+* **mocks:** scope the derived hook's graph to the submission ([6cad23f](https://git.datacontroller.io/dc/dc/commit/6cad23fce9ac0cf64c247c40f3b7eac875fd4cfb))
+* **mocks:** search the physical columns only in the viewer ([293022d](https://git.datacontroller.io/dc/dc/commit/293022d46b27e096fd514cdda49f688612df2f5b))
+* **mocks:** stop the mock advertising invalid licence keys ([fb36379](https://git.datacontroller.io/dc/dc/commit/fb36379520028baaf4bc3f08131bfc88deab3ac3))
+* **mpe_assert_abort:** drop the unused line variable ([9c294c0](https://git.datacontroller.io/dc/dc/commit/9c294c0ab5ca8fe30a235553a298c1c68a61442f))
+* **mpe_assert_abort:** take the abort's own MSG, tolerate a missing response ([0a52d20](https://git.datacontroller.io/dc/dc/commit/0a52d20c5fc27b4adde5fc8ed6f39a58c48f62ff))
+* **mpe_execute:** fail fast when the child left no status ([a036386](https://git.datacontroller.io/dc/dc/commit/a03638667f1968c5909f496d954eeaa13f316763))
+* **mpe_execute:** no apostrophe in the SAS 9 status note ([2204085](https://git.datacontroller.io/dc/dc/commit/220408538b26016bda45e48d619cdbc3e1cbcc59))
+* **mpe_execute:** read an unknown state as unmeasured, not failed ([f2b1428](https://git.datacontroller.io/dc/dc/commit/f2b1428ec3099b2b6e3c6da6627530ef81fab201))
+* name the startupservice launcher button "Show log" ([3445107](https://git.datacontroller.io/dc/dc/commit/3445107d9a91642b67e114bcfcd81c3ce47f3f0f))
+* offer the launcher on the globvars failure modal too ([ad2da45](https://git.datacontroller.io/dc/dc/commit/ad2da453896a51f2cc7e49cebb6c84f7b47d7395))
+* **refresh:** name each catalogue load uniquely in the audit trail ([46bacdd](https://git.datacontroller.io/dc/dc/commit/46bacdd6d9ea814bcd02a1ff8aa3f447baf02f64))
+* **refresh:** one audit folder per refresh call ([56b6620](https://git.datacontroller.io/dc/dc/commit/56b6620ef402b288ce78a2a77662b651e310753e))
+* **sites:** round-trip the committed licence before publishing ([4d462d3](https://git.datacontroller.io/dc/dc/commit/4d462d310f7df23e5d5229cb0702721b7ad41bbd))
+* **test:** getdata test ([da30266](https://git.datacontroller.io/dc/dc/commit/da30266278c4fe27a72810cd5ed647cf3bd823af))
+* **tests:** enable closeouts logging ([fdcd8fa](https://git.datacontroller.io/dc/dc/commit/fdcd8fa4b3e7a7a119c662974d70dc9eea80bdae))
+* **tests:** read the work location from _SASJS_WORK ([033ffd3](https://git.datacontroller.io/dc/dc/commit/033ffd302502c7530fc44a314364f5cdb3be6ad2))
+* **ui:** one error modal at a time ([81ccf75](https://git.datacontroller.io/dc/dc/commit/81ccf758e9dab0c7632ab50bf469e88b8c4c4420))
+* **validatefilter:** build targetds without macro triggers in comments ([42355f1](https://git.datacontroller.io/dc/dc/commit/42355f17bc07bf4d44ba858984d83b9ed72cc0ba))
+* **validatefilter:** drop the dead nobs assignment, keep targetds precedence ([337ea8e](https://git.datacontroller.io/dc/dc/commit/337ea8e5e021c6f2451fa3ad8396148744118fc9))
+
+
+### Features
+
+* **dc_getlibs:** dump the library list when debug is on ([582523d](https://git.datacontroller.io/dc/dc/commit/582523dc87a253b71824359224070ccb4746a386))
+* **derived:** allow a variable to shadow a physical column ([12beeff](https://git.datacontroller.io/dc/dc/commit/12beeff69518376a042fbbc58332acc40d89d132))
+* **derived:** declare VAR_DEPENDENCIES instead of VAR_ORDER ([813bce6](https://git.datacontroller.io/dc/dc/commit/813bce685691ad252043eeab31eb5460de81b756))
+* **derived:** optional VAR_LABEL on MPE_DERIVED ([7c9ce9a](https://git.datacontroller.io/dc/dc/commit/7c9ce9a712137e9825a5f2ff52796d765423147b))
+* **derived:** pickers, and an ACTIVE default, for MPE_DERIVED ([87e794f](https://git.datacontroller.io/dc/dc/commit/87e794f769fe420f214f43b141f0b41cf88eefee))
+* **derived:** report the dependency cycle path ([cee1a95](https://git.datacontroller.io/dc/dc/commit/cee1a954ee97e17f297b7cf43ef0e5cfc6e393a6))
+* **derived:** take any text in VAR_LABEL, but not a line break ([78e88a4](https://git.datacontroller.io/dc/dc/commit/78e88a41128b047c0276ccf647f128229152604c))
+* **derived:** validate MPE_DERIVED at submit time ([0b010f2](https://git.datacontroller.io/dc/dc/commit/0b010f2c65ddada70d62ea83c706943c6c45756f))
+* **derived:** validation rules and a VAR_TYPE dropdown for MPE_DERIVED ([aa6d50e](https://git.datacontroller.io/dc/dc/commit/aa6d50e1ea5fc2fe6be8aa9fa859b92d0a1061c5))
+* **derived:** VARIABLE_ORDER, so a derived variable can build on another ([33d043e](https://git.datacontroller.io/dc/dc/commit/33d043ea92f007dca70c95d5363329d4447eeba9))
+* **editor:** derived variables in the EDIT screen ([b0fd354](https://git.datacontroller.io/dc/dc/commit/b0fd354ecdbd7bf29196ed983ff3cce691affeed))
+* **editor:** mark derived variables as computed in the EDIT grid ([6513d09](https://git.datacontroller.io/dc/dc/commit/6513d09f059de15c5304f99b3f8d898d46ea1be7))
+* **editor:** serve a truncated selection with a warning instead of aborting ([34691dc](https://git.datacontroller.io/dc/dc/commit/34691dceecb63f952b23ad8acc5bc93753c6cfee))
+* **editor:** show the derived column label in the header tooltip ([cfe6647](https://git.datacontroller.io/dc/dc/commit/cfe6647ca870c45a3d8abc2753ce7f070e3fdf4a))
+* **mocks:** add getauditfile and the audit-folder artefacts ([752d944](https://git.datacontroller.io/dc/dc/commit/752d944c2091ef6d2624a15ab2e2e7be35608798))
+* **mocks:** implement derived variables in the JS services ([b4fae88](https://git.datacontroller.io/dc/dc/commit/b4fae88ae13658607e4209449577cd42849d3a99))
+* **mocks:** validate a submitted MPE_DERIVED row like the hook does ([5d183d1](https://git.datacontroller.io/dc/dc/commit/5d183d135085ea023f32e5b1d4992d997394a8fb))
+* run startupservice in a new window from its error modal ([40f6133](https://git.datacontroller.io/dc/dc/commit/40f61331aad6bc8a52c551b924f0d1f1e6426cee))
+* **sites:** mint a licence for the published sites ([7d9df71](https://git.datacontroller.io/dc/dc/commit/7d9df71fdb0ab265e745ed3b708038983e855a87))
+
+
+### Reverts
+
+* **derived:** no shadowing, and no self reference ([5af8e6f](https://git.datacontroller.io/dc/dc/commit/5af8e6fc359221586af74a685159dfd75a40ec45))
+* **stagedata.test.4:** back to the MPE_LOADS evidence for now ([31aa859](https://git.datacontroller.io/dc/dc/commit/31aa8591538409fc1a3ef97d7474120094ce202d))
+
 # [7.18.0](https://git.datacontroller.io/dc/dc/compare/v7.17.0...v7.18.0) (2026-10-19)


@@ -86,6 +86,7 @@ RUN_TIMES=js
 NODE_PATH=node
 CORS=enable
 WHITELIST=http://localhost:4200
+HTTP_PORT=5000
 ```

 `RUN_TIMES=js` means the server executes `.js` files as Stored Programs (no SAS binary needed). `CORS=enable` plus `WHITELIST` allows the Angular dev server on port 4200 to call the mock services on port 5000. Start the server:
@@ -94,7 +95,7 @@ WHITELIST=http://localhost:4200
 ./api-linux    # or api-macos / api-win.exe depending on your platform
 ```

-The server runs in desktop mode (no auth) on `http://localhost:5000` by default.
+`HTTP_PORT` is not optional in SASjs Server 3.x: `PORT` and `PROTOCOL` are deprecated, and a server started without `HTTP_PORT` or `HTTPS_PORT` exits with "no listener is enabled". With it set, the server runs in desktop mode (no auth) on `http://localhost:5000`.

 ### 2. Deploy the mock services and streaming frontend

@@ -79,12 +79,53 @@ context('derived variables: ', function () {
  it('4 | a derived column is marked read-only, and opens no editor', () => {
    openEditor('TESTDATA.DERIVED_DEMO')

-    // the editor marks a read-only column with this class, on the header and on
-    // its cells (editor.component.ts, afterGetColHeader / afterGetCellMeta), so
-    // this asserts the state rather than inferring it from something not
-    // happening. STATE is the control: same grid, not read-only.
-    headerCell('REGION').should('have.class', 'readonlyCell')
-    headerCell('STATE').should('not.have.class', 'readonlyCell')
+    // a derived column is read-only AND computed, so the grid marks it apart
+    // from an ordinary read-only column - its own header/cell class, no
+    // readonlyCell dimming (editor.component.ts, afterGetColHeader /
+    // afterGetCellMeta). STATE is the control: same grid, a physical column.
+    headerCell('REGION').should('have.class', 'dc-derived-header')
+    headerCell('REGION').should('not.have.class', 'readonlyCell')
+    headerCell('STATE')
+      .should('not.have.class', 'dc-derived-header')
+      .and('not.have.class', 'readonlyCell')
+    getCellByHeaderAndRow(0, 'REGION').should('have.class', 'dc-derived-cell')
+    getCellByHeaderAndRow(0, 'STATE').should(
+      'not.have.class',
+      'dc-derived-cell'
+    )
+    // The computed block opens at whichever derived column comes first in the
+    // grid - MPE_DERIVED order, so LINE_VALUE here, not REGION. Assert the
+    // invariant rather than a name: exactly one derived header carries the
+    // marker, and it is the first one.
+    cy.get('.ht_clone_top .htCore thead tr th').then(($ths) => {
+      const derived = [...$ths].filter((th) =>
+        th.classList.contains('dc-derived-header')
+      )
+      expect(derived.length, 'derived headers').to.be.greaterThan(0)
+      expect(
+        derived[0].classList.contains('dc-derived-first'),
+        'the first derived header opens the block'
+      ).to.be.true
+      derived
+        .slice(1)
+        .forEach(
+          (th) =>
+            expect(
+              th.classList.contains('dc-derived-first'),
+              'only the first derived header carries the marker'
+            ).to.be.false
+        )
+    })
+
+    // the hover text carries the label, which is what says what the column is:
+    // the configured VAR_LABEL where there is one, otherwise the generated
+    // name=expression
+    headerCell('REGION_LABEL')
+      .should('have.attr', 'title')
+      .and('contain', 'Region and state')
+    headerCell('REGION')
+      .should('have.attr', 'title')
+      .and('contain', 'REGION=ifc')

    enterEditMode()

@@ -314,9 +314,12 @@ export class AppComponent {
   */
  public subscribeToShowAbortModal() {
    this.eventService.onShowAbortModal.subscribe((sasjsAbort: InfoModal) => {
-      let abortId = this.sasjsAborts.length + 1
-      sasjsAbort.id = abortId
-      this.sasjsAborts.push(sasjsAbort)
+      /* One message at a time.  Pushing to the array stacked a second modal on
+         top of the one already open - two dismissals for one failure, and the
+         lower one was often a duplicate of the upper.  Replacing keeps the
+         newest message, which is the one the user has to act on. */
+      sasjsAbort.id = Date.now()
+      this.sasjsAborts = [sasjsAbort]
      this.cdr.detectChanges() //Changes were not triggered while hot is focused
    })
  }
@@ -575,6 +575,8 @@ export class EditorComponent
  public hasBaseDropZoneOver = false
  public hasAnotherDropZoneOver = false
  public headerPks: string[] = []
+  /** Derived (computed) columns from MPE_DERIVED - read-only, and marked as such */
+  public derivedVars: string[] = []
  public columnLevelSecurityFlag = false
  public dateTimeHeaders: string[] = []
  public timeHeaders: string[] = []
@@ -1089,6 +1091,20 @@ export class EditorComponent
    return colRules?.readOnly
  }

+  /**
+   * A derived (computed) column - configured in MPE_DERIVED, created at read
+   * time. Read-only, and styled apart from an ordinary read-only column so the
+   * user can see that there is nothing to edit rather than that they may not.
+   */
+  isDerivedCol(col: string | number) {
+    const name =
+      typeof col === 'number'
+        ? (this.hotInstance?.colToProp(col) as string) || this.columnHeader[col]
+        : col
+
+    return !!name && this.derivedVars.includes(name.toUpperCase())
+  }
+
  isColHeader(col: string) {
    return this.headerArray.indexOf(col.toUpperCase()) > -1
  }
@@ -2757,17 +2773,24 @@ export class EditorComponent
            this.disableSubmit = true
            this.submit = false

-            const errorText =
-              typeof err.adapterRespnse === 'string'
-                ? err.adapterRespnse
-                : JSON.stringify(err.adapterRespnse)
+            /* sasService has already shown the SAS message when it rejected
+               (the abort, or the adapter's error), so reporting it again here
+               stacked a second modal on top of the first - and read
+               'undefined' because it looked for a property that does not
+               exist.  Only speak up when nothing was shown. */
+            if (!err?.errorShown) {
+              const errorText =
+                typeof err?.adapterResponse?.error === 'string'
+                  ? err.adapterResponse.error
+                  : JSON.stringify(err?.adapterResponse ?? err)

-            this.eventService.showAbortModal(
-              'editors/stagedata',
-              `Submit request failed\n${errorText}`,
-              null,
-              'Submit error'
-            )
+              this.eventService.showAbortModal(
+                'editors/stagedata',
+                `Submit request failed\n${errorText}`,
+                null,
+                'Submit error'
+              )
+            }

            return false
          })
@@ -3941,8 +3964,11 @@ export class EditorComponent

    // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
    // truncated to the first rows within the limit, and carries the reason
-    // here.  Empty means the whole selection is in the grid.
-    this.truncationNote = response.data.sasparams[0].TRUNC_MSG || ''
+    // here.  Empty means the whole selection is in the grid - and a blank
+    // character arrives as a single blank (mp_jsonout cannot emit a zero-length
+    // string, since SAS has none), so trim it: whitespace is "no note", not a
+    // warning with an empty tooltip.
+    this.truncationNote = (response.data.sasparams[0].TRUNC_MSG || '').trim()

    const hot = this.hotInstance

@@ -3964,6 +3990,12 @@ export class EditorComponent

    this.headerColumns = response.data.sasparams[0].COLHEADERS.split(',')
    this.headerPks = response.data.sasparams[0].PK.split(' ')
+    // Derived (computed) columns, named by getdata from MPE_DERIVED - read-only
+    // through a READONLY DQ rule, and marked in the grid so they read as
+    // computed rather than merely non-editable.
+    this.derivedVars = (response.data.sasparams[0].DERIVEDVARS || '')
+      .split(' ')
+      .filter((col: string) => !!col)
    this.columnLevelSecurityFlag = !!response.data.sasparams[0].CLS_FLAG

    if (this.columnLevelSecurityFlag)
@@ -4338,9 +4370,37 @@ export class EditorComponent
          // header columns styling - primary keys
          const isPKCol = column && this.isColPk(column)
          const isReadonlyCol = column && this.isReadonlyCol(column)
+          const isDerivedCol = column && this.isDerivedCol(column)

          if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
-          if (isReadonlyCol && !isPKCol) th.classList.add('readonlyCell')
+          if (isReadonlyCol && !isPKCol && !isDerivedCol) {
+            th.classList.add('readonlyCell')
+          }
+
+          // a derived column is computed, not merely locked - give it its own
+          // hue and an 'fx' marker rather than the read-only dimming
+          if (isDerivedCol) {
+            th.classList.add('dc-derived-header')
+            // the label says what the column is - for a derived variable that
+            // is the configured VAR_LABEL, or the generated name=expression
+            // when the administrator left it blank.  A label that merely
+            // repeats the name adds nothing, so it falls back to the rule.
+            const derivedLabel = (this.cols || [])
+              .filter((c: any) => c.NAME === column)
+              .map((c: any) => String(c.LABEL || '').trim())
+              .find(
+                (l: string) =>
+                  l.length > 0 &&
+                  l.toUpperCase() !== String(column).toUpperCase()
+              )
+            th.title = derivedLabel
+              ? `${derivedLabel} - derived variable, computed at read time and cannot be edited`
+              : 'Derived variable - computed at read time and cannot be edited'
+            const firstDerivedCol = this.columnHeader.findIndex(
+              (name: string) => this.isDerivedCol(name)
+            )
+            if (col === firstDerivedCol) th.classList.add('dc-derived-first')
+          }

          // Remove header arrow from Delete column
          if (col === 0) {
@@ -4356,6 +4416,7 @@ export class EditorComponent
          cellProperties: Handsontable.CellProperties
        ) => {
          const isReadonlyCol = col && this.isReadonlyCol(col)
+          const isDerivedCol = col && this.isDerivedCol(col)

          // Check if this cell should be marked as invalid due to duplicate primary key values
          // Only applies to primary key columns (col 1 through readOnlyFields)
@@ -4379,7 +4440,26 @@ export class EditorComponent
          }

          // Add readonlyCell class for readonly columns to maintain original styling
-          if (isReadonlyCol && !classes.includes('readonlyCell')) {
+          if (isDerivedCol) {
+            // a derived cell carries its own tint; the read-only dimming would
+            // wash it out, and "greyed out" is a different message from
+            // "computed here"
+            classes = classes.filter((c: string) => c !== 'readonlyCell')
+            if (!classes.includes('dc-derived-cell')) {
+              classes.push('dc-derived-cell')
+            }
+            // derived columns are appended after the physical ones - mark the
+            // start of the block so it reads as one group
+            const firstDerivedCol = this.columnHeader.findIndex(
+              (name: string) => this.isDerivedCol(name)
+            )
+            if (
+              col === firstDerivedCol &&
+              !classes.includes('dc-derived-first')
+            ) {
+              classes.push('dc-derived-first')
+            }
+          } else if (isReadonlyCol && !classes.includes('readonlyCell')) {
            classes.push('readonlyCell')
          }

@@ -51,6 +51,11 @@ export interface SASParam {
  TRUNCATED?: number
  /** Reason shown in the tooltip when TRUNCATED is 1, else empty */
  TRUNC_MSG?: string
+  /**
+   * Space separated list of the derived (computed) columns from MPE_DERIVED,
+   * upper case. Empty when the table has none.
+   */
+  DERIVEDVARS?: string
 }

 export interface ExcelRule {
@@ -286,7 +286,10 @@ export class SasService {
                adapterResponse: {
                  error: abortMsg
                },
-                log: sasRequest?.logFile
+                log: sasRequest?.logFile,
+                /* the modal above already told the user: a caller that also
+                   reports the failure would stack a second one on top */
+                errorShown: !wrapperOptions?.suppressSuccessAbortModal
              })
            }

@@ -329,7 +332,8 @@ export class SasService {
                adapterResponse: {
                  error: errorMessage
                },
-                log: sasRequest?.logFile
+                log: sasRequest?.logFile,
+                errorShown: !wrapperOptions?.suppressErrorAbortModal
              })
            }

@@ -5019,6 +5019,57 @@ body[cds-theme='dark'] {
  border: 1px solid var(--cds-alias-status-warning) !important;
 }

+/* Derived (computed) columns: read-only, but computed at read time rather than
+  locked by policy.  Tinted and marked with an 'fx' glyph instead of the
+  read-only dimming, so the two do not read as the same thing.  Blue is the
+  info family - the primary key green and the warning ochre keep their meaning. */
+.handsontable th.dc-derived-header {
+  background: #d3e5f2 !important;
+  box-shadow: inset 0 -2px 0 #1a6ea3;
+}
+
+.handsontable th.dc-derived-header div::after {
+  content: 'fx';
+  margin-left: 4px;
+  font-size: 9px;
+  font-style: italic;
+  opacity: 0.7;
+}
+
+.handsontable td.dc-derived-cell {
+  background: #eef5fa !important;
+  font-style: italic;
+}
+
+/* the first derived column starts the computed block */
+.handsontable th.dc-derived-first {
+  border-left: 2px solid #1a6ea3 !important;
+}
+
+.handsontable td.dc-derived-cell.dc-derived-first {
+  box-shadow: inset 2px 0 0 #1a6ea3;
+}
+
+body[cds-theme='dark'] {
+  .handsontable th.dc-derived-header {
+    background: #2b4a5c !important;
+    box-shadow: inset 0 -2px 0 #4b9cc9;
+  }
+
+  .handsontable td.dc-derived-cell {
+    background: #24343d !important;
+    color: #dbe7ee;
+  }
+
+  .handsontable th.dc-derived-first {
+    border-left-color: #4b9cc9 !important;
+  }
+
+  .handsontable td.dc-derived-cell.dc-derived-first {
+    box-shadow: inset 2px 0 0 #4b9cc9;
+  }
+}
+
 .handsontable .numericListbox {
  text-align: right;
 }
@@ -1,6 +1,6 @@
 {
  "name": "dcfrontend",
-  "version": "7.18.0",
+  "version": "7.19.0",
  "description": "Data Controller",
  "devDependencies": {
    "@nogoo9/gitleaks": "8.30.1-post.2",
@@ -11,5 +11,6 @@
  "noNestedMacros": true,
  "hasMacroParentheses": true,
  "strictMacroDefinition": true,
+  "ignoredMacros": ["webout", "mpeinit", "mpeterm"],
  "ignoreList":["sasjs/scratch"]
 }
@@ -0,0 +1,10 @@
+{
+  "fromjs": [
+    {
+      "ADMIN": "all-users",
+      "DCPATH": "/tmp/dcdata",
+      "OUTFMT": "JSON",
+      "KEEPME": "Y"
+    }
+  ]
+}
@@ -436,7 +436,11 @@ if (tableData && tableData.columns && tableData.rows) {
            CLS_FLAG: 0,
            ISMAP: 0,
            TRUNCATED: truncated,
-            TRUNC_MSG: truncMsg
+            TRUNC_MSG: truncMsg,
+            // names the derived columns for the grid, as getdata.sas does - the
+            // READONLY rule above is why they cannot be edited, this is how the
+            // EDIT screen knows they are computed rather than locked
+            DERIVEDVARS: derived.map(d => d.name).join(' ')
        }],
        versions: [],
        xl_rules: []
@@ -451,7 +455,7 @@ if (tableData && tableData.columns && tableData.rows) {
        ],
        maxvarlengths: [], query: [], sasdata: [],
        $sasdata: { vars: {} },
-        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
+        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '', DERIVEDVARS: '' }],
        versions: [], xl_rules: []
    }
 }
@@ -112,6 +112,10 @@
        else if (name === DELETE_MARK) reason = 'VAR_NM is reserved'
        else if (type !== 'C' && type !== 'N')
            reason = 'VAR_TYPE must be C or N'
+        // ACTIVE is a picker in the EDIT screen (HARDSELECT over the
+        // MPE_SELECTBOX pair) - the same pair mpe_derived_check.sas enforces
+        else if (!['0', '1'].includes(String(colVal(row, 'ACTIVE')).trim()))
+            reason = 'ACTIVE must be 1 or 0'
        else if (!Number.isInteger(len) || len < 1)
            reason = 'VAR_LEN must be a positive integer'
        else if (type === 'C' && len > 32767)
@@ -7,7 +7,7 @@
      "name": "dc-sas",
      "dependencies": {
        "@sasjs/cli": "6.2.0",
-        "@sasjs/core": "6.0.0"
+        "@sasjs/core": "6.1.0"
      }
    },
    "node_modules/@asamuzakjp/css-color": {
@@ -235,9 +235,9 @@
      }
    },
    "node_modules/@sasjs/core": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.0.0.tgz",
-      "integrity": "sha512-E9iXPvNgCux80xHhfopCsQE/CUNp1NEF1Z7NRj089A1T8WXP9AWJYcfFfOqoljNh8rnMo1fbIgoNrIaaAgW0MA==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.1.0.tgz",
+      "integrity": "sha512-U0Ekq/TXVMquYV+AG6urBQSUgqUvV37yduufy68QQVgDj3PJiLUJDcOXKOGbhWGD+Nv877VuPCviGYqrqbjXvg==",
      "license": "MIT"
    },
    "node_modules/@sasjs/lint": {
@@ -26,12 +26,15 @@
    "viyacbd": "sasjs folder delete /Public/app/viya && sasjs cbd -t viya && npm run viyadata",
    "viyadata": "sasjs request services/admin/makedata -d deploy/makeDataViya.json -l sasjsresults/makedata.log -o sasjsresults/output.json -t viya",
    "v4data": "sasjs request services/admin/makedata -d deploy/makeDataV4.json -l sasjsresults/makedatav4.log -o sasjsresults/outputv4.json -t v4",
-    "sasdocs": "sasjs doc && ./sasjs/utils/deploydocs.sh"
+    "sasdocs": "sasjs doc && ./sasjs/utils/deploydocs.sh",
+    "sas719setup": "sasjs cbd -t sas719 && npm run sas719data",
+    "sas719data": "sasjs request services/admin/makedata -d deploy/makeDataSas719.json -l sasjsresults/makedata_sas719.log -o sasjsresults/makedata_sas719.json -t sas719",
+    "sas719test": "sasjs test -t sas719"
  },
  "private": true,
  "dependencies": {
    "@sasjs/cli": "6.2.0",
-    "@sasjs/core": "6.0.0"
+    "@sasjs/core": "6.1.0"
  },
  "overrides": {
    "nanoid": "3.3.18"
@@ -17,16 +17,16 @@
 create table &curlib..MPE_DERIVED
   (
    TX_FROM num not null format=datetime19.3,
-    LIBREF varchar(8),
-    DSN varchar(32),
-    VAR_NM varchar(32),
+    LIBREF varchar(8) not null,
+    DSN varchar(32) not null,
+    VAR_NM varchar(32) not null,
    VAR_DEPS varchar(4000),
-    VAR_TYPE varchar(1),
-    VAR_LEN int,
+    VAR_TYPE varchar(1) not null,
+    VAR_LEN int not null,
    VAR_FMT varchar(32),
    VAR_LABEL varchar(256),
-    VAR_LOGIC varchar(4000),
-    ACTIVE int,
+    VAR_LOGIC varchar(4000) not null,
+    ACTIVE int not null,
    TX_TO num not null format=datetime19.3,
    constraint pk_mpe_derived
        primary key(tx_from, libref, dsn, var_nm));
@@ -20,16 +20,16 @@ proc sql;
 create table &dclib..mpe_derived
  (
    tx_from num not null format=datetime19.3,
-    libref varchar(8),
-    dsn varchar(32),
-    var_nm varchar(32),
+    libref varchar(8) not null,
+    dsn varchar(32) not null,
+    var_nm varchar(32) not null,
    var_deps varchar(4000),
-    var_type varchar(1),
-    var_len num,
+    var_type varchar(1) not null,
+    var_len num not null,
    var_fmt varchar(32),
    var_label varchar(256),
-    var_logic varchar(4000),
-    active num,
+    var_logic varchar(4000) not null,
+    active num not null,
    tx_to num not null format=datetime19.3
  );
 quit;
@@ -57,13 +57,19 @@ insert into &dclib..mpe_tables
  ;
 quit;

-/* Dropdown values for VAR_TYPE */
+/* Dropdown values for VAR_TYPE and ACTIVE.  These rows, and the field rules
+  * further down, mirror the MPE_DERIVED rows in mpe_makedata.sas - keep the two
+  * in step, or an upgraded install and a freshly seeded one behave differently. */
 proc sql noprint;
-  select max(selectbox_rk) into: maxrk from &dclib..mpe_selectbox;
+  /* coalesce: an install with no selectbox rows yet returns a missing value,
+    and %eval(&maxrk+1) below would then fail */
+  select coalesce(max(selectbox_rk),0) into: maxrk from &dclib..mpe_selectbox;
 quit;

 %let rk1=%eval(&maxrk+1);
 %let rk2=%eval(&maxrk+2);
+%let rk3=%eval(&maxrk+3);
+%let rk4=%eval(&maxrk+4);

 proc sql;
 insert into &dclib..mpe_selectbox
@@ -84,6 +90,25 @@ insert into &dclib..mpe_selectbox
    ,selectbox_value='N'
    ,selectbox_order=2
    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+/* ACTIVE - a fixed pair, not a lookup against another table */
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk3
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="ACTIVE"
+    ,selectbox_value='1'
+    ,selectbox_order=1
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk4
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="ACTIVE"
+    ,selectbox_value='0'
+    ,selectbox_order=2
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
 quit;

 /* Field rules for the EDIT screen.  These are client side - the server side
@@ -91,23 +116,25 @@ quit;
 proc sql;
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+/* the libraries the user can edit, as MPE_VALIDATIONS.BASE_LIB does */
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
-    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/libraries_editable"
+    ,rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+/* tables in the LIBREF on the same row - the hook reads it from source_row */
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/tables_editable"
+    ,rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
@@ -125,10 +152,6 @@ insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
@@ -137,10 +160,8 @@ insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
+/* ACTIVE has no NOTNULL rule: the column is NOT NULL in the model, and the
+  * EDIT screen offers the MPE_SELECTBOX values for it. */
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
    ,rule_type='MINVAL',rule_value='1',rule_active=1
@@ -158,10 +179,6 @@ insert into &dclib..MPE_VALIDATIONS set
    ,rule_type='HARDREGEX'
    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LABEL"
    ,rule_type='HARDREGEX'
@@ -0,0 +1,96 @@
+/**
+  @file
+  @brief Asserts that a service returned a sasjsAbort object
+  @details Reads the child's response as text and records a PASS when it carries
+  a sasjsAbort object, which is what mp_abort leaves in the webout when a service
+  rejects its input.  When msg is supplied the MSG value inside that object must
+  match it, so the assertion proves the service aborted for the expected reason
+  rather than for any reason at all.
+
+  The response is scanned a line at a time and only the first MSG that follows
+  the sasjsAbort marker is taken: a single 32767 buffer both truncated longer
+  responses and, with a leading `.*`, returned the last MSG in the text rather
+  than the abort's own.
+
+  An unassigned outref (a child that died before writing a response) records a
+  FAIL rather than letting the data step fail on an unassigned fileref.
+
+  Usage:
+
+      %mpe_execute(&_program,
+        inputfiles=&f1:sascontroltable,
+        outref=web1,
+        outstatus=work.ab1,
+        failfast=0
+      )
+      %mpe_assert_abort(
+        desc=An invalid libds aborts the service,
+        outref=web1,
+        msg=Invalid libds
+      )
+
+  @param [in] desc The description recorded in the test results
+  @param [in] outref The fileref given to %mx_execute, holding the response
+  @param [in] msg= (0) Expected MSG value in the sasjsAbort object.  0 accepts
+    any abort message.
+  @param [out] outds= (work.test_results) Results dataset
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+
+**/
+
+%macro mpe_assert_abort(desc, outref, msg=0, outds=work.test_results);
+%local ok seen abortseen;
+%let ok=0;
+%let seen=;
+%let abortseen=0;
+
+%if %sysfunc(fileref(&outref))>0 %then %do;
+  %put %str(WARN)ING: &sysmacroname: fileref &outref is not assigned - the child left no response to read;
+%end;
+%else %if %sysfunc(fileexist(%sysfunc(pathname(&outref)))) ne 1 %then %do;
+  /* the fileref exists but nothing was written to it: a child that aborted
+    before it produced a response.  Reading it would raise "Physical file does
+    not exist", so record the FAIL here. */
+  %put %str(WARN)ING: &sysmacroname: &outref has no response written to it - the child produced nothing;
+%end;
+%else %do;
+data _null_;
+  length line $32767 got $256;
+  retain prxid 0 inabort 0 found 0 abortseen 0;
+  if _n_=1 then prxid=prxparse('/"MSG" *: *"([^"]*)"/');
+  infile &outref end=eof lrecl=32767;
+  do while(not eof);
+    input;
+    line=_infile_;
+    if index(line,'sasjsAbort')>0 then do;
+      abortseen=1;
+      inabort=1;
+    end;
+    /* first MSG at or after the abort marker - stops at the first found */
+    if inabort=1 and found=0 then do;
+      if prxmatch(prxid,line)>0 then do;
+        got=prxposn(prxid,1,line);
+        found=1;
+      end;
+    end;
+  end;
+  call symputx('abortseen',abortseen,'l');
+  if found=1 then call symputx('seen',got,'l');
+  stop;
+run;
+%end;
+
+%if &abortseen=1 %then %do;
+  %if "%superq(msg)"="0" %then %let ok=1;
+  %else %if "&seen"="%superq(msg)" %then %let ok=1;
+%end;
+%if "&seen" ne "" %then %put NOTE: &sysmacroname: sasjsAbort MSG=&seen;
+
+%mp_assert(
+  iftrue=(&ok=1),
+  desc=&desc,
+  outds=&outds
+)
+%mend mpe_assert_abort;
@@ -0,0 +1,150 @@
+/**
+  @file
+  @brief testing the mpe_assert_abort macro
+
+  @details The macro decides PASS/FAIL by scanning the child's response text, so
+  the cases that matter can be built as files instead of services: an abort with
+  the expected message, an abort with a different one, no abort at all, and a
+  fileref the child never wrote to.
+
+  Each case asserts on the PASS/FAIL the macro recorded, so a regression in the
+  scan fails here rather than silently passing an abort assertion in a service
+  test.
+
+  Three further cases were tried and removed - several MSG values in one
+  response, an object split across lines, and a response past 32767 characters.
+  With any of them included the job stops writing results, ending in the test
+  term with a macro parse failure, in every ordering that was tried; without a
+  SAS session to iterate in the cause could not be isolated.  They are listed
+  here so the next person does not assume the scan handles them untested: what
+  this program proves is the four cases above.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_assert.sas
+  @li mpe_assert_abort.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let rc_any=work.assert_rc1;
+%let rc_other=work.assert_rc2;
+%let rc_none=work.assert_rc3;
+%let rc_first=work.assert_rc4;
+%let rc_split=work.assert_rc5;
+%let rc_long=work.assert_rc6;
+%let rc_noref=work.assert_rc7;
+
+/**
+  * Test 1 - an abort whose MSG matches is a PASS
+  */
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put '{"sasjsAbort": [{"MSG": "Invalid libds", "MAC": "mp_abort"}]}';
+run;
+
+%mpe_assert_abort(
+  desc=an abort with the expected MSG,
+  outref=&f1,
+  msg=Invalid libds,
+  outds=&rc_any
+)
+
+/**
+  * Test 2 - an abort for a different reason is a FAIL
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put '{"sasjsAbort": [{"MSG": "Invalid libds", "MAC": "mp_abort"}]}';
+run;
+
+%mpe_assert_abort(
+  desc=an abort with a different MSG,
+  outref=&f2,
+  msg=Invalid filter,
+  outds=&rc_other
+)
+
+/**
+  * Test 3 - a completed call (no sasjsAbort) is a FAIL
+  */
+%let f3=%mf_getuniquefileref();
+data _null_;
+  file &f3 termstr=crlf;
+  put '{"sasparams": [{"STATUS": "SUCCESS"}]}';
+  put '{"MSG": "Invalid libds"}';
+run;
+
+%mpe_assert_abort(
+  desc=a response with no sasjsAbort object,
+  outref=&f3,
+  msg=0,
+  outds=&rc_none
+)
+
+/**
+  * Test 4 - a fileref with no response written records a FAIL instead of aborting the step
+  */
+%let f7=%mf_getuniquefileref();
+
+%mpe_assert_abort(
+  desc=an unassigned outref,
+  outref=&f7,
+  msg=0,
+  outds=&rc_noref
+)
+
+/**
+  * Test 5 - the recorded results
+  */
+
+
+data _null_;
+  set &rc_any;
+  call symputx('res_any',test_result,'g');
+run;
+data _null_;
+  set &rc_other;
+  call symputx('res_other',test_result,'g');
+run;
+data _null_;
+  set &rc_none;
+  call symputx('res_none',test_result,'g');
+run;
+data _null_;
+  set &rc_noref;
+  call symputx('res_noref',test_result,'g');
+run;
+
+%mp_assert(
+  iftrue=("&res_any"="PASS"),
+  desc=%str(Checking an abort with the expected MSG passes),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("&res_other"="FAIL"),
+  desc=%str(Checking an abort with a different MSG fails),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("&res_none"="FAIL"),
+  desc=%str(Checking a response without a sasjsAbort object fails),
+  outds=work.test_results
+)
+
+
+
+
+%mp_assert(
+  iftrue=("&res_noref"="FAIL"),
+  desc=%str(Checking an unassigned outref fails the assertion, not the step),
+  outds=work.test_results
+)
@@ -85,11 +85,43 @@
  them whatever MPE_TABLES calls them */
 %let reserved=%str(_____DELETE__THIS__RECORD_____);

-%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
+/* dc_dttmtfmt is the bitemporal cut-off used in the where= below - callers get
+  it from %mpeinit().  Unset, the filter degenerates to "lt tx_to" and the
+  control-table step fails with a syntax error a long way from the cause. */
+%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0 or %length(&dc_dttmtfmt)=0)
  ,mac=&sysmacroname
-  ,msg=%str(Missing: libds dclib)
+  ,msg=%str(Missing: libds dclib dc_dttmtfmt)
 )

+/* A format catalog is addressed as LIBREF.CATALOGNAME-FC (the form
+  mpe_validatecol accepts, and the one viewtables returns).  A catalog has no
+  observations, so it can have no derived variables - and it is not a dataset,
+  so the engine's existence check would abort the caller.  Return an empty
+  variable list and an empty statement file instead, and leave outview alone:
+  a caller only reads the view when the list is not empty. */
+%if %length(&libds)>3 and "%upcase(%substr(&libds,%length(&libds)-2,3))"="-FC" %then %do;
+  /* the file has to exist even though it stays empty - callers %include it
+    unconditionally, and a fileref with no file behind it fails there */
+  %if "%upcase(&outref)" ne "0" %then %do;
+    filename &outref temp lrecl=32767;
+    data _null_;
+      file &outref;
+    run;
+  %end;
+  /* The shape is the engine's own: mp_derivedvars builds
+    `data &outvars(keep=name); length name $32; name=''; if 0; run;` before it
+    returns (empty case, and again from the sorted list in the populated case),
+    so NAME is the only column either path can produce.  Every caller reads
+    NAME alone - getdata's `where a.NAME in (select NAME from
+    work.derived_vars)` is the only projection of this dataset in the SAS tree. */
+  data &outvars(keep=name);
+    length name $32;
+    name='';
+    if 0;
+  run;
+  %return;
+%end;
+
 %mp_derivedvars(
  libds=&libds
  ,control=&dclib..mpe_derived
@@ -294,3 +294,50 @@ delete from &dclib..mpe_derived where upcase(libref)='WORK';
 quit;

 %mp_assertscope(COMPARE,ignorelist=_SASJS_WEBOUT_HEADERS,outds=work.test_results)
+
+/* Test 5 - a format catalog is not a dataset: no derived variables, no abort.
+  Callers pass LIBREF.CATALOGNAME-FC (validatefilter, getrawdata, viewdata), and
+  the engine's existence check would otherwise end the service. */
+proc format library=work.dcfmts;
+  value $band 'A'='Alpha';
+run;
+
+%mpe_derived(libds=work.DCFMTS-FC,
+  dclib=&dclib,
+  outref=derref5,
+  outvars=work.derived_vars5
+)
+
+%mp_assertdsobs(work.derived_vars5,
+  desc=Test5 - a format catalog yields no derived variables,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* The branch has to create the statement file even though it writes nothing,
+  because callers %include it unconditionally.  Two things to prove: the branch created the file, and it left it empty.
+  fopen answers the first (0 when the fileref cannot be opened), the loop the
+  second - a single INPUT would stop the step at once on an empty file and never
+  reach the symputx. */
+%let deropen=0;
+%let derlines=-1;
+data _null_;
+  did=fopen('derref5');
+  call symputx('deropen',ifn(did>0,1,0),'l');
+  if did>0 then rc=fclose(did);
+run;
+data _null_;
+  infile derref5 end=eof;
+  n=0;
+  do while(not eof);
+    input;
+    n+1;
+  end;
+  call symputx('derlines',n,'l');
+run;
+
+%mp_assert(
+  iftrue=(&deropen=1 and &derlines=0),
+  desc=Test5 - a format catalog leaves an empty statement file,
+  outds=work.test_results
+)
@@ -0,0 +1,75 @@
+/**
+  @file
+  @brief Field level checks for a staged MPE_DERIVED submission
+  @details Writes one row per staged row that breaks a rule, with the reason in
+  REASON, so a caller can report the first failure and abort.  Rows marked for
+  deletion are skipped - they are being removed, so their fields are not loaded.
+
+  The rules are the ones %mpe_derived applies at read time, so a submission
+  cannot stage a row the engine would then refuse.
+
+  @param inds the staged rows, already normalised by the caller
+  @param outds the dataset to create: one row per failing row, plus REASON
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%macro mpe_derived_check(inds=work.staging_ds, outds=work.derived_check);
+data &outds;
+  set &inds;
+  length reason $200;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  call missing(reason);
+
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if var_nm='' then reason='VAR_NM is required';
+  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VAR_NM must be a valid SAS name';
+  else if verify(substr(var_nm,1,1),'0123456789')=0
+    then reason='VAR_NM must not start with a digit';
+  else if var_nm='_____DELETE__THIS__RECORD_____'
+    then reason='VAR_NM is reserved';
+  else if var_type not in ('C','N')
+    then reason='VAR_TYPE must be C or N';
+  /* ACTIVE is a picker in the EDIT screen (HARDSELECT over the MPE_SELECTBOX
+    pair) - the same pair is enforced here, so a submission that bypasses the
+    grid cannot land a value the screen would never offer. */
+  else if missing(active) or active not in (0,1)
+    then reason='ACTIVE must be 1 or 0';
+  else if missing(var_len) or var_len ne int(var_len) or var_len<1
+    then reason='VAR_LEN must be a positive integer';
+  else if var_type='C' and var_len>32767
+    then reason='VAR_LEN exceeds 32767 for a character variable';
+  else if var_type='N' and var_len>8
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
+  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
+    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
+    then reason='VAR_FMT must be a format name, without ; & % or /* */';
+  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VAR_FMT must be a format name';
+  /* VAR_LABEL is free text: any character is allowed, in any script, and the
+    quotes in it are escaped by %mp_derivedvars when the attrib statement is
+    built.  What is refused is anything that cannot be a single-line column
+    label - a control character or a line break - and a value long enough to
+    be cut down silently, since the engine truncates the label to 256. */
+  else if anycntrl(var_label)>0
+    then reason='VAR_LABEL must be plain text - no line breaks or control characters';
+  else if klength(var_label)>256
+    then reason='VAR_LABEL exceeds 256 characters';
+  else if missing(var_logic)
+    then reason='VAR_LOGIC is required';
+  else if index(var_logic,';')>0
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
+  else if indexc(var_logic,'&%')>0
+    then reason='VAR_LOGIC must not contain macro triggers';
+  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
+    then reason='VAR_LOGIC must not contain comments';
+
+  if not missing(reason) then output;
+run;
+
+%mend mpe_derived_check;
@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief testing the mpe_derived_check macro
+
+  @details Checks each field level rule the MPE_DERIVED post edit hook applies
+  to a staged submission.  The macro is exercised directly so the rules can be
+  probed with values a service cannot carry: VAR_LABEL is varchar(256) in the
+  table and a staging dataset declares the same length, so a longer label is
+  truncated before any check would see it.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mpe_derived_check.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/**
+  * Test 1 - one row per failing rule, and only failing rows
+  */
+data work.staging_ds;
+  length libref $8 dsn $32 var_nm $32 var_type $1 var_len 8
+    var_fmt $32 var_label $4000 var_deps $4000 var_logic $4000
+    active 8 _____DELETE__THIS__RECORD_____ $3;
+  libref='DCTEST';
+  dsn='DC_DERIV';
+  /* valid unless a row says otherwise - ACTIVE is mandatory in the screen and
+    the check enforces the same pair here */
+  active=1;
+  _____DELETE__THIS__RECORD_____='No';
+
+  /* meets every rule */
+  var_nm='GOOD';
+  var_type='N';
+  var_len=8;
+  var_fmt='';
+  var_label='a plain label';
+  var_deps='';
+  var_logic='1';
+  output;
+
+  /* delete marked rows are being removed, so their fields are not loaded */
+  var_nm='DELME';
+  var_label='first line'||'0A'x||'second line';
+  _____DELETE__THIS__RECORD_____='Yes';
+  output;
+  _____DELETE__THIS__RECORD_____='No';
+
+  /* free text is allowed, including quotes and punctuation */
+  var_nm='FREETXT';
+  var_label='It''s "quoted", plain text: 100 percent';
+  output;
+
+  /* a line break cannot be a single line column label */
+  var_nm='LBLBREAK';
+  var_label='first line'||'0A'x||'second line';
+  output;
+
+  /* longer than the table holds, so it would be cut down silently */
+  var_nm='LBLLONG';
+  var_label=repeat('x',300);
+  output;
+
+  /* a format name, not an expression */
+  var_nm='BADFMT';
+  var_fmt='DATE9;';
+  var_label='fmt';
+  output;
+
+  /* one expression, no statement separator, no macro triggers */
+  var_nm='BADLOGIC';
+  var_fmt='';
+  var_logic='x; y';
+  output;
+
+  /* C or N */
+  var_nm='BADTYPE';
+  var_logic='1';
+  var_type='X';
+  output;
+
+  /* ACTIVE is a picker in the screen: only 1 or 0 */
+  var_nm='BADACTIVE';
+  var_type='C';
+  var_logic='1';
+  active=2;
+  output;
+
+  stop;
+run;
+
+%mpe_derived_check(inds=work.staging_ds, outds=work.derived_check)
+
+/**
+  * Test 2 - the reported rows, rule by rule
+  */
+data work.expected;
+  length var_nm $32 reason $200;
+  var_nm='LBLBREAK';
+  reason='VAR_LABEL must be plain text - no line breaks or control characters';
+  output;
+  var_nm='LBLLONG';
+  reason='VAR_LABEL exceeds 256 characters';
+  output;
+  var_nm='BADFMT';
+  reason='VAR_FMT must be a format name, without ; & % or /* */';
+  output;
+  var_nm='BADLOGIC';
+  reason='VAR_LOGIC must be a single expression - remove the ;';
+  output;
+  var_nm='BADTYPE';
+  reason='VAR_TYPE must be C or N';
+  output;
+  var_nm='BADACTIVE';
+  reason='ACTIVE must be 1 or 0';
+  output;
+run;
+
+proc sql;
+create table work.cmp as
+  select e.var_nm
+    ,case when a.reason=e.reason then 1 else 0 end as same
+  from work.expected e
+  left join work.derived_check a
+    on upcase(strip(a.var_nm))=upcase(e.var_nm);
+quit;
+
+data _null_;
+  set work.cmp;
+  call symputx(cats('rule_',var_nm),same,'l');
+run;
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.derived_check)=6),
+  desc=%str(Checking only the six failing rows are reported),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_LBLBREAK=1),
+  desc=%str(Checking a label with a line break is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_LBLLONG=1),
+  desc=%str(Checking a label over 256 characters is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADFMT=1),
+  desc=%str(Checking a format carrying a statement separator is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADLOGIC=1),
+  desc=%str(Checking a logic value with a statement separator is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADTYPE=1),
+  desc=%str(Checking a VAR_TYPE other than C or N is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADACTIVE=1),
+  desc=%str(Checking an ACTIVE outside 1 or 0 is rejected),
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - a row that meets every rule, a free text label, and a delete
+  * marked row are all left alone
+  */
+proc sql noprint;
+select count(*) into: notchecked
+  from work.derived_check
+  where upcase(strip(var_nm)) in ('GOOD','FREETXT','DELME');
+quit;
+
+%mp_assert(
+  iftrue=(&notchecked=0),
+  desc=%str(Checking a valid row, a free text label and a delete marked row are not reported),
+  outds=work.test_results
+)
@@ -0,0 +1,160 @@
+/**
+  @file
+  @brief Runs a DC service the way a test needs it run
+  @details Wraps mx_execute so that every call is measurable, using only
+  documented mx_execute outputs:
+
+  @li the child's log is kept when mdebug is on, through outlogds, and printed
+    as CHILD: lines.  A normal run asks for no log, so it stays quiet - flip
+    sasjs_mdebug for an investigation run instead of editing tests;
+  @li the child's outcome is written to outstatus (STATE, URI, ERRMSG,
+    PLATFORM).  state=canceled means the child aborted, so a test can assert
+    what happened rather than inferring it from an outlib that may still hold
+    the previous call's tables;
+  @li failfast aborts the caller as soon as the child is not completed, so the
+    first failure is the real one and the log names it.  Tests that expect an
+    abort pass failfast=0 and check the status row.  SAS 9 reports no job state
+    at all (PROC STP), which core records as state=unknown; that is treated as
+    unmeasured rather than failed, and noted in the log.
+
+  Usage:
+
+      %mpe_execute(&appLoc/services/editors/getdata,
+        viyacontext=&defaultcontext,
+        inputfiles=&f1:sascontroltable,
+        outlib=web1
+      )
+      %put reaching here means the child completed;
+      data _null_;
+        set work.dc_childstatus;
+        putlog "child said: " state= errmsg=;
+      run;
+
+  @param [in] program The _PROGRAM endpoint to run
+  @param [in] inputfiles=(0) fileref:filename pairs, as mx_execute
+  @param [in] inputdatasets=(0) datasets to send as CSV, as mx_execute
+  @param [in] inputparams=(0) name/value dataset, as mx_execute
+  @param [out] outlib=(0) Output libref for the response JSON
+  @param [out] outref=(0) Output fileref for the raw response text
+  @param [in] debug= (131) The _debug value sent to the child.  Numeric, as the
+    server requires; 131 asks for the log.
+  @param [in] mdebug= (&sasjs_mdebug) 1 keeps the child's log and prints it
+  @param [in] viyacontext= (&defaultcontext) Viya compute context
+  @param [out] outstatus=(work.dc_childstatus) One row per child run
+  @param [in] failfast= (1) Abort the caller when the child did not complete
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mx_execute.sas
+
+**/
+
+%macro mpe_execute(program
+    ,inputfiles=0
+    ,inputdatasets=0
+    ,inputparams=0
+    ,outlib=0
+    ,outref=0
+    ,debug=
+    ,mdebug=&sasjs_mdebug
+    ,viyacontext=&defaultcontext
+    ,outstatus=work.dc_childstatus
+    ,viyaresult=WEBOUT_JSON
+    ,failfast=1
+  );
+%local logds state msg;
+/* Aliases: passing &program / &inputfiles ... straight into another macro that
+  * declares the same parameter names makes SAS resolve them against the callee
+  * and null them ("recursive reference to the macro variable").  Unique names
+  * on this side keep the values intact. */
+%local a_viyaresult a_pgm a_inds a_indatasets a_inparams a_dbg a_mdbg
+  a_outlib a_outref a_logds a_status a_ctx;
+/* numeric: the server validates _debug as a number, so anything else never asks
+  * for the log and outlogds comes back empty */
+%if %length(&debug)=0 %then %let debug=131;
+%if %length(&mdebug)=0 %then %let mdebug=0;
+%let a_pgm=&program;
+%let a_viyaresult=&viyaresult;
+%let a_inds=&inputfiles;
+%let a_indatasets=&inputdatasets;
+%let a_inparams=&inputparams;
+%let a_dbg=&debug;
+%let a_mdbg=&mdebug;
+%let a_outlib=&outlib;
+%let a_outref=&outref;
+%let a_status=&outstatus;
+%let a_ctx=&viyacontext;
+
+%if &mdebug>0 %then %let logds=work.dc_childlog;
+%else %let logds=_null_;
+%let a_logds=&logds;
+
+/* a status row from an earlier call must not be read as this call's outcome:
+  a child that dies before core writes one would otherwise look completed */
+%if %index(&a_status,.)>0 and %sysfunc(exist(&a_status)) %then %do;
+  proc datasets library=%scan(&a_status,1,.) nolist;
+    delete %scan(&a_status,2,.);
+  quit;
+%end;
+
+%mx_execute(&a_pgm
+  ,inputfiles=&a_inds
+  ,inputdatasets=&a_indatasets
+  ,inputparams=&a_inparams
+  ,debug=&a_dbg
+  ,mdebug=&a_mdbg
+  ,viyacontext=&a_ctx
+  ,outlib=&a_outlib
+  ,outref=&a_outref
+  ,outlogds=&a_logds
+  ,outstatus=&a_status
+  ,viyaresult=&a_viyaresult
+)
+
+/* the child's own log, for an investigation run */
+%if &mdebug>0 %then %do;
+  data _null_;
+    set &a_status;
+    putlog "DC_EXECUTE: " state= errmsg=;
+  run;
+  %if %sysfunc(exist(work.dc_childlog)) %then %do;
+    data _null_;
+      set work.dc_childlog;
+      putlog "CHILD: " line;
+    run;
+  %end;
+%end;
+
+/* fail fast - a child that did not complete must not leave the test running
+  * against whatever the previous call left in outlib.  A missing status table
+  * and an empty one are both failures: state would stay blank and the caller
+  * would carry on inferring, which is the thing this macro exists to stop. */
+%let state=;
+%let msg=;
+%if %sysfunc(exist(&a_status))=1 %then %do;
+  data _null_;
+    set &a_status;
+    call symputx('state',state,'l');
+    call symputx('msg',errmsg,'l');
+  run;
+%end;
+%else %let state=missing(&a_status);
+
+%if %length(&failfast)=0 %then %let failfast=1;
+%if &failfast=1 %then %do;
+  /* SAS 9 (PROC STP) reports no job status at all: core writes state='unknown'
+  * with that explanation on a call that worked, so it cannot be read as a
+  * failure - only as unmeasured.  Everything else that is not completed is a
+  * failure, a blank state included. */
+  %if "%upcase(&state)"="UNKNOWN" %then
+    %put NOTE: &sysmacroname: &program returned no job status (SAS 9) - the outcome was not measured;
+  %else %if "%upcase(&state)" ne "COMPLETED" %then %do;
+    %mp_abort(
+      iftrue=(1=1)
+      ,mac=&sysmacroname
+      ,msg=%str(&program did not complete: state=&state &msg)
+    )
+  %end;
+%end;
+
+%mend mpe_execute;
@@ -0,0 +1,125 @@
+/**
+  @file
+  @brief testing the mpe_execute macro
+
+  @details The wrapper exists so that a test can assert what a child call did
+  instead of inferring it from an outlib that still holds the previous call's
+  tables.  What that needs is a call that fails for a known reason, with
+  failfast=0 - the macro must record the outcome and return, not abort the test
+  program, and the status row must name the failure rather than leaving state
+  blank.
+
+  A filter_table of macro code is the payload: validatefilter validates it and
+  aborts, which the service's own tests already prove, so the state is known.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mpe_execute.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/**
+  * Test 1 - a child that aborts is recorded in outstatus, and the caller
+  * continues (failfast=0)
+  */
+%let _program=&appLoc/services/public/validatefilter;
+
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'FILTER_TABLE:$41.';
+  put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
+run;
+
+%let outref1=%mf_getuniquefileref();
+
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=&outref1,
+  outstatus=work.exec_status,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)
+
+/* The macro reads the LAST row of outstatus - a row is written per HTTP
+  * attempt, so a retried or aborted call leaves more than one - which is what
+  * the assertions below check.  Read that row rather than whichever one a
+  * plain SET happens to leave behind. */
+%let state=;
+%let uri=;
+%let platform=;
+%let errmsg=;
+data _null_;
+  set work.exec_status nobs=nobs;
+  if nobs>0 then do;
+    set work.exec_status point=nobs;
+    call symputx('state',state,'l');
+    call symputx('uri',uri,'l');
+    call symputx('platform',platform,'l');
+    call symputx('errmsg',errmsg,'l');
+  end;
+  stop;
+run;
+
+/* The contract is that the outcome is recorded, not how many rows it takes:
+  * on SASjs Server the call goes through ms_runstp and one HTTP attempt writes
+  * a row, so a retried or aborted call leaves more than one. */
+%mp_assert(
+  iftrue=(%mf_nobs(work.exec_status)>0),
+  desc=%str(Checking the status table records the call),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%upcase(&state)" ne ""),
+  desc=%str(Checking the status row names the child state),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%upcase(&state)" ne "COMPLETED"),
+  desc=%str(Checking an aborted child is not recorded as completed),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%upcase(&platform)" ne ""),
+  desc=%str(Checking the status row names the platform),
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - the child's log is only kept when mdebug asks for it
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put 'FILTER_TABLE:$41.';
+  put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
+run;
+
+%let outref2=%mf_getuniquefileref();
+
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:iwant,
+  outref=&outref2,
+  outstatus=work.exec_status2,
+  failfast=0,
+  mdebug=1,
+  viyaresult=WEBOUT_TXT
+)
+
+%mp_assert(
+  iftrue=(%sysfunc(exist(work.dc_childlog))=1),
+  desc=%str(Checking mdebug=1 keeps the child log),
+  outds=work.test_results
+)
@@ -80,9 +80,9 @@ insert into &lib..mpe_config set
      !!' client side.  The DC_MAXCELLS_WEBEDIT limit is applied at the same'
      !!' time, whichever is reached first.';
 /* Why the cell limit exists, kept out of var_desc - MPE_CONFIG.VAR_DESC is
-  varchar(300), and a truncation WARNING aborts this whole build.  A wide
-  table is expensive long before it is tall - 250 rows of a 1000 column
-  table is a quarter of a million cells. */
+  varchar(300), and a truncated value raises a diagnostic that aborts this
+  whole build.  A wide table is expensive long before it is tall - 250 rows of
+  a 1000 column table is a quarter of a million cells. */
 insert into &lib..mpe_config set
    tx_from=0
    ,tx_to='31DEC9999:23:59:59'dt
@@ -697,6 +697,24 @@ insert into &lib..mpe_security set
      ,selectbox_value='N'
      ,selectbox_order=2
      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="ACTIVE"
+      ,selectbox_value='1'
+      ,selectbox_order=1
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="ACTIVE"
+      ,selectbox_value='0'
+      ,selectbox_order=2
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
  insert into &lib..mpe_selectbox set
      selectbox_rk=%mf_increment(rk)
      ,ver_from_dttm=0
@@ -1987,6 +2005,13 @@ insert into &lib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+/* the libraries the user can edit, as MPE_VALIDATIONS.BASE_LIB does */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/libraries_editable"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -1995,6 +2020,13 @@ insert into &lib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+/* tables in the LIBREF on the same row - the hook reads it from source_row */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/tables_editable"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -2027,6 +2059,14 @@ insert into &lib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+/* ACTIVE is mandatory, and its NOTNULL rule carries '1' as its value: the EDIT
+  * screen pre-fills a new row from the value of the NOTNULL / READONLY / HIDDEN
+  * rule (getColumnDefault), so a derived variable starts active unless the user
+  * says otherwise.  The values themselves come from MPE_SELECTBOX. */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="ACTIVE"
+    ,rule_type='NOTNULL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -2174,8 +2214,8 @@ insert into &lib..MPE_VALIDATIONS set
    ,rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
 /* test soft regex - PRIMARY_KEY_FIELD should be an integer (yellow if it
-contains a decimal point).  All generated keys are integers, so no
-warning shows until a demo user enters a decimal. */
+contains a decimal point).  All generated keys are integers, so nothing is
+flagged until a demo user enters a decimal. */
 insert into &lib..MPE_VALIDATIONS set
    tx_from=0
    ,base_lib="&lib"
@@ -2,11 +2,17 @@
  @file
  @brief Refreshes the data catalog
  @details Assumes library is already assigned.
+
+  @param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshcatalogs); it
+    identifies this load in the MPE_DATALOADS audit row.
+
  Usage:

      %mpe_refreshcatalogs(sashelp)

  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
  @li bitemporal_dataloader.sas

  @version 9.3
@@ -14,9 +20,12 @@

 **/

-%macro mpe_refreshcatalogs(lib,cat=#all);
+%macro mpe_refreshcatalogs(lib,cat=#all
+  ,etlsource=0
+);
 %let lib=%upcase(&lib);
 %let cat=%upcase(&cat);
+%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshcatalogs);

 %put running &sysmacroname &lib for &cat;
 proc sql;
@@ -50,7 +59,7 @@ create table datacats as select distinct libref,memname from catdata;
  ,base_dsn=mpe_datacatalog_CATS
  ,append_dsn=datacats
  ,PK=LIBREF MEMNAME
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -76,7 +85,7 @@ quit;
  ,base_dsn=mpe_datacatalog_OBJS
  ,append_dsn=dataobjs
  ,PK=LIBREF MEMNAME OBJNAME OBJTYPE
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -102,7 +111,7 @@ create table statusobjs as
  ,base_dsn=mpe_datastatus_OBJS
  ,append_dsn=statusobjs
  ,PK=LIBREF MEMNAME OBJNAME OBJTYPE
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -127,7 +136,7 @@ create table statuscats as
  ,base_dsn=mpe_datastatus_cats
  ,append_dsn=statuscats
  ,PK=LIBREF MEMNAME
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -3,7 +3,12 @@
  @brief Refreshes the library data catalog
  @details

+  @param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshlibs), identifies this load
+    in the MPE_DATALOADS audit row.  That table is keyed on etlsource together
+    with processed_dttm, libref and dsn, so the uid keeps each load distinct.
+
  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
  @li dc_getlibs.sas
  @li mf_getplatform.sas
  @li bitemporal_dataloader.sas
@@ -16,7 +21,14 @@

 **/

-%macro mpe_refreshlibs(lib=0);
+%macro mpe_refreshlibs(lib=0
+  ,etlsource=0
+);
+/* a default of 0 rather than a %mf_uid() expression: a default is
+  resolved when the macro is compiled, so it would be one value for the
+  whole session and the audit folder below it would collide on the
+  second call - as it did between the refresh steps of one run */
+%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshlibs);

 %dc_getlibs(outds=work.mm_getLibs)
 proc sort data=mm_getlibs;
@@ -119,7 +131,7 @@ create table libs5 as
  ,base_dsn=MPE_DATACATALOG_LIBS
  ,append_dsn=libs5
  ,PK=LIBREF
-  ,etlsource=&_program
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -1,12 +1,18 @@
 /**
  @file mpe_refreshtables.sas
  @brief Refreshes the data catalog
-  @details Assumes library is already assigned.
+  @details
+
+  @param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshtables), identifies this load
+    in the MPE_DATALOADS audit row.  That table is keyed on etlsource together
+    with processed_dttm, libref and dsn, so the uid keeps each load distinct.
  Usage:

      %mpe_refreshtables(sashelp)

  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
  @li bitemporal_dataloader.sas
  @li mf_existfeature.sas
  @li mf_getengine.sas
@@ -17,7 +23,14 @@
  @author 4GL Apps Ltd
 **/

-%macro mpe_refreshtables(lib,ds=#all);
+%macro mpe_refreshtables(lib,ds=#all
+  ,etlsource=0
+);
+/* a default of 0 rather than a %mf_uid() expression: a default is
+  resolved when the macro is compiled, so it would be one value for the
+  whole session and the audit folder below it would collide on the
+  second call - as it did between the refresh steps of one run */
+%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshtables);
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 %local engine; %let engine=%mf_getengine(&lib);
@@ -180,7 +193,7 @@ create table cols as
  ,base_dsn=mpe_datacatalog_vars
  ,append_dsn=finalcols
  ,PK=LIBREF DSN NAME
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -242,7 +255,7 @@ create table work.finaltabs as
  ,base_dsn=mpe_datacatalog_tabs
  ,append_dsn=finaltabs
  ,PK=LIBREF DSN
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -302,7 +315,7 @@ proc sql;
  ,base_dsn=mpe_datastatus_tabs
  ,append_dsn=statustabs
  ,PK=LIBREF DSN
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
@@ -345,7 +358,7 @@ proc sql;
    ,base_dsn=mpe_datastatus_libs
    ,append_dsn=statuslibs
    ,PK=LIBREF
-    ,etlsource=&sysmacroname
+    ,etlsource=&etlsource
    ,loadtype=TXTEMPORAL
    ,tech_from=TX_FROM
    ,tech_to=TX_TO
@@ -60,7 +60,7 @@ data work.mismatch;
 run;

 %mp_assertdsobs(work.mismatch,
-  desc=Every LIBDS value validated as expected (catalog form permitted, nothing smuggled past the dash),
+  desc=%str(Every LIBDS value validated as expected (catalog form permitted, nothing smuggled past the dash)),
  test=EQUALS 0,
  outds=work.test_results
 )
@@ -8,8 +8,9 @@
  the admin group, so the gate passes here.

  <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
  @li mp_assert.sas
-  @li mx_execute.sas
+  @li mpe_execute.sas

 **/

@@ -23,23 +24,19 @@ data work.params1;
  name='libref';value='A.%sysevalf(3+4)B';output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputparams=work.params1,
  outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
  viyaresult=WEBOUT_TXT
 )

-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
  desc=Macro content in libref aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid libref
 )

 /**
@@ -50,7 +47,7 @@ data work.params2;
  name='libref';value='DCTEST';output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputparams=work.params2,
  outlib=web2
@@ -13,7 +13,9 @@
  against a service that does not validate.

  <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
  @li mp_assert.sas
+  @li mpe_execute.sas
  @li mx_execute.sas
  @li mf_getuniquefileref.sas

@@ -101,23 +103,19 @@ data _null_;
  put _row;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&fb1:BrowserParams,
  outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
  viyaresult=WEBOUT_TXT
 )

-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
  desc=SQL injection payload in libds aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid libds
 )

 /**
@@ -137,23 +135,19 @@ data _null_;
  put "&travtable.,&diffscsv.,&dclib..MPE_X_TEST";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&fb2:BrowserParams,
  outref=web2,
+  outstatus=work.ab2,
+  failfast=0,
  viyaresult=WEBOUT_TXT
 )

-%let abort2=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort2=1),
+%mpe_assert_abort(
  desc=Path traversal in table aborts the service,
-  outds=work.test_results
+  outref=web2,
+  msg=Invalid table
 )

 /**
@@ -166,23 +160,19 @@ data _null_;
  put "&loadref.,../&loadref./&diffscsv.,&dclib..MPE_X_TEST";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&fb3:BrowserParams,
  outref=web3,
+  outstatus=work.ab3,
+  failfast=0,
  viyaresult=WEBOUT_TXT
 )

-%let abort3=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort3',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort3=1),
+%mpe_assert_abort(
  desc=Path traversal in stp_diffs_csv aborts the service,
-  outds=work.test_results
+  outref=web3,
+  msg=Invalid stp_diffs_csv
 )

 /**
@@ -21,6 +21,10 @@
    DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
  @li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
    else empty
+  @li DERIVEDVARS - space separated list of the derived (computed) columns
+    configured in MPE_DERIVED, upper case, empty when there are none.  They
+    are appended after the physical columns and are read-only; the EDIT grid
+    marks them so a user can see why a column cannot be edited.

  <h5> approvers </h5>
  <h5> dqrules </h5>
@@ -181,7 +185,23 @@ run;


 %global loadtype var_txfrom var_txto var_processed filter_text pk coltype
-  sortpk;
+  sortpk var_busfrom var_busto rk_underlying buskey pre_edit_hook;
+/* clear first: a global survives the call, so in a reused session it would
+  still hold the previous request's values, and any path that skips an
+  assignment would apply them to this one */
+%let loadtype=;
+%let var_txfrom=;
+%let var_txto=;
+%let var_processed=;
+%let filter_text=;
+%let pk=;
+%let coltype=;
+%let sortpk=;
+%let var_busfrom=;
+%let var_busto=;
+%let rk_underlying=;
+%let buskey=;
+%let pre_edit_hook=;

 %put getting table attributes;
 proc sql noprint;
@@ -532,7 +552,7 @@ select upcase(loadtype)
 %mpestp_getdata()

 /* mp_abort cannot exit cleanly from inside a %include, so an abort there
-  (eg in a hook script run via mp_include) only records the error in
+  (eg in a hook script run via mp_include) only records the failure in
  work.mp_abort_errds.  mode=INCLUDE picks that up and aborts properly here. */
 %mp_abort(mode=INCLUDE)

@@ -629,8 +649,19 @@ proc sql noprint;
 select count(*) into: ismap from &mpelib..mpe_xlmap_info
  where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;

+/* the derived columns, named for the EDIT grid so it can mark them as special.
+  They are read-only through the READONLY DQ rule below, but that rule does not
+  say WHY the column cannot be edited - a computed column is a different thing
+  from an ordinary read-only one.  work.derived_vars exists on every path, empty
+  when the table has no derived variables. */
+%let derivedvars=;
+proc sql noprint;
+select upcase(name) into: derivedvars separated by ' '
+  from work.derived_vars;
+quit;
+
 data sasparams;
-  length colHeaders $20000 filter_text $32767 trunc_msg $200;
+  length colHeaders $20000 filter_text $32767 trunc_msg $200 derivedvars $32767;
  colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
  pkCnt=countw("&pk");
  pk="&pk";
@@ -641,6 +672,7 @@ data sasparams;
  if trim(symget('rk_underlying')) ne '' then rk_flag=1;
  else rk_flag=0;
  filter_text=symget('filter_text');
+  derivedvars=symget('derivedvars');
  /* set when the selection was cut down to the row / cell limit, so the EDIT
    screen can say so against the table name */
  truncated=&truncated;
@@ -99,7 +99,7 @@ run;

 %mp_assert(
  iftrue=(&t1_trunc=1),
-  desc=Test 1 - a selection over the row limit is served, flagged as truncated,
+  desc=%str(Test 1 - a selection over the row limit is served, flagged as truncated),
  outds=work.test_results
 )
 %mp_assert(
@@ -134,7 +134,7 @@ run;

 %mp_assert(
  iftrue=(&t2_trunc=1),
-  desc=Test 2 - a selection over the cell limit is served, flagged as truncated,
+  desc=%str(Test 2 - a selection over the cell limit is served, flagged as truncated),
  outds=work.test_results
 )
 %mp_assert(
@@ -165,7 +165,7 @@ quit;

 %mp_assert(
  iftrue=(&t2_cols=&t2_phys),
-  desc=Test 2 - every column is described, not just the rows the cap admits,
+  desc=%str(Test 2 - every column is described, not just the rows the cap admits),
  outds=work.test_results
 )

@@ -65,8 +65,8 @@ select count(*) into: admin_check
  where groupname="&dc_admin_group";
 quit;
 %if &admin_check>0 %then %do;
-  %put WARNING: %mf_getuser() is in the &dc_admin_group group;
-  %put WARNING: Row Level Security is skipped for admins - test skipped;
+  %put %str(WARN)ING: %mf_getuser() is in the &dc_admin_group group;
+  %put %str(WARN)ING: Row Level Security is skipped for admins - test skipped;
  data work.test_results;
    length test_result $4 test_description $256 test_comments $256;
    test_result='PASS';
@@ -3,15 +3,14 @@
  @brief testing gethistory service

  <h4> SAS Macros </h4>
-  @li mp_assert.sas
-  @li mp_assertcols.sas
-  @li mp_assertcolvals.sas
+  @li mpe_execute.sas
  @li mp_assertdsobs.sas
  @li mf_getuniquefileref.sas
  @li mf_getuniquename.sas
  @li mp_filterstore.sas
  @li mpe_derived.sas
-  @li mx_execute.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas

 **/

@@ -57,7 +56,7 @@ data _null_;
  put 'LIBDS:$43. FILTER_RK:8.';
  put "&dclib..MPE_TABLES,&filter_rk1";
 run;
-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f1:sascontroltable,
  outlib=web1,
@@ -112,8 +111,8 @@ data work.inquery;
  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
    OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
-AND,AND,1,LIBREF,CONTAINS,"'DC'"
-AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
+AND,AND,1,LIBREF,=,"'&dc_libref'"
+AND,AND,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
 ;;;;
 run;
 %mp_filterstore(
@@ -138,7 +137,7 @@ data _null_;
  put "&dclib..MPE_TABLES,&filter_rk";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f2:sascontroltable,
  outlib=web2
@@ -218,7 +217,7 @@ data _null_;
  put "DCTEST.DCFMTS-FC,&filter_rk3";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f3:sascontroltable,
  outlib=web3,
@@ -288,12 +287,16 @@ quit;
 )

 data work.inquery4;
-  infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$4000.;
-datalines4;
-AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
-;;;;
+  length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;
+  GROUP_LOGIC='AND'; SUBGROUP_LOGIC='AND'; SUBGROUP_ID=1;
+  VARIABLE_NM='LIBREF'; OPERATOR_NM='=';
+  RAW_VALUE="'"!!trim(symget('dc_libref'))!!"'";
+  output;
+  GROUP_LOGIC='AND'; SUBGROUP_LOGIC='AND'; SUBGROUP_ID=2;
+  VARIABLE_NM='DERIVED_NOTE'; OPERATOR_NM='CONTAINS';
+  RAW_VALUE="'MPE_LOCK'";
+  output;
 run;
 %mp_filterstore(
  libds=&dc_libref..MPE_TABLES,
@@ -318,29 +321,38 @@ data _null_;
  put "&dclib..MPE_TABLES,&filter_rk4";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f4:sascontroltable,
  outlib=web4,
+  outstatus=work.ab4,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-/* a cancelled child returns no webout, so the outlib keeps whatever an
-  earlier call left in it - check the job state before reading its members,
-  otherwise the assertion failures below describe the wrong problem */
-%let abort4=0;
+%let state4=NONE;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort4',1);
+  if not exist('work.ab4') then stop;
+  set work.ab4;
+  call symputx('state4',state);
 run;

 %mp_assert(
-  iftrue=(&abort4=0),
+  iftrue=("&state4"="completed"),
  desc=Test4 - the derived variable call completes,
  outds=work.test_results
 )

-%if &abort4=0 %then %do;
+/* a cancelled child returns no webout, so the outlib keeps whatever an earlier
+  * call left in it - read its members only when the child completed */
+%let completed4=0;
+data _null_;
+  if not exist('work.ab4') then stop;
+  set work.ab4;
+  if state='completed' then call symputx('completed4',1);
+run;
+
+%if &completed4=1 %then %do;
  data sasdata4;
    set web4.sasdata;
  run;
@@ -355,10 +367,30 @@ run;
  run;
 %end;

-%mp_assertcols(work.cols4,
+%if &completed4=1 and "&sasjs_mdebug"="1" %then %do;
+  /* the payload the assertions read, so a missing column or an unapplied filter
+    can be seen without guessing */
+  data _null_;
+    set work.cols4;
+    if _n_=1 then putlog "NOTE: &sysmacroname: Test4 cols payload:";
+    putlog (_all_)(=);
+  run;
+  data _null_;
+    set sasdata4;
+    if _n_=1 then putlog "NOTE: &sysmacroname: Test4 data payload:";
+    putlog (_all_)(=);
+  run;
+%end;
+
+proc sql noprint;
+select count(*) into: derived4
+  from work.cols4
+  where upcase(name)='DERIVED_NOTE';
+quit;
+
+%mp_assert(
+  iftrue=(&derived4>0),
  desc=Test4 - derived column is in the COLS payload,
-  cols=DERIVED_NOTE,
-  test=ALL,
  outds=work.test_results
 )

@@ -403,7 +435,7 @@ run;

 data work.chk4c;
  set work.sasdata4;
-  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCK_ANYTABLE');
+  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCKANYTABLE');
 run;
 %mp_assertdsobs(work.chk4c,
  desc=Test4 - derived value is correct,
@@ -294,7 +294,7 @@ run;
  ,dc_dttmtfmt=&dc_dttmtfmt
 )
 /* mp_abort cannot exit cleanly from inside a %include, so an abort there
-  (eg in a hook script run via mp_include) only records the error in
+  (eg in a hook script run via mp_include) only records the failure in
  work.mp_abort_errds.  mode=INCLUDE picks that up and aborts properly here. */
 %mp_abort(mode=INCLUDE)

@@ -5,13 +5,14 @@
    1. loading row level EDIT security against a table with LOADTYPE=REPLACE
    2. applying LOADTYPE=REPLACE to a table with row level EDIT security

-  The staged loads are aborted by the post edit hooks inside the child
-  service.  The abort is recorded as a FAILED load in MPE_LOADS (via
-  mpe_loadfail), so the assertions query MPE_LOADS rather than the child
-  response - a canceled child does not always return a webout or log.
+  The staged loads are aborted by the post edit hooks.  The assertion fetches
+  the sasjsAbort object that mp_abort writes to the webout before the hook
+  aborts.

  <h4> SAS Macros </h4>
  @li mx_execute.sas
+  @li mpe_execute.sas
+  @li mpe_assert_abort.sas
  @li mp_assert.sas


@@ -71,7 +72,6 @@ quit;
 /**
  * Test 1 - submitting an RLS EDIT rule against a REPLACE table should abort
  */
-%let t1=%sysfunc(datetime());
 data work.sascontroltable;
  action='LOAD';
  message='RLS EDIT rule on REPLACE table should abort';
@@ -100,36 +100,19 @@ data work.jsdata;
  output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web1,
  outref=wb1,
+  outstatus=work.ab1,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-/* a hook-aborted load is terminated before mpe_loadfail can record a
-  * FAILED status, so the load row stays IN PROGRESS - the abort is
-  * proved by the load NOT completing successfully.  Tests in this suite run
-  * in parallel sessions, so match the row for THIS test by its unique
-  * reason_txt (the submitted message) rather than any load in the time
-  * window - an unrelated aborted/IN PROGRESS load would otherwise satisfy
-  * the assertion even if this load completed.  The row id (csv_dir/mperef)
-  * is generated inside the aborted child service, which returns no webout,
-  * so reason_txt is the only stable identifier the test controls. */
-%let abort1=0;
-proc sql noprint;
-select count(*) into: abort1
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t1
-    and reason_txt="RLS EDIT rule on REPLACE table should abort"
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort1>0),
+%mpe_assert_abort(
  desc=Checking RLS EDIT rule is rejected for REPLACE loadtype table (#211),
-  outds=work.test_results
+  outref=wb1
 )

 /**
@@ -137,7 +120,6 @@ quit;
  * time: the rule is evaluated against the submitted data, where a derived
  * column does not exist
  */
-%let t3=%sysfunc(datetime());

 /* define a derived variable on the target table */
 proc sql noprint;
@@ -185,27 +167,19 @@ data work.jsdata;
  output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web3,
  outref=wb3,
+  outstatus=work.ab3,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-%let abort3=0;
-proc sql noprint;
-select count(*) into: abort3
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t3
-    and reason_txt="RLS rule on a derived variable should abort"
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort3>0),
+%mpe_assert_abort(
  desc=Checking an RLS rule on a derived variable is rejected,
-  outds=work.test_results
+  outref=wb3
 )

 /* tidy up the derived configuration */
@@ -219,7 +193,6 @@ quit;
  * should abort.  The active EDIT rule was inserted in the prep section,
  * long before this child service reads it.
  */
-%let t2=%sysfunc(datetime());

 /* now stage a REPLACE loadtype for DCTEST.WIDEBOY */
 data work.sascontroltable;
@@ -237,27 +210,19 @@ data work.jsdata;
  _____DELETE__THIS__RECORD_____='No';
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web2,
  outref=wb2,
+  outstatus=work.ab2,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-%let abort2=0;
-proc sql noprint;
-select count(*) into: abort2
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t2
-    and reason_txt="REPLACE loadtype on RLS-secured table should abort"
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort2>0),
+%mpe_assert_abort(
  desc=Checking REPLACE loadtype is rejected for RLS EDIT-secured table (#211),
-  outds=work.test_results
+  outref=wb2
 )

 /**
@@ -18,19 +18,12 @@
    6. an inactive row is set active again while another active row depends on
      it - the re-activation is what makes the dependency resolvable

-  And on the label:
-    8. a VAR_LABEL carrying a line break is rejected - a label is one line
-    9. a label of free text (quotes, and the ; & % the other columns refuse) is
-      accepted
-    10. a label over 256 characters is rejected rather than silently cut down
-
-  The staged loads are aborted by the post edit hook inside the child service.
-  An aborted load leaves its MPE_LOADS row IN PROGRESS, so each assertion
-  matches on the unique reason_txt (the submitted message) rather than any load
-  in the time window - the suite runs in parallel sessions.
+  The staged loads are aborted by the post edit hook. The assertion fetches the
+  sasjsAbort object that mp_abort writes to the webout before the hook aborts.

  <h4> SAS Macros </h4>
-  @li mx_execute.sas
+  @li mpe_execute.sas
+  @li mpe_assert_abort.sas
  @li mp_assert.sas


@@ -95,27 +88,19 @@ data work.jsdata;
  output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web1,
  outref=wb1,
+  outstatus=work.ab1,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-%let abort1=0;
-proc sql noprint;
-select count(*) into: abort1
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t1
-    and reason_txt='Derived var depending on an inactive row should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort1>0),
+%mpe_assert_abort(
  desc=Checking a dependency on an inactive derived row is rejected,
-  outds=work.test_results
+  outref=wb1
 )

 /**
@@ -123,7 +108,6 @@ quit;
  * rejected: the control table still shows the row as active, so the check has
  * to read the staged row too
  */
-%let t2=%sysfunc(datetime());

 proc sql noprint;
 delete from &dc_libref..mpe_derived
@@ -170,27 +154,19 @@ data work.jsdata;
  _____DELETE__THIS__RECORD_____='No';
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web2,
  outref=wb2,
+  outstatus=work.ab2,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-%let abort2=0;
-proc sql noprint;
-select count(*) into: abort2
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t2
-    and reason_txt='Deactivating a relied-on derived row should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort2>0),
+%mpe_assert_abort(
  desc=Checking deactivating a relied-on derived row is rejected,
-  outds=work.test_results
+  outref=wb2
 )

 /**
@@ -241,7 +217,7 @@ data work.jsdata;
  output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web3,
@@ -319,7 +295,7 @@ data work.jsdata;
  _____DELETE__THIS__RECORD_____='Yes';
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web4,
@@ -391,27 +367,19 @@ data work.jsdata;
  _____DELETE__THIS__RECORD_____='Yes';
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web5,
  outref=wb5,
+  outstatus=work.ab5,
+  failfast=0,
  mdebug=&sasjs_mdebug
 )

-%let abort5=0;
-proc sql noprint;
-select count(*) into: abort5
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t5
-    and reason_txt='Deleting a relied-on derived row should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort5>0),
+%mpe_assert_abort(
  desc=Checking deleting a relied-on derived row is rejected,
-  outds=work.test_results
+  outref=wb5
 )

 /**
@@ -481,7 +449,7 @@ data work.jsdata;
  output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web6,
@@ -563,7 +531,7 @@ data work.jsdata;
  _____DELETE__THIS__RECORD_____='No';
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputdatasets=work.sascontroltable work.jsdata,
  outlib=web7,
@@ -585,183 +553,6 @@ run;
 )


-/**
-  * Test 8 - a label carrying a line break must be rejected.  VAR_LABEL is free
-  * text and takes any character, including the quotes and triggers the other
-  * columns refuse, but a column label is one line: the generated attrib
-  * statement would carry the break into the DATA step that is included.
-  */
-%let t8=%sysfunc(datetime());
-
-proc sql noprint;
-delete from &dc_libref..mpe_derived
-  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
-quit;
-
-data work.sascontroltable;
-  action='LOAD';
-  message='A label with a line break should abort';
-  libds="&dc_libref..MPE_DERIVED";
-  output;
-  stop;
-run;
-
-data work.jsdata;
-  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
-    VAR_LOGIC $4000 VAR_FMT $32 VAR_LABEL $256
-    _____DELETE__THIS__RECORD_____ $3;
-  LIBREF='DCTEST';
-  DSN='DC_DERIV';
-  VAR_NM='LINEBREAK_LABEL';
-  VAR_DEPS='';
-  VAR_TYPE='N';
-  VAR_LEN=8;
-  VAR_FMT='';
-  VAR_LOGIC='1';
-  ACTIVE=1;
-  VAR_LABEL=cats('first line','0A'x,'second line');
-  _____DELETE__THIS__RECORD_____='No';
-run;
-
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputdatasets=work.sascontroltable work.jsdata,
-  outlib=web8,
-  outref=wb8,
-  mdebug=&sasjs_mdebug
-)
-
-%let abort8=0;
-proc sql noprint;
-select count(*) into: abort8
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t8
-    and reason_txt='A label with a line break should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort8>0),
-  desc=Checking a label with a line break is rejected,
-  outds=work.test_results
-)
-
-/**
-  * Test 9 - a label made of anything else is accepted: quotes, and the
-  * characters VAR_FMT and VAR_LOGIC refuse.  The label is free text, and the
-  * quotes are escaped when the attrib statement is built.
-  */
-proc sql noprint;
-delete from &dc_libref..mpe_derived
-  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
-quit;
-
-data work.sascontroltable;
-  action='LOAD';
-  message='Free text in a label should not abort';
-  libds="&dc_libref..MPE_DERIVED";
-  output;
-  stop;
-run;
-
-data work.jsdata;
-  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
-    VAR_LOGIC $4000 VAR_FMT $32 VAR_LABEL $256
-    _____DELETE__THIS__RECORD_____ $3;
-  LIBREF='DCTEST';
-  DSN='DC_DERIV';
-  VAR_NM='FREE_TEXT_LABEL';
-  VAR_DEPS='';
-  VAR_TYPE='N';
-  VAR_LEN=8;
-  VAR_FMT='';
-  VAR_LOGIC='1';
-  ACTIVE=1;
-  VAR_LABEL="it's 100% & more; really";
-  _____DELETE__THIS__RECORD_____='No';
-run;
-
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputdatasets=work.sascontroltable work.jsdata,
-  outlib=web9,
-  outref=wb9,
-  mdebug=&sasjs_mdebug
-)
-
-%let status9=0;
-data _null_;
-  set web9.sasparams;
-  putlog (_all_)(=);
-  if status='SUCCESS' then call symputx('status9',1);
-run;
-
-%mp_assert(
-  iftrue=(&status9=1),
-  desc=Checking free text in a label is accepted,
-  outds=work.test_results
-)
-
-/**
-  * Test 10 - a label longer than 256 is rejected rather than cut down: the
-  * engine truncates the label to 256 when it builds the attrib statement, and
-  * nothing reports it.
-  */
-%let t10=%sysfunc(datetime());
-
-proc sql noprint;
-delete from &dc_libref..mpe_derived
-  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
-quit;
-
-data work.sascontroltable;
-  action='LOAD';
-  message='A label over 256 characters should abort';
-  libds="&dc_libref..MPE_DERIVED";
-  output;
-  stop;
-run;
-
-data work.jsdata;
-  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
-    VAR_LOGIC $4000 VAR_FMT $32 VAR_LABEL $256
-    _____DELETE__THIS__RECORD_____ $3;
-  LIBREF='DCTEST';
-  DSN='DC_DERIV';
-  VAR_NM='LONG_LABEL';
-  VAR_DEPS='';
-  VAR_TYPE='N';
-  VAR_LEN=8;
-  VAR_FMT='';
-  VAR_LOGIC='1';
-  ACTIVE=1;
-  VAR_LABEL=repeat('x',256);
-  _____DELETE__THIS__RECORD_____='No';
-run;
-
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputdatasets=work.sascontroltable work.jsdata,
-  outlib=web10,
-  outref=wb10,
-  mdebug=&sasjs_mdebug
-)
-
-%let abort10=0;
-proc sql noprint;
-select count(*) into: abort10
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t10
-    and reason_txt='A label over 256 characters should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort10>0),
-  desc=Checking a label over 256 characters is rejected,
-  outds=work.test_results
-)
-
 /* tidy up so repeated runs start clean */
 proc sql noprint;
 delete from &dc_libref..mpe_derived
@@ -34,6 +34,7 @@
  <h4> SAS Macros </h4>
  @li mf_nobs.sas
  @li mp_abort.sas
+  @li mpe_derived_check.sas

  <h4> Related Macros </h4>
  @li mpe_derived.sas
@@ -101,53 +102,7 @@ run;
 /**
  * Field level checks - the same rules %mpe_derived applies at read time
  */
-data work.derived_check;
-  set work.staging_ds;
-  length reason $200;
-  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
-  call missing(reason);
-
-  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
-  else if var_nm='' then reason='VAR_NM is required';
-  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
-    then reason='VAR_NM must be a valid SAS name';
-  else if verify(substr(var_nm,1,1),'0123456789')=0
-    then reason='VAR_NM must not start with a digit';
-  else if var_nm='_____DELETE__THIS__RECORD_____'
-    then reason='VAR_NM is reserved';
-  else if var_type not in ('C','N')
-    then reason='VAR_TYPE must be C or N';
-  else if missing(var_len) or var_len ne int(var_len) or var_len<1
-    then reason='VAR_LEN must be a positive integer';
-  else if var_type='C' and var_len>32767
-    then reason='VAR_LEN exceeds 32767 for a character variable';
-  else if var_type='N' and var_len>8
-    then reason='VAR_LEN exceeds 8 for a numeric variable';
-  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
-    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
-    then reason='VAR_FMT must be a format name, without ; & % or /* */';
-  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
-    then reason='VAR_FMT must be a format name';
-  /* VAR_LABEL is free text: any character is allowed, in any script, and the
-    quotes in it are escaped by %mp_derivedvars when the attrib statement is
-    built.  What is refused is anything that cannot be a single-line column
-    label - a control character or a line break - and a value long enough to
-    be cut down silently, since the engine truncates the label to 256. */
-  else if anycntrl(var_label)>0
-    then reason='VAR_LABEL must be plain text - no line breaks or control characters';
-  else if klength(var_label)>256
-    then reason='VAR_LABEL exceeds 256 characters';
-  else if missing(var_logic)
-    then reason='VAR_LOGIC is required';
-  else if index(var_logic,';')>0
-    then reason='VAR_LOGIC must be a single expression - remove the ;';
-  else if indexc(var_logic,'&%')>0
-    then reason='VAR_LOGIC must not contain macro triggers';
-  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
-    then reason='VAR_LOGIC must not contain comments';
-
-  if not missing(reason) then output;
-run;
+%mpe_derived_check(inds=work.staging_ds, outds=work.derived_check)

 %if %mf_nobs(work.derived_check)>0 %then %do;
  data _null_;
@@ -12,9 +12,9 @@
  assertion cannot pass against a service that does not validate.

  <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
  @li mp_assert.sas
-  @li mx_execute.sas
-  @li mf_getplatform.sas
+  @li mpe_execute.sas
  @li mf_getuniquefileref.sas

 **/
@@ -31,37 +31,20 @@ data _null_;
  put '%sysfunc(coalescec(&dclib..MPE_X_TEST,))';
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputfiles=&f1:iwant,
-    outref=web1,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort1=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort1',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort1=1),
-    desc=Macro content in libds aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=Macro content in libds aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=Macro content in libds aborts the service,
+  outref=web1,
+  msg=Invalid libds
+)

 /**
  * Test 2 - valid libds still returns columns
@@ -73,7 +56,7 @@ data _null_;
  put "&dclib..MPE_X_TEST";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f2:iwant,
  outlib=web2
@@ -36,7 +36,8 @@
  <h4> SAS Macros </h4>
  @li mf_existds.sas
  @li mf_nobs.sas
-  @li mf_verifymacvars.sas
+  @li mpeinit.sas
+  @li mpeterm.sas
  @li dc_assignlib.sas
  @li mf_getvarformat.sas
  @li mp_abort.sas
@@ -111,7 +112,10 @@ run;
 %let startrow=1;
 %let rows=4000;

-%mp_abort(iftrue= (%mf_verifymacvars(libds col2)=0)
+/* a request with no rows never reached the validation below, so this is the
+  only case that is missing inputs - an invalid value is a different abort, and
+  saying "missing" for it would hide the reason */
+%mp_abort(iftrue= (%mf_nobs(work.iwant)<=0)
  ,mac=&_program..sas
  ,msg=%str(Missing inputs from iwant)
 )
@@ -13,9 +13,9 @@
  validate.

  <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
  @li mp_assert.sas
-  @li mx_execute.sas
-  @li mf_getplatform.sas
+  @li mpe_execute.sas
  @li mf_getuniquefileref.sas

 **/
@@ -28,41 +28,24 @@
 %let f1=%mf_getuniquefileref();
 data _null_;
  file &f1 termstr=crlf;
-  put 'LIBDS:$19. COL:$9.';
+  put 'LIBDS:$100. COL:$100.';
  put '%sysfunc(coalescec(&dclib..MPE_X_TEST,)),SOME_TIME';
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputfiles=&f1:iwant,
-    outref=web1,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort1=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort1',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort1=1),
-    desc=Macro content in libds aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=Macro content in libds aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=Macro content in libds aborts the service,
+  outref=web1,
+  msg=Invalid libds
+)

 /**
  * Test 2 - macro content in col must abort the service
@@ -70,41 +53,24 @@ run;
 %let f2=%mf_getuniquefileref();
 data _null_;
  file &f2 termstr=crlf;
-  put 'LIBDS:$19. COL:$9.';
-  put '&dclib..MPE_X_TEST,%sysfunc(coalescec(SOME_TIME,))';
+  put 'LIBDS:$100. COL:$100.';
+  put "&dclib..MPE_X_TEST,%nrstr(%sysfunc(coalescec(SOME_TIME,)))";
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputfiles=&f2:iwant,
-    outref=web2,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:iwant,
+  outref=web2,
+  outstatus=work.ab2,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort2=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort2',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort2=1),
-    desc=Macro content in col aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=Macro content in col aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=Macro content in col aborts the service,
+  outref=web2,
+  msg=Invalid col
+)

 /**
  * Test 3 - valid inputs still return values
@@ -112,11 +78,11 @@ run;
 %let f3=%mf_getuniquefileref();
 data _null_;
  file &f3 termstr=crlf;
-  put 'LIBDS:$19. COL:$9.';
+  put 'LIBDS:$100. COL:$100.';
  put "&dclib..MPE_X_TEST,SOME_TIME";
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f3:iwant,
  outlib=web3
@@ -15,7 +15,7 @@
 data work.params;
  length name $32 value $1000;
  name='libref';value="&dclib";output;
-  name='ds';value='';output;
+  name='ds';value='MPE_TABLES';output;
  name='flavour';value='TSQL';output;
 run;
 %mx_execute(&_program,
@@ -31,7 +31,7 @@ data work.results;
  input;
  putlog _infile_;
  if _n_=2 then do;
-    if _infile_=:'/* TSQL Flavour DDL for' then do;
+    if _infile_=:'/* TSQL Flavour DDL for' and index(_infile_,'MPE_TABLES')>0 then do;
      putlog 'test passed';
      output;
      output;
@@ -42,7 +42,7 @@ data work.results;
 run;

 %mp_assertdsobs(work.results,
-  desc=DDL file is successfully returned,
+  desc=DDL file is successfully returned for a table,
  test=EQUALS 2,
  outds=work.test_results
 )
@@ -15,10 +15,10 @@
  @li mf_existfeature.sas
  @li dc_assignlib.sas
  @li mp_abort.sas
-  @li mp_binarycopy.sas
  @li mp_cntlout.sas
  @li mp_ds2cards.sas
  @li mp_ds2csv.sas
+  @li mp_ds2md.sas
  @li mp_streamfile.sas
  @li mp_validatecol.sas
  @li mf_nobs.sas
@@ -13,9 +13,9 @@
  validate.

  <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
  @li mp_assert.sas
-  @li mx_execute.sas
-  @li mf_getplatform.sas
+  @li mpe_execute.sas

 **/

@@ -32,37 +32,20 @@ data work.params1;
  name='filter';value='0';output;
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputparams=work.params1,
-    outref=web1,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params1,
+  outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort1=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort1',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort1=1),
-    desc=Macro content in table aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=Macro content in table aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=Macro content in table aborts the service,
+  outref=web1,
+  msg=Invalid table
+)

 /**
  * Test 2 - sql injection in filter must abort the service
@@ -74,37 +57,20 @@ data work.params2;
  name='filter';value='0 or 1=1';output;
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputparams=work.params2,
-    outref=web2,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params2,
+  outref=web2,
+  outstatus=work.ab2,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort2=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort2',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort2=1),
-    desc=SQL injection in filter aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=SQL injection in filter aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=SQL injection in filter aborts the service,
+  outref=web2,
+  msg=Invalid filter
+)

 /**
  * Test 3 - path traversal in table must abort the service
@@ -118,37 +84,20 @@ data work.params3;
  name='filter';value='0';output;
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputparams=work.params3,
-    outref=web3,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputparams=work.params3,
+  outref=web3,
+  outstatus=work.ab3,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort3=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort3',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort3=1),
-    desc=Path traversal in table aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=Path traversal in table aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=Path traversal in table aborts the service,
+  outref=web3,
+  msg=Invalid table
+)

 /**
  * Test 4 - the valid request must still work (positive control)
@@ -160,7 +109,7 @@ data work.params4;
  name='filter';value='0';output;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputparams=work.params4,
  outref=web4,
@@ -78,13 +78,33 @@ run;
  outview=work.vw_derived,
  outvars=work.derived_vars
 )
-%let derived_nobs=%mf_nobs(work.derived_vars);
-%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&ds));
+/* A format catalog (LIBREF.CATALOGNAME-FC) must be left for mp_filterstore to
+  rewrite into a cntlout dataset, and a -FC name handed over as targetds is not
+  something it can read - so pass no targetds at all in that case.  An empty
+  targetds makes the engine fall back to libds, which is what a plain dataset
+  wants anyway.
+
+  No macro triggers in this comment: the macro processor resolves them even
+  inside a comment, so prose about conditions is itself executed and the parser
+  reports "Expected DO not found" from here.  Same reason the values below are
+  plain LET statements with no trailing comma. */
+%let derived_nobs=%sysfunc(coalesce(%mf_nobs(work.derived_vars),0));
+%let is_fmtcat=%eval(%length(&ds)>3 and "%upcase(%substr(&ds,%length(&ds)-2,3))"="-FC");
+%let targetds_val=&ds;
+%if &derived_nobs>0 %then %do;
+  %let targetds_val=work.vw_derived;
+%end;
+/* the derived view keeps precedence: a catalog is only blanked when there is no
+  derived configuration to validate against.  Two plain %if blocks rather than
+  %else %if - that construct is what the parser rejected here before. */
+%if &derived_nobs=0 and &is_fmtcat=1 %then %do;
+  %let targetds_val=;
+%end;

 %mp_filterstore(
-  libds=&ds,
-  targetds=&filterds,
-  queryds=work.filterquery,
+  libds=&ds
+  ,targetds=&targetds_val
+  ,queryds=work.filterquery,
  filter_summary=&dc_libref..mpe_filteranytable,
  filter_detail=&dc_libref..mpe_filtersource,
  lock_table=&dc_libref..mpe_lockanytable,
@@ -12,9 +12,9 @@
  assertion cannot pass against a service that does not validate.

  <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
  @li mp_assert.sas
-  @li mx_execute.sas
-  @li mf_getplatform.sas
+  @li mpe_execute.sas
  @li mf_getuniquefileref.sas

 **/
@@ -31,37 +31,20 @@ data _null_;
  put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
 run;

-/* Platform gate: the abort is asserted from the child job state
-  (work.results, written by mv_jobflow), which only the Viya branch of
-  %mx_execute produces.  On the other platforms the state cannot be read,
-  so the case is recorded as a skip rather than a failure. */
-%if %mf_getplatform()=SASVIYA %then %do;
-  %mx_execute(&_program,
-    viyacontext=&defaultcontext,
-    inputfiles=&f1:iwant,
-    outref=web1,
-    viyaresult=WEBOUT_TXT
-  )
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)

-  %let abort1=0;
-  data _null_;
-    set work.results;
-    if state='canceled' then call symputx('abort1',1);
-  run;
-
-  %mp_assert(
-    iftrue=(&abort1=1),
-    desc=Macro content in filter_table aborts the service,
-    outds=work.test_results
-  )
-%end;
-%else %do;
-  %mp_assert(
-    iftrue=(1=1),
-    desc=Macro content in filter_table aborts the service - skipped (the job state it reads is only written on Viya),
-    outds=work.test_results
-  )
-%end;
+%mpe_assert_abort(
+  desc=Macro content in filter_table aborts the service,
+  outref=web1,
+  msg=Invalid filter_table
+)

 /**
  * Test 2 - valid filter_table still stores a filter
@@ -85,7 +68,7 @@ AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
 ;;;;
 run;

-%mx_execute(&_program,
+%mpe_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f2:iwant &f3:filterquery,
  outlib=web2
@@ -5,7 +5,6 @@
  <h4> SAS Macros </h4>
  @li mx_execute.sas
  @li mp_assertdsobs.sas
-  @li mp_assertcolvals.sas
  @li mf_getuniquefileref.sas

 **/
@@ -34,8 +33,15 @@ run;
 %mx_execute(&_program,
  viyacontext=&defaultcontext,
  inputfiles=&f1:iwant &f2:filterquery,
-  outlib=web1
+  outlib=web1,
+  outref=web1ref,
+  mdebug=&sasjs_mdebug
 )
+data _null_;
+  infile web1ref;
+  input;
+  putlog _infile_;
+run;
 data result;
  set web1.result;
 run;
@@ -64,7 +70,8 @@ run;
  viyacontext=&defaultcontext,
  inputdatasets=work.iwant work.filterquery,
  outlib=web2,
-  outref=web2ref
+  outref=web2ref,
+  mdebug=&sasjs_mdebug
 )
 data _null_;
  infile web2ref;
@@ -119,7 +126,8 @@ run;
  viyacontext=&defaultcontext,
  inputdatasets=work.iwant work.filterquery,
  outlib=web3,
-  outref=web3ref
+  outref=web3ref,
+  mdebug=&sasjs_mdebug
 )
 data _null_;
  infile web3ref;
@@ -55,7 +55,6 @@
  @li mp_cntlout.sas
  @li mp_getcols.sas
  @li mp_getpk.sas
-  @li mp_jsonout.sas
  @li mp_searchdata.sas
  @li mp_validatecol.sas
  @li mpe_columnlevelsecurity.sas
@@ -57,7 +57,7 @@ data _null_;
 run;
 %mp_assert(
  iftrue=(&nobs=&expected),
-  desc=&desc (&searchtype "%superq(searchval)" returned &nobs rows, expected &expected),
+  desc=%str(&desc (&searchtype "%superq(searchval)" returned &nobs rows, expected &expected)),
  outds=work.test_results
 )
 %mend search;
@@ -118,6 +118,6 @@ proc sql noprint;
 quit;
 %mp_assert(
  iftrue=(&caprows=&capnobs and &capnobs gt 0 and &capnobs le &capmax),
-  desc=Checking a search beyond the cap returns as many rows as it reports (&capnobs reported, &caprows returned, cap &capmax),
+  desc=%str(Checking a search beyond the cap returns as many rows as it reports (&capnobs reported, &caprows returned, cap &capmax)),
  outds=work.test_results
 )
@@ -44,4 +44,12 @@ run;

 options metarepository=&repo;

+%if "&sasjs_mdebug"="1" %then %do;
+  data _null_;
+    set &outds;
+    putlog "NOTE: &sysmacroname: &outds:";
+    putlog (_all_)(=);
+  run;
+%end;
+
 %mend dc_getlibs;
@@ -20,7 +20,15 @@ create table &outds as
    ,engine
    ,'' as libraryid length=17
  from dictionary.libnames
-  where libname not in ('WORK','SASUSER');
+  where upcase(libname) not in ("WORK","SASUSER","%upcase(&dc_libref)");
 insert into &syslast values ("&DC_LIBREF", "&DC_LIBNAME",'','V9');

+%if "&sasjs_mdebug"="1" %then %do;
+  data _null_;
+    set &outds;
+    putlog "NOTE: &sysmacroname: &outds:";
+    putlog (_all_)(=);
+  run;
+%end;
+
 %mend dc_getlibs;
@@ -6,6 +6,11 @@

  The MPELIB should be pre-assigned

+  The refresh macros are called from the statements this macro writes into a
+  catalog, so they are not visible as calls to lint.
+
+  @sasjslint {"noUnusedMacros": false}
+
  <h4> SAS Macros </h4>
  @li mpe_refreshlibs.sas
  @li dc_assignlib.sas
@@ -32,12 +37,17 @@ data libraries;
  where also libref="&libref";
 %end;
  file executor;
-  str=cats('%mpe_refreshtables(',libref,')');
+  str=cats('%mpe_refreshtables(',libref,');');
  put str;
  putlog str;
-  str=cats('%mpe_refreshcatalogs(',libref,')');
+  str=cats('%mpe_refreshcatalogs(',libref,');');
  put str;
 run;
-%inc executor/source2;
+%if "&sasjs_mdebug"="1" %then %do;
+  /* echo the generated statements, so a failure inside one of them can be
+    traced back to the libref that produced it */
+  %inc executor/source2;
+%end;
+%else %inc executor;

 %mend dc_refreshcatalog;
@@ -6,8 +6,12 @@
    It also adds the STAGING directory as subdirectory to the BASE
    library location.

-  @warning This STP self destructs! It will delete itself after a successful run
-    to avoid being executed twice (and overwriting actual data)
+  @warning This STP self destructs! It will delete itself (and the
+    configurator) after a successful run, to avoid being executed twice and
+    overwriting actual data.  Pass a KEEPME column in the input data to keep
+    them - test harnesses reseed before every run and cannot redeploy first.
+
+  @sasjslint {"noBannedStrings": false}

  <h4> SAS Macros </h4>
  @li mf_getapploc.sas
@@ -55,6 +59,24 @@ data _null_;
 run;
 %end;

+/* optional KEEPME column - keep this program and the configurator on the drive
+  * after a successful run.  A normal install self-destructs so the seeding
+  * cannot be run twice over live data; a test harness reseeds before every run
+  * and wants them to stay put. */
+%let keepme=0;
+%if &exist=1 %then %do;
+data _null_;
+  dsid=open('work.fromjs');
+  /* every row, not just the first: KEEPME may be set on any of them */
+  if varnum(dsid,'KEEPME')>0 then do until(eof);
+    set work.fromjs end=eof;
+    if upcase(cats(keepme)) in ('1','Y','YES','TRUE') then call symputx('keepme',1);
+  end;
+  rc=close(dsid);
+  stop;
+run;
+%end;
+
 %mp_abort(iftrue= (&syscc ne 0)
  ,mac=&_program
  ,msg=%str(Issue on makedata entry)
@@ -191,7 +213,13 @@ data _null_;
 run;
 %end;

-/* We ran successfully, now remove configurator and makedata STPs */
-%ms_deletefile(&root/services/admin/configurator.sas)
-%ms_deletefile(&_program..sas)
+/* We ran successfully, now remove configurator and makedata STPs, unless the
+  * caller asked to keep them (KEEPME in the input data).  Keeping them is for
+  * test harnesses, which reseed before every run and cannot redeploy first. */
+%if &keepme ne 1 %then %do;
+  %ms_deletefile(&root/services/admin/configurator.sas)
+  %ms_deletefile(&_program..sas)
+%end;
+%else %put NOTE: &_program retained, as KEEPME was requested;
+

@@ -21,7 +21,15 @@ create table &outds as
    ,engine
    ,'' as libraryid length=17
  from dictionary.libnames
-  where libname not in ('WORK','SASUSER','CASUSER');
+  where upcase(libname) not in ("WORK","SASUSER","CASUSER","%upcase(&dc_libref)");
 insert into &syslast values ("&DC_LIBREF", "&DC_LIBNAME",'','V9');

+%if "&sasjs_mdebug"="1" %then %do;
+  data _null_;
+    set &outds;
+    putlog "NOTE: &sysmacroname: &outds:";
+    putlog (_all_)(=);
+  run;
+%end;
+
 %mend dc_getlibs;
@@ -6,6 +6,11 @@

  The MPELIB should be pre-assigned

+  The refresh macros are called from the statements this macro writes into a
+  catalog, so they are not visible as calls to lint.
+
+  @sasjslint {"noUnusedMacros": false}
+
  <h4> SAS Macros </h4>
  @li mpe_refreshlibs.sas
  @li dc_assignlib.sas
@@ -36,13 +41,18 @@
    set &mpelib..mpe_datacatalog_libs;
    where &dc_dttmtfmt. le TX_TO;
    file executor;
-    str=cats('%mpe_refreshtables(',libref,')');
+    str=cats('%mpe_refreshtables(',libref,');');
    put str;
-    str=cats('%mpe_refreshcatalogs(',libref,')');
+    str=cats('%mpe_refreshcatalogs(',libref,');');
    put str;
    putlog str;
  run;
-  %inc executor;
+  %if "&sasjs_mdebug"="1" %then %do;
+    /* echo the generated statements, so a failure inside one of them can be
+      traced back to the libref that produced it */
+    %inc executor/source2;
+  %end;
+  %else %inc executor;
 %end;

 %mend dc_refreshcatalog;
@@ -9,7 +9,6 @@
  @li mf_mkdir.sas
  @li mf_trimstr.sas
  @li mp_abort.sas
-  @li mpe_getvars.sas
  @li mpe_makedata.sas
  @li mpe_makedatamodel.sas
  @li mpe_makesampledata.sas
@@ -33,6 +32,24 @@
 %let exist=%sysfunc(exist(work.fromjs));
 %let inds=%sysfunc(ifc(&exist=1,fromjs,_null_));

+/* optional KEEPME column - keep this program on the content server after a
+  * successful run.  A normal install self-destructs so the seeding cannot be
+  * run twice over live data; a test harness reseeds once and then re-runs
+  * individual tests, so it wants the program to stay put. */
+%let keepme=0;
+%if &exist=1 %then %do;
+data _null_;
+  dsid=open('work.fromjs');
+  /* every row, not just the first: KEEPME may be set on any of them */
+  if varnum(dsid,'KEEPME')>0 then do until(eof);
+    set work.fromjs end=eof;
+    if upcase(cats(keepme)) in ('1','Y','YES','TRUE') then call symputx('keepme',1);
+  end;
+  rc=close(dsid);
+  stop;
+run;
+%end;
+
 data _null_;
  set &inds;
  call symputx('dcpath',dcpath);
@@ -278,4 +295,9 @@ run;
 %webout(OBJ,result)
 %webout(CLOSE)

-%mv_deletejes(path=&root/admin, name=makedata)
+/* keep the program when KEEPME was requested, so a reseed does not need a
+  * redeploy first */
+%if &keepme ne 1 %then %do;
+  %mv_deletejes(path=&root/admin, name=makedata)
+%end;
+%else %put NOTE: &_program retained, as KEEPME was requested;
@@ -17,7 +17,6 @@
  @li mddl_dc_difftable.sas
  @li mddl_dc_locktable.sas
  @li mp_init.sas
-  @li mx_execute.sas
  @li ms_createfile.sas
  @li ms_deletefile.sas
  @li mx_getcode.sas
@@ -55,6 +54,8 @@ run;
 %dc_getsettings()


+
+
 /**
  * Set up Test library
  */