fix(editor): assign columnHeader before the grid renders

The derived-block marker (dc-derived-first) landed on the wrong column, so the
computed columns did not read as a group and derived-variables.cy.ts failed on
the header assertion.

initSetup reversed the order it needs: hot.updateSettings() installs
afterGetColHeader / afterGetCellMeta at line ~4167 and renders, but
this.columnHeader was only assigned at line ~4506 - after that render. The
hooks therefore resolved visual column indexes against the previous table's
list, so 'is this the first derived column' compared an index in one array
against an index in another. The header name check still matched, which is why
dc-derived-header appeared while its sibling class did not.

columnHeader is now taken from headerColumns once, where that list is final
(the Delete? rename and the EDIT_STATUS append have been applied), and the
later divergent copy is gone - it never carried EDIT_STATUS, so it could not
have matched the grid anyway. Comments at both sites record why the order
matters.

.gitea/workflows/build.yaml

@@ -142,6 +142,7 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          echo HTTP_PORT=5000 >> .env
           # The server refuses to run as root unless RUN_AS names an account
           # (a security change in the recent releases). This job runs as root
           # in a throwaway container, so opt in explicitly.
@@ -181,7 +182,7 @@ jobs:
           sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true
 
       - name: Prepare and run frontend and cypress
-        timeout-minutes: 35
+        timeout-minutes: 50
         run: |
           cd ./client
           mv ./cypress.env.example.json ./cypress.env.json
@@ -196,9 +197,9 @@ jobs:
           replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
           cat ./cypress.config.ts
           # Start frontend and run cypress
-          # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
+          # timeout 2700: SIGTERM after 45 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 2700 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

.gitea/workflows/lighthouse.yaml

@@ -33,6 +33,7 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          echo HTTP_PORT=5000 >> .env
           # The server refuses to run as root unless RUN_AS names an account
           # (a security change in the recent releases). This job runs as root
           # in a throwaway container, so opt in explicitly.

.gitea/workflows/release.yaml

@@ -117,6 +117,7 @@ jobs:
           echo NODE_PATH=node >> .env
           echo CORS=enable >> .env
           echo WHITELIST=http://localhost:4200 >> .env
+          echo HTTP_PORT=5000 >> .env
           # The server refuses to run as root unless RUN_AS names an account
           # (a security change in the recent releases). This job runs as root
           # in a throwaway container, so opt in explicitly.
@@ -193,11 +194,15 @@ jobs:
     # and its permissions are derived from this block.
     #   contents: write -> push CHANGELOG.md / package.json commit + tag, and
     #                      create the Release via the Gitea API.
+    #   actions: write  -> dispatch the Sites workflow for the new tag (that
+    #                      route is scoped to the actions unit, which contents
+    #                      does not grant).
     # The TSDOC_TOKEN and CODE_DATACONTROLLER_IO secrets continue to be used
     # because they target *different* servers (webdoc.datacontroller.io and
     # code.datacontroller.io), not the Gitea API.
     permissions:
       contents: write
+      actions: write
 
     steps:
       - uses: actions/checkout@v4
@@ -401,3 +406,27 @@ jobs:
           done
           echo "Uploading SHA256SUMS ..."
           curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$URL" -F "attachment=@SHA256SUMS"
+
+      - name: Publish the demo site for this release
+        # The tag is created by this job's semantic-release step, so it carries
+        # the Actions user as its author - and the forge suppresses ref events
+        # from that user, which means `on: push: tags` in sites.yaml never fires.
+        # Dispatch it instead: the API call is authenticated, and the run it
+        # starts checks out the tag the same way.
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          set -euo pipefail
+          AUTH_HEADER="Authorization: token ${GITEA_TOKEN}"
+          BASE="${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}"
+
+          TAG=$(curl -k --fail-with-body -sS -H "$AUTH_HEADER" "$BASE/releases/latest" | jq -r '.tag_name')
+          echo "Dispatching Sites for ${TAG}"
+
+          # the dispatch endpoint resolves a bare name as a branch; a tag needs
+          # the refs/tags/ prefix
+          curl -k --fail-with-body -sS -X POST \
+            -H "$AUTH_HEADER" \
+            -H 'Content-Type: application/json' \
+            --data "$(jq -n --arg ref "refs/tags/$TAG" '{ref: $ref}')" \
+            "$BASE/actions/workflows/sites.yaml/dispatches"

.gitea/workflows/sites.yaml

@@ -113,6 +113,39 @@ jobs:
 
       - name: Build the browser-hosted site
         run: |
+          # The published sites ship a working licence so they are usable beyond
+          # the free tier. The pair is committed in sas/mocks/mock-licence.json
+          # rather than minted here, so every deploy of both sites carries the
+          # same key; the CI estate stays unlicensed because that file is outside
+          # the deployed sync folder, so the specs still exercise the free tier.
+          # The pair must round trip through the same WebCrypto the app uses,
+          # because a committed pair can drift (one half regenerated, a value
+          # truncated on copy) and nothing else would notice before publishing.
+          # Identical values in the two fields is the HTTP-format marker the app
+          # refuses in a secure context, so refuse to publish one.
+          node -e "
+            const d = require('./sas/mocks/mock-licence.json')
+            const b = (s) => Buffer.from(String(s || ''), 'base64')
+            if (!d.licenceKey || !d.activationKey || d.licenceKey === d.activationKey) {
+              console.error('sas/mocks/mock-licence.json must hold an HTTPS-format key pair')
+              process.exit(1)
+            }
+            crypto.subtle
+              .importKey('pkcs8', b(d.activationKey), { name: 'RSA-OAEP', hash: 'SHA-256' }, false, ['decrypt'])
+              .then((key) => crypto.subtle.decrypt({ name: 'RSA-OAEP', hash: 'SHA-256' }, key, b(d.licenceKey)))
+              .then((plain) => {
+                const data = JSON.parse(new TextDecoder().decode(plain))
+                console.log('licence pair round trips: site', data.site_id, 'until', data.valid_until)
+              })
+              .catch((e) => {
+                console.error('sas/mocks/mock-licence.json does not decrypt:', e.message || e)
+                process.exit(1)
+              })
+          "
+          LICENCE_DIR="sasjs_root/drive/files$APP_LOC/mock-storage"
+          mkdir -p "$LICENCE_DIR"
+          cp sas/mocks/mock-licence.json "$LICENCE_DIR/licence.json"
+
           # DC's data files exist only after makedata has run, so the drive is
           # captured first and shipped as the site's starting drive. services/web9
           # is the streamed frontend, which is already the site itself.

.gitleaksignore

@@ -0,0 +1,5 @@
+# sas/mocks/mock-licence.json holds a licence key pair for the mock estate: the
+# activation key is RSA key material the browser demo decrypts with, not a
+# credential for any service.
+sas/mocks/mock-licence.json:generic-api-key:2
+sas/mocks/mock-licence.json:generic-api-key:3

CONTRIBUTING.md

@@ -86,6 +86,7 @@ RUN_TIMES=js
 NODE_PATH=node
 CORS=enable
 WHITELIST=http://localhost:4200
+HTTP_PORT=5000
 ```
 
 `RUN_TIMES=js` means the server executes `.js` files as Stored Programs (no SAS binary needed). `CORS=enable` plus `WHITELIST` allows the Angular dev server on port 4200 to call the mock services on port 5000. Start the server:
@@ -94,7 +95,7 @@ WHITELIST=http://localhost:4200
 ./api-linux    # or api-macos / api-win.exe depending on your platform
 ```
 
-The server runs in desktop mode (no auth) on `http://localhost:5000` by default.
+`HTTP_PORT` is not optional in SASjs Server 3.x: `PORT` and `PROTOCOL` are deprecated, and a server started without `HTTP_PORT` or `HTTPS_PORT` exits with "no listener is enabled". With it set, the server runs in desktop mode (no auth) on `http://localhost:5000`.
 
 ### 2. Deploy the mock services and streaming frontend
 
@@ -135,7 +136,7 @@ After changing mock services, re-run `sasjs cbd -t server-ci` from `sas/mocks/sa
 
 ### Demo and staging sites
 
-A push to `release/**` publishes [staging.datacontroller.io](https://staging.datacontroller.io); a release tag (`v*`, created by the release job after its tests pass) publishes [demo.datacontroller.io](https://demo.datacontroller.io) - workflow file `.gitea/workflows/sites.yaml`, using the `DEMO_DATACONTROLLER_IO` Surfer app password (user `allan`). Both are sites on one multi-site Surfer instance; the demo is its default site. The job builds the mocks as a browser-hosted site (SASjs CLI >= 6.1, `streamConfig.mockWeb`), so the services run in a service worker and the drive lives in the visitor's IndexedDB - there is no server. It runs `makedata` on a throwaway SASjs Server first and ships that drive as `sasjsbuild/mock-snapshot.json`; without it the site opens with no data. The mock ships no licence key, so both sites run on the free tier.
+A push to `release/**` publishes [staging.datacontroller.io](https://staging.datacontroller.io); a release tag (`v*`, created by the release job after its tests pass) publishes [demo.datacontroller.io](https://demo.datacontroller.io) - workflow file `.gitea/workflows/sites.yaml`, using the `DEMO_DATACONTROLLER_IO` Surfer app password (user `allan`). Both are sites on one multi-site Surfer instance; the demo is its default site. The job builds the mocks as a browser-hosted site (SASjs CLI >= 6.1, `streamConfig.mockWeb`), so the services run in a service worker and the drive lives in the visitor's IndexedDB - there is no server. It runs `makedata` on a throwaway SASjs Server first and ships that drive as `sasjsbuild/mock-snapshot.json`; without it the site opens with no data. The estate carries no licence key, so CI runs on the free tier until a key is registered through the licensing screen (`admin/registerkey`) - the Cypress setup does that for the specs that need one. The Sites job writes a licence into the published drive, so demo and staging are fully usable. The pair lives in `sas/mocks/mock-licence.json`: the two values are one RSA key and are regenerated together, and the `.gitleaksignore` entries for it are line-scoped, so inserting a line above the pair re-fires the scanner rather than revealing a new secret.
 
 ## GUI Elements
 

client/cypress/clips/special-missings-clip.cy.ts

@@ -40,8 +40,8 @@ context('special missings clip (DEMO_01)', function () {
   this.beforeEach(() => {
     cy.visit(hostUrl + appLocation)
 
-    // The mock estate carries a valid licence key (mock-storage/licence.json),
-    // so the app activates directly with no free-tier banner in shot.
+    // Assumes the estate is already activated (the licensing spec registers a
+    // valid key); otherwise a licensing page would sit in shot.
     cy.get('.nav-tree', { timeout: longerCommandTimeout }).should('exist')
   })
 

client/cypress/e2e/derived-variables.cy.ts

@@ -75,6 +75,55 @@ context('derived variables: ', function () {
 
     cy.screenshot('derived-3-filter-value', { capture: 'viewport' })
   })
+
+  it('4 | a derived column is marked read-only, and opens no editor', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    // a derived column is read-only AND computed, so the grid marks it apart
+    // from an ordinary read-only column - its own header/cell class, no
+    // readonlyCell dimming (editor.component.ts, afterGetColHeader /
+    // afterGetCellMeta). STATE is the control: same grid, a physical column.
+    headerCell('REGION').should('have.class', 'dc-derived-header')
+    headerCell('REGION').should('not.have.class', 'readonlyCell')
+    headerCell('STATE')
+      .should('not.have.class', 'dc-derived-header')
+      .and('not.have.class', 'readonlyCell')
+    getCellByHeaderAndRow(0, 'REGION').should('have.class', 'dc-derived-cell')
+    getCellByHeaderAndRow(0, 'STATE').should(
+      'not.have.class',
+      'dc-derived-cell'
+    )
+    // REGION is the first derived column, so it opens the computed block
+    headerCell('REGION').should('have.class', 'dc-derived-first')
+
+    // the hover text carries the label, which is what says what the column is:
+    // the configured VAR_LABEL where there is one, otherwise the generated
+    // name=expression
+    headerCell('REGION_LABEL')
+      .should('have.attr', 'title')
+      .and('contain', 'Region and state')
+    headerCell('REGION')
+      .should('have.attr', 'title')
+      .and('contain', 'REGION=ifc')
+
+    enterEditMode()
+
+    getCellByHeaderAndRow(0, 'REGION').dblclick({ force: true })
+    cy.get('.handsontableInput').should('not.exist')
+    getCellByHeaderAndRow(0, 'REGION').should('have.text', 'West')
+  })
+
+  /**
+   * The positive control for the case above: same screen, same edit mode, a
+   * physical column - so its silence is about the column, not the screen.
+   */
+  it('5 | a physical column in the same grid does accept an edit', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+    enterEditMode()
+
+    getCellByHeaderAndRow(0, 'STATE').dblclick({ force: true })
+    cy.get('.handsontableInput').should('exist')
+  })
 })
 
 /**
@@ -103,6 +152,23 @@ const openEditor = (libMem: string) => {
   cy.wait(2500)
 }
 
+/** The EDIT screen opens read-only; this is the button that starts an edit. */
+const enterEditMode = () => {
+  cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
+    .click()
+    .then(() => cy.wait(2500))
+}
+
+/** The grid header cell for a column, by its rendered text. */
+const headerCell = (headerText: string) =>
+  cy.get('.ht_clone_top .htCore thead tr th').then(($ths) => {
+    const th = [...$ths].find(
+      (candidate) => (candidate as HTMLElement).innerText.trim() === headerText
+    )
+    if (!th) throw new Error(`header not found: ${headerText}`)
+    return cy.wrap(th)
+  })
+
 const openFilterPopup = () => {
   cy.contains('button.btnView', 'Filter', {
     timeout: longerCommandTimeout

client/licenseChecker.js

@@ -10,7 +10,7 @@ const check = (cwd) => {
         onlyAllow:
           'AFLv2.1;Apache 2.0;Apache-2.0;Apache*;Artistic-2.0;BlueOak-1.0.0;0BSD;BSD*;BSD-2-Clause;BSD-3-Clause;CC0-1.0;CC-BY-3.0;CC-BY-4.0;ISC;MIT;MPL-2.0;ODC-By-1.0;Python-2.0;Unlicense;',
         excludePackages:
-          '@cds/city@1.1.0;@handsontable/angular-wrapper@16.0.1;@handsontable/angular-wrapper@17.1.0;@handsontable/angular-wrapper@18.0.0;@handsontable/angular-wrapper@18.1.1;handsontable@^16.0.1;handsontable@16.2.0;handsontable@17.1.0;handsontable@18.0.0;handsontable@18.1.1;hyperformula@2.7.1;hyperformula@3.0.0;hyperformula@3.1.0;hyperformula@3.2.0;hyperformula@3.3.0;jackspeak@3.4.3;path-scurry@1.11.1;package-json-from-dist@1.0.1;buffers@0.1.1'
+          '@cds/city@1.1.0;@handsontable/angular-wrapper@16.0.1;@handsontable/angular-wrapper@17.1.0;@handsontable/angular-wrapper@18.0.0;@handsontable/angular-wrapper@18.1.1;@handsontable/angular-wrapper@18.1.2;handsontable@^16.0.1;handsontable@16.2.0;handsontable@17.1.0;handsontable@18.0.0;handsontable@18.1.1;handsontable@18.1.2;hyperformula@2.7.1;hyperformula@3.0.0;hyperformula@3.1.0;hyperformula@3.2.0;hyperformula@3.3.0;jackspeak@3.4.3;path-scurry@1.11.1;package-json-from-dist@1.0.1;buffers@0.1.1'
       },
       (error, json) => {
         if (error) {

client/package-lock.json

@@ -20,7 +20,7 @@
         "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
         "@clr/icons": "^13.0.2",
         "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
-        "@handsontable/angular-wrapper": "18.1.1",
+        "@handsontable/angular-wrapper": "18.1.2",
         "@sasjs/adapter": "5.1.2",
         "@sasjs/utils": "4.0.0",
         "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
@@ -35,7 +35,7 @@
         "d3-graphviz": "^5.0.2",
         "exceljs": "^4.4.0",
         "fs-extra": "^7.0.1",
-        "handsontable": "18.1.1",
+        "handsontable": "18.1.2",
         "https-browserify": "1.0.0",
         "hyperformula": "^2.5.0",
         "iconv-lite": "^0.5.0",
@@ -969,7 +969,6 @@
       "integrity": "sha512-pQiOg+se1AU/ncMlnJ9V6xYnMQ84qI1BGWuJpbU6A99VTXJg90scg0+T7DWmKssR1YjP5qmmBtrZfKsHEcLW/A==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@angular-eslint/bundled-angular-compiler": "19.8.1",
         "eslint-scope": "^8.0.2"
@@ -1000,7 +999,6 @@
       "integrity": "sha512-HdDfbLwpxzx49wAWz4oOUQKSkZcFNRpWEGsDEgufZ3ExwxRk1/2QPJuvZ2CbhKeEcLPrLKUorQ8XlGrNsJmNiQ==",
       "deprecated": "@angular/animations is deprecated. Use `animate.enter` and `animate.leave` instead. For more information see: https://v22.angular.dev/guide/animations.",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.3.0"
       },
@@ -1146,7 +1144,6 @@
       "resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-20.2.14.tgz",
       "integrity": "sha512-7bZxc01URbiPiIBWThQ69XwOxVduqEKN4PhpbF2AAyfMc/W8Hcr4VoIJOwL0O1Nkq5beS8pCAqoOeIgFyXd/kg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "parse5": "^8.0.0",
         "tslib": "^2.3.0"
@@ -1443,7 +1440,6 @@
       "integrity": "sha512-nqhDw2ZcAUrKNPwhjinJny903bRhI0rQhiDz1LksjeRxqa36i3l75+4iXbOy0rlDpLJGxqtgoPavQjmmyS5UJw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@inquirer/checkbox": "^4.2.1",
         "@inquirer/confirm": "^5.1.14",
@@ -1586,7 +1582,6 @@
       "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.34.tgz",
       "integrity": "sha512-OhB/l0cNldOvQeICqbWqRUzK5fF6zNYy44Q4cIGJZql8HXL/invGiqWLE9f+kU8z4jRisofHIGTBtyJQYYOudg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.3.0"
       },
@@ -1603,7 +1598,6 @@
       "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.34.tgz",
       "integrity": "sha512-8agPW384c/JER1ersPTmJJfcCOnlYcBbdj0MYRxNdiYe+7gtnspF3ORuJmxqCEpMS17AljSmk2skwUcYnHB6xA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.3.0"
       },
@@ -1617,7 +1611,6 @@
       "integrity": "sha512-PKHzhSOyDUFh64Ull3fUwgFlxnsQNEAuRyz4KqQ8FsTGZ4pOpUaZntO+2Xn1nX/gT2whSyGBJhD3H37zA8ZGEQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@babel/core": "7.29.7",
         "@jridgewell/sourcemap-codec": "^1.4.14",
@@ -1650,7 +1643,6 @@
       "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.34.tgz",
       "integrity": "sha512-xOvSmGoao3SVPz//R7zTrq7+WdrIGi+WNpAdxJEvJJXuSpLGS3kSUqukjY4kOMochg2v0NCEPIdCyH+yPs8JSw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.3.0"
       },
@@ -1694,7 +1686,6 @@
       "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.34.tgz",
       "integrity": "sha512-0bd2Bj05WPoBTFM0A13IwbzxKVQPstx/N+7X0ZWKCifliyydIc7DBHu+r37ybJpl8ZsCVBe7DoR9Gon4lZ+zlw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.3.0"
       },
@@ -1790,7 +1781,6 @@
       "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@babel/code-frame": "^7.29.7",
         "@babel/generator": "^7.29.7",
@@ -2317,6 +2307,7 @@
       "integrity": "sha512-oBNVCvnO5tND+xSopWvV8WNGfpTfgP4Zr/YXXSj8zfmcPktp5Ku/aZlsIowgSD4fjmgHn6sGmB9APVsU5zOdhA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.29.7",
         "@babel/helper-skip-transparent-expression-wrappers": "^7.29.7"
@@ -3362,6 +3353,7 @@
       "integrity": "sha512-GYzX36n1nsciIb0uyH0GHwxwtNwPQIcpxSeiVLDtG/B7jB5xXgchnmL1f/jCX5o+pwnaDBtO60ONSJhEBJfxYA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/compat-data": "^7.29.7",
         "@babel/helper-compilation-targets": "^7.29.7",
@@ -3448,6 +3440,7 @@
       "integrity": "sha512-d98gXZkgswvkyohMBABkhm3GeXhYj8psWfwQ2C7gtfrKGTykQa/iOIi+JJhwMjPlZ6Vm2XN+DCf3Es1EoG4ZLA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/helper-plugin-utils": "^7.29.7",
         "@babel/helper-remap-async-to-generator": "^7.29.7",
@@ -3466,6 +3459,7 @@
       "integrity": "sha512-pcUb2SS+RMo9TWVBwKGI5ShtoG7R+zBsFmCKDa6fe8c+hPr3XJlZgoE5j6i8W7gDjhyvy+85vmYexanvXh3d1w==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/helper-module-imports": "^7.29.7",
         "@babel/helper-plugin-utils": "^7.29.7",
@@ -3484,6 +3478,7 @@
       "integrity": "sha512-coWpDLJ410R781Npmn/SIBZEsAetR4xVi0SxLMXPaMO4lSf1MwnkGYMtkFxew0Dn8B3/CpbpYxN0JCgg8mn67g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/helper-define-polyfill-provider": "^0.6.8",
         "core-js-compat": "^3.48.0"
@@ -3498,6 +3493,7 @@
       "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
       "dev": true,
       "license": "ISC",
+      "peer": true,
       "bin": {
         "semver": "bin/semver.js"
       }
@@ -3604,7 +3600,6 @@
       "resolved": "https://registry.npmjs.org/@cds/core/-/core-6.17.0.tgz",
       "integrity": "sha512-afG+KwNE4BDc6CAylgGbJrJUeGYFbOIfagylaqqbS7fgIX2h3ZqFcUOv/2hujSZ9GZiuCSa+A/BkJi7K83QEgg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "lit": "^2.1.3",
         "ramda": "^0.29.0",
@@ -3786,7 +3781,6 @@
       "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "readdirp": "^5.0.0"
       },
@@ -4904,9 +4898,9 @@
       }
     },
     "node_modules/@handsontable/angular-wrapper": {
-      "version": "18.1.1",
-      "resolved": "https://registry.npmjs.org/@handsontable/angular-wrapper/-/angular-wrapper-18.1.1.tgz",
-      "integrity": "sha512-paiINuWOevybN6RFUrtSOgRv5SaFlMd5dRbBOzeLyJJzqhLW7Jg+QSy6+51FAL6FKHZPfWHYoP0F2YgNPEXYbg==",
+      "version": "18.1.2",
+      "resolved": "https://registry.npmjs.org/@handsontable/angular-wrapper/-/angular-wrapper-18.1.2.tgz",
+      "integrity": "sha512-LoUc3jIO7Y5rkSzHW+njvQWA6x8RipkWmBkR7vmnEqgxcytejgd7RIy6/b3O0AqYuv2XUT+vXrNacn9wEekWhQ==",
       "license": "SEE LICENSE IN LICENSE.txt",
       "optionalDependencies": {
         "tslib": "^2.3.0"
@@ -7176,7 +7170,6 @@
       "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "engines": {
         "node": ">=8.0.0"
       }
@@ -7200,7 +7193,6 @@
       "integrity": "sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "engines": {
         "node": ">=14"
       },
@@ -7214,7 +7206,6 @@
       "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "@opentelemetry/semantic-conventions": "1.28.0"
       },
@@ -7241,7 +7232,6 @@
       "integrity": "sha512-BdBGhQBh8IjZ2oIIX6F2/Q3LKm/FDDKi6ccYKcBTeilh6SNdNKveDOLk73BkSJjQLJk6qe4Yh+hHw1UPhCDdrg==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "@opentelemetry/api-logs": "0.57.2",
         "@types/shimmer": "^1.2.0",
@@ -7672,7 +7662,6 @@
       "integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "@opentelemetry/core": "1.30.1",
         "@opentelemetry/semantic-conventions": "1.28.0"
@@ -7700,7 +7689,6 @@
       "integrity": "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "@opentelemetry/core": "1.30.1",
         "@opentelemetry/resources": "1.30.1",
@@ -7729,7 +7717,6 @@
       "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "engines": {
         "node": ">=14"
       }
@@ -9377,7 +9364,8 @@
       "resolved": "https://registry.npmjs.org/@types/hammerjs/-/hammerjs-2.0.46.tgz",
       "integrity": "sha512-ynRvcq6wvqexJ9brDMS4BnBLzmr0e14d6ZJTEShTBWKymQiHwlAyGu0ZPEFI2Fh1U53F7tN9ufClWM5KvqkKOw==",
       "dev": true,
-      "license": "MIT"
+      "license": "MIT",
+      "peer": true
     },
     "node_modules/@types/http-errors": {
       "version": "2.0.5",
@@ -9721,7 +9709,6 @@
       "integrity": "sha512-CZ4nMxWwgu1HEEFNkeaCptra9QCtkmKdgf3sWh1rl1trIhmxLilgTV4cwcbQ4wemnT4sWQN8CaKOmdYx+g2gMA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@typescript-eslint/scope-manager": "8.65.0",
         "@typescript-eslint/types": "8.65.0",
@@ -9895,7 +9882,6 @@
       "integrity": "sha512-Adfoy59kxNEjT5E5fnyrp7GpHYuWs8ONbdohu4ez6IU46qDq8LltG90tLmkucZB2uhYL8mqBjtXqGd8yhfjXqQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       },
@@ -9990,6 +9976,7 @@
       "integrity": "sha512-bzHngBbmIgzj6TsTjVUCntGx6FFLgE7mPjpO0PzHycOaZHXqS43XnrkFIHRS16LLr/xO6FHSXG3GJFo/F4bSBQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/tsconfig-utils": "^8.72.0",
         "@typescript-eslint/types": "^8.72.0",
@@ -10012,6 +9999,7 @@
       "integrity": "sha512-JewPyJOYcXBSAS7PIgufscHOwZqhTGyBRiQJhhuDln66tMchxvvt7nJ90JKKIoOelmdHA3kAQbm9cEUHc4JSEA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/types": "8.72.0",
         "@typescript-eslint/visitor-keys": "8.72.0"
@@ -10030,6 +10018,7 @@
       "integrity": "sha512-0vbxCf7xLVPSjcOdm6eQbgViH8EDJ8xnl1laLT9yoqSz2YmcSfSg5Vdl0JUvZf6rcuJ0ZivJIuMD4edTg8obaA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       },
@@ -10047,6 +10036,7 @@
       "integrity": "sha512-gwLfFgWcw15tN2KXDCJvxEOQjPn+P3j/n3eSqkZ5sX4VpW9Cosodr+tWwBZRvJT3TcKx6HnPRGZMCNAiP1Xhug==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/project-service": "8.72.0",
         "@typescript-eslint/tsconfig-utils": "8.72.0",
@@ -10075,6 +10065,7 @@
       "integrity": "sha512-E2mroviasaWecDzTGiGMtuEguuLVOeRggnrIn93CJuxYJzQBHbyymlT08aVQydbBal5538o+JkBKXh2sRuCyIA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/types": "8.72.0",
         "eslint-visitor-keys": "^5.0.0"
@@ -10093,6 +10084,7 @@
       "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==",
       "dev": true,
       "license": "Apache-2.0",
+      "peer": true,
       "engines": {
         "node": "^20.19.0 || ^22.13.0 || >=24"
       },
@@ -10106,6 +10098,7 @@
       "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
       "dev": true,
       "license": "ISC",
+      "peer": true,
       "bin": {
         "semver": "bin/semver.js"
       },
@@ -10398,7 +10391,6 @@
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz",
       "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==",
       "license": "MIT",
-      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -10469,7 +10461,6 @@
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
       "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
         "fast-uri": "^3.0.1",
@@ -10933,7 +10924,6 @@
       "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz",
       "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "follow-redirects": "^1.16.0",
         "form-data": "^4.0.6",
@@ -11069,7 +11059,6 @@
       "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz",
       "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==",
       "license": "Apache-2.0",
-      "peer": true,
       "peerDependencies": {
         "bare-abort-controller": "*"
       },
@@ -11441,7 +11430,8 @@
       "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz",
       "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==",
       "dev": true,
-      "license": "ISC"
+      "license": "ISC",
+      "peer": true
     },
     "node_modules/browserify-aes": {
       "version": "1.2.0",
@@ -11499,7 +11489,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "baseline-browser-mapping": "^2.11.12",
         "caniuse-lite": "^1.0.30001809",
@@ -11873,7 +11862,6 @@
       "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "readdirp": "^4.0.1"
       },
@@ -12305,7 +12293,8 @@
       "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz",
       "integrity": "sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==",
       "dev": true,
-      "license": "ISC"
+      "license": "ISC",
+      "peer": true
     },
     "node_modules/common-tags": {
       "version": "1.8.2",
@@ -12986,7 +12975,6 @@
       "dev": true,
       "hasInstallScript": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@cypress/request": "^4.0.0",
         "@cypress/xvfb": "^1.2.4",
@@ -13388,7 +13376,6 @@
       "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
       "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
       "license": "ISC",
-      "peer": true,
       "engines": {
         "node": ">=12"
       }
@@ -13664,8 +13651,7 @@
       "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1625959.tgz",
       "integrity": "sha512-wRBSU330hwOLLcb3N4NIe3eFs6MgT6ku3AiZONjnTSJ7f3dVchJfn6nE0Lfos9jK1na15bgp7xLhaCx40Y47NQ==",
       "dev": true,
-      "license": "BSD-3-Clause",
-      "peer": true
+      "license": "BSD-3-Clause"
     },
     "node_modules/dezalgo": {
       "version": "1.0.4",
@@ -13691,6 +13677,7 @@
       "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==",
       "dev": true,
       "license": "BSD-3-Clause",
+      "peer": true,
       "engines": {
         "node": ">=0.3.1"
       }
@@ -14311,7 +14298,6 @@
       "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.6.1",
@@ -14757,7 +14743,6 @@
       "integrity": "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
@@ -15185,6 +15170,7 @@
       "integrity": "sha512-9ZonPT4ZAK4a+1pUPVPZJapbi7O5qbbJPdYw/NOQWZZbVLdDTYM3A4R9z/DpAM08IDaFGsvPgiGZ82WEwUDWjg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "common-path-prefix": "^3.0.0",
         "pkg-dir": "^7.0.0"
@@ -15219,6 +15205,7 @@
       "integrity": "sha512-afd4O7zpqHeRyg4PfDQsXmlDe2PfdHtJt6Akt8jOWaApLOZk5JXs6VMR29lz03pRe9mpykrRCYIYxaJYcfpncQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -15744,11 +15731,10 @@
       }
     },
     "node_modules/handsontable": {
-      "version": "18.1.1",
-      "resolved": "https://registry.npmjs.org/handsontable/-/handsontable-18.1.1.tgz",
-      "integrity": "sha512-8vz/jp96m0aJQPRepOBoImsC/bLe9mE/O4nOB6mll9DCjN5nqXHX9LV4zu/XDpD89TKJwtA+TNhVUkFQxW+q7w==",
-      "license": "SEE LICENSE IN LICENSE.txt",
-      "peer": true
+      "version": "18.1.2",
+      "resolved": "https://registry.npmjs.org/handsontable/-/handsontable-18.1.2.tgz",
+      "integrity": "sha512-iL4Cia0Fsl+md1oCqUS8PJqfft3583d5ptc5M7ewfp6JTUsNsrIcpF3GPYvZy57e7P5PACX8WboGlkwgHn0h9Q==",
+      "license": "SEE LICENSE IN LICENSE.txt"
     },
     "node_modules/has-flag": {
       "version": "4.0.0",
@@ -15926,7 +15912,6 @@
       "integrity": "sha512-6E2QDAc9Ick9Sq77ZrGS/dk2WUYni91aufTw6LJKpV7w8kW5/GxVUc650FOADOmlwg3K+f7Pun6XlV+pYmW6gw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=16.9.0"
       }
@@ -17124,8 +17109,7 @@
       "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-5.1.2.tgz",
       "integrity": "sha512-2oIUMGn00FdUiqz6epiiJr7xcFyNYj3rDcfmnzfkBnHyBQ3cBQUs4mmyGsOb7TTLb9kxk7dBcmEmqhDKkBoDyA==",
       "dev": true,
-      "license": "MIT",
-      "peer": true
+      "license": "MIT"
     },
     "node_modules/jest-worker": {
       "version": "27.5.1",
@@ -17162,7 +17146,6 @@
       "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "jiti": "bin/jiti.js"
       }
@@ -17449,7 +17432,6 @@
       "integrity": "sha512-LrtUxbdvt1gOpo3gxG+VAJlJAEMhbWlM4YrFQgql98FwF7+K8K12LYO4hnDdUkNjeztYrOXEMqgTajSWgmtI/w==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@colors/colors": "1.5.0",
         "body-parser": "^1.19.0",
@@ -18005,7 +17987,6 @@
       "integrity": "sha512-umRhrCH7fCi8Uj2RcwKjJdvUORTjeWqkdKx0LbcZvjIwsAVsnIAGcxHaqowPeBFBjQuWOeC/bve0AlpFzF/+SQ==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "copy-anything": "^3.0.5",
         "parse-node-version": "^1.0.1"
@@ -18467,7 +18448,6 @@
       "integrity": "sha512-SL0JY3DaxylDuo/MecFeiC+7pedM0zia33zl0vcjgwcq1q1FWWF1To9EIauPbl8GbMCU0R2e0uJ8bZunhYKD2g==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "cli-truncate": "^4.0.0",
         "colorette": "^2.0.20",
@@ -19594,6 +19574,7 @@
       "integrity": "sha512-beammieDdiWdkjANWeWPpkxUROYGi5FN88fPOkLiticz7HTg3Y+Pho3EYE6SUdcuWnZ3h2WD8EXCA55JF4o6wg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "browser-stdout": "^1.3.1",
         "chokidar": "^5.0.0",
@@ -19634,7 +19615,8 @@
           "url": "https://github.com/sponsors/nodeca"
         }
       ],
-      "license": "PSF-2.0"
+      "license": "PSF-2.0",
+      "peer": true
     },
     "node_modules/mocha/node_modules/chokidar": {
       "version": "5.0.0",
@@ -19642,6 +19624,7 @@
       "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "readdirp": "^5.0.0"
       },
@@ -19658,6 +19641,7 @@
       "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==",
       "dev": true,
       "license": "BlueOak-1.0.0",
+      "peer": true,
       "dependencies": {
         "minimatch": "^10.2.2",
         "minipass": "^7.1.3",
@@ -19676,6 +19660,7 @@
       "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -19699,6 +19684,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "argparse": "^3.0.3"
       },
@@ -19712,6 +19698,7 @@
       "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">= 20.19.0"
       },
@@ -19726,6 +19713,7 @@
       "integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=14.16"
       },
@@ -19739,6 +19727,7 @@
       "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "has-flag": "^4.0.0"
       },
@@ -21446,6 +21435,7 @@
       "integrity": "sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "find-up": "^6.3.0"
       },
@@ -21462,6 +21452,7 @@
       "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "locate-path": "^7.1.0",
         "path-exists": "^5.0.0"
@@ -21479,6 +21470,7 @@
       "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "p-locate": "^6.0.0"
       },
@@ -21495,6 +21487,7 @@
       "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "yocto-queue": "^1.0.0"
       },
@@ -21511,6 +21504,7 @@
       "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "p-limit": "^4.0.0"
       },
@@ -21527,6 +21521,7 @@
       "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
       }
@@ -21537,6 +21532,7 @@
       "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=12.20"
       },
@@ -21609,7 +21605,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "nanoid": "^3.3.16",
         "picocolors": "^1.1.1",
@@ -22845,7 +22840,6 @@
       "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
       "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.1.0"
       }
@@ -22900,7 +22894,6 @@
       "integrity": "sha512-9GUyuksjw70uNpb1MTYWsH9MQHOHY6kwfnkafC24+7aOMZn9+rVMBxRbLvw756mrBFbIsFg6Xw9IkR2Fnn3k+Q==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "chokidar": "^4.0.0",
         "immutable": "^5.0.2",
@@ -24209,7 +24202,6 @@
       "integrity": "sha512-+6erLbBm0+LROX2sPXlUYx/ux5PyE9K/a92Wrt6oA+WDAoFTdpHE5tCYCI5PNzq2y8df4rA+QgHLJuR4jNymsg==",
       "dev": true,
       "license": "BSD-2-Clause",
-      "peer": true,
       "dependencies": {
         "@jridgewell/source-map": "^0.3.3",
         "acorn": "^8.14.0",
@@ -24438,7 +24430,6 @@
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
       "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
       "license": "BSD-3-Clause",
-      "peer": true,
       "dependencies": {
         "tldts": "^6.1.32"
       },
@@ -24701,8 +24692,7 @@
       "version": "2.8.1",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
       "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
-      "license": "0BSD",
-      "peer": true
+      "license": "0BSD"
     },
     "node_modules/tsyringe": {
       "version": "4.10.0",
@@ -24869,7 +24859,6 @@
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "dev": true,
       "license": "Apache-2.0",
-      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -24925,19 +24914,10 @@
       "integrity": "sha512-EvRlUt/QqAEvXdQeRMYUfwOQsfzpDztykrG2owGEE/rBOBoYv9uF39ZzsNDh4lTFHhTr2XOZjKkYgueD12ZmGA==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18.17"
       }
     },
-    "node_modules/undici-types": {
-      "version": "8.9.0",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
-      "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
-      "dev": true,
-      "license": "MIT",
-      "optional": true
-    },
     "node_modules/unicode-canonical-property-names-ecmascript": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz",
@@ -25127,7 +25107,6 @@
         "https://github.com/sponsors/ctavan"
       ],
       "license": "MIT",
-      "peer": true,
       "bin": {
         "uuid": "dist/esm/bin/uuid"
       }
@@ -25251,7 +25230,6 @@
       "integrity": "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "esbuild": "^0.27.0 || ^0.28.0",
         "fdir": "^6.5.0",
@@ -25442,7 +25420,6 @@
       "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.112.0.tgz",
       "integrity": "sha512-NIBpsRuCll036haDDt/CfcTDfsWNR9Z782nMRMI9D377BsGcYzZDTNU5TyzhCJzPIcihAxMBaMAhq2WSaDrH8Q==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@types/estree": "^1.0.8",
         "@types/json-schema": "^7.0.15",
@@ -25514,7 +25491,6 @@
       "integrity": "sha512-HNLRmamRvVavZQ+avceZifmv8hmdUjg43t6MI4SqJDwFdW7RPQwH5vzGhDRZSX59SgfbeHhLnq3g+uooWo7pVw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@types/bonjour": "^3.5.13",
         "@types/connect-history-api-fallback": "^1.5.4",
@@ -25906,7 +25882,8 @@
       "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-10.0.3.tgz",
       "integrity": "sha512-6z2Iis68Wqth93/G/wJP9u+R3O+d2XTlgWChGCwuT1qLbBsOYueGRZuJ++v3mtDP5KjYdy+WzvWC+VWETSVXJA==",
       "dev": true,
-      "license": "Apache-2.0"
+      "license": "Apache-2.0",
+      "peer": true
     },
     "node_modules/wrap-ansi": {
       "version": "9.0.2",
@@ -26197,7 +26174,6 @@
       "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "pend": "~1.2.0"
       },
@@ -26314,7 +26290,6 @@
       "integrity": "sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"
       }
@@ -26333,8 +26308,7 @@
       "version": "0.15.1",
       "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.15.1.tgz",
       "integrity": "sha512-XE96n56IQpJM7NAoXswY3XRLcWFW83xe0BiAOeMD7K5k5xecOeul3Qcpx6GqEeeHNkW5DWL5zOyTbEfB4eti8w==",
-      "license": "MIT",
-      "peer": true
+      "license": "MIT"
     }
   }
 }

client/package.json

@@ -54,7 +54,7 @@
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
     "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
-    "@handsontable/angular-wrapper": "18.1.1",
+    "@handsontable/angular-wrapper": "18.1.2",
     "@sasjs/adapter": "5.1.2",
     "@sasjs/utils": "4.0.0",
     "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
@@ -69,7 +69,7 @@
     "d3-graphviz": "^5.0.2",
     "exceljs": "^4.4.0",
     "fs-extra": "^7.0.1",
-    "handsontable": "18.1.1",
+    "handsontable": "18.1.2",
     "https-browserify": "1.0.0",
     "hyperformula": "^2.5.0",
     "iconv-lite": "^0.5.0",

client/src/app/app.component.ts

@@ -314,9 +314,12 @@ export class AppComponent {
    */
   public subscribeToShowAbortModal() {
     this.eventService.onShowAbortModal.subscribe((sasjsAbort: InfoModal) => {
-      let abortId = this.sasjsAborts.length + 1
-      sasjsAbort.id = abortId
-      this.sasjsAborts.push(sasjsAbort)
+      /* One message at a time.  Pushing to the array stacked a second modal on
+         top of the one already open - two dismissals for one failure, and the
+         lower one was often a duplicate of the upper.  Replacing keeps the
+         newest message, which is the one the user has to act on. */
+      sasjsAbort.id = Date.now()
+      this.sasjsAborts = [sasjsAbort]
       this.cdr.detectChanges() //Changes were not triggered while hot is focused
     })
   }

client/src/app/editor/editor.component.ts

@@ -575,6 +575,8 @@ export class EditorComponent
   public hasBaseDropZoneOver = false
   public hasAnotherDropZoneOver = false
   public headerPks: string[] = []
+  /** Derived (computed) columns from MPE_DERIVED - read-only, and marked as such */
+  public derivedVars: string[] = []
   public columnLevelSecurityFlag = false
   public dateTimeHeaders: string[] = []
   public timeHeaders: string[] = []
@@ -1089,6 +1091,20 @@ export class EditorComponent
     return colRules?.readOnly
   }
 
+  /**
+   * A derived (computed) column - configured in MPE_DERIVED, created at read
+   * time. Read-only, and styled apart from an ordinary read-only column so the
+   * user can see that there is nothing to edit rather than that they may not.
+   */
+  isDerivedCol(col: string | number) {
+    const name =
+      typeof col === 'number'
+        ? (this.hotInstance?.colToProp(col) as string) || this.columnHeader[col]
+        : col
+
+    return !!name && this.derivedVars.includes(name.toUpperCase())
+  }
+
   isColHeader(col: string) {
     return this.headerArray.indexOf(col.toUpperCase()) > -1
   }
@@ -2757,17 +2773,24 @@ export class EditorComponent
             this.disableSubmit = true
             this.submit = false
 
-            const errorText =
-              typeof err.adapterRespnse === 'string'
-                ? err.adapterRespnse
-                : JSON.stringify(err.adapterRespnse)
+            /* sasService has already shown the SAS message when it rejected
+               (the abort, or the adapter's error), so reporting it again here
+               stacked a second modal on top of the first - and read
+               'undefined' because it looked for a property that does not
+               exist.  Only speak up when nothing was shown. */
+            if (!err?.errorShown) {
+              const errorText =
+                typeof err?.adapterResponse?.error === 'string'
+                  ? err.adapterResponse.error
+                  : JSON.stringify(err?.adapterResponse ?? err)
 
-            this.eventService.showAbortModal(
-              'editors/stagedata',
-              `Submit request failed\n${errorText}`,
-              null,
-              'Submit error'
-            )
+              this.eventService.showAbortModal(
+                'editors/stagedata',
+                `Submit request failed\n${errorText}`,
+                null,
+                'Submit error'
+              )
+            }
 
             return false
           })
@@ -3941,8 +3964,11 @@ export class EditorComponent
 
     // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
     // truncated to the first rows within the limit, and carries the reason
-    // here.  Empty means the whole selection is in the grid.
-    this.truncationNote = response.data.sasparams[0].TRUNC_MSG || ''
+    // here.  Empty means the whole selection is in the grid - and a blank
+    // character arrives as a single blank (mp_jsonout cannot emit a zero-length
+    // string, since SAS has none), so trim it: whitespace is "no note", not a
+    // warning with an empty tooltip.
+    this.truncationNote = (response.data.sasparams[0].TRUNC_MSG || '').trim()
 
     const hot = this.hotInstance
 
@@ -3964,6 +3990,12 @@ export class EditorComponent
 
     this.headerColumns = response.data.sasparams[0].COLHEADERS.split(',')
     this.headerPks = response.data.sasparams[0].PK.split(' ')
+    // Derived (computed) columns, named by getdata from MPE_DERIVED - read-only
+    // through a READONLY DQ rule, and marked in the grid so they read as
+    // computed rather than merely non-editable.
+    this.derivedVars = (response.data.sasparams[0].DERIVEDVARS || '')
+      .split(' ')
+      .filter((col: string) => !!col)
     this.columnLevelSecurityFlag = !!response.data.sasparams[0].CLS_FLAG
 
     if (this.columnLevelSecurityFlag)
@@ -3990,6 +4022,19 @@ export class EditorComponent
     // rules must stay index-aligned with headerColumns).
     this.headerColumns.push(EDIT_STATUS_COLUMN_NAME)
 
+    /**
+     * columnHeader is the same list, taken once it is final (Delete? renamed,
+     * EDIT_STATUS appended) - it is the grid's own column order, which is what
+     * every hook that maps a visual column index to a name needs.
+     *
+     * This has to happen HERE, before the updateSettings below installs
+     * afterGetColHeader / afterGetCellMeta: those fire during the render that
+     * updateSettings triggers, and reading a columnHeader that was still being
+     * assigned further down meant they saw the previous table's columns (or
+     * nothing). That is what put the derived-block marker on the wrong column.
+     */
+    this.columnHeader = this.headerColumns.slice()
+
     if (response.data.sasparams[0].DTVARS !== '') {
       this.dateHeaders = response.data.sasparams[0].DTVARS.split(' ')
     }
@@ -4111,8 +4156,9 @@ export class EditorComponent
     this.dataSourceUnchanged = this.helperService.deepClone(this.dataSource)
     this.overlayFormulaRawValuesOnUnchanged(this.dataSourceUnchanged)
 
-    // Note: this.headerColumns and this.columnHeader contains same data
-    // need to resolve redundancy
+    // columnHeader is a copy of headerColumns taken above, once the Delete?
+    // rename and the EDIT_STATUS append had been applied - the hooks read it
+    // for grid-index-to-name mapping, so the two must not drift apart again.
 
     // default schema - includes NOTNULL defaults from DQ rules
     for (let i = 0; i < this.headerColumns.length; i++) {
@@ -4338,9 +4384,37 @@ export class EditorComponent
           // header columns styling - primary keys
           const isPKCol = column && this.isColPk(column)
           const isReadonlyCol = column && this.isReadonlyCol(column)
+          const isDerivedCol = column && this.isDerivedCol(column)
 
           if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
-          if (isReadonlyCol && !isPKCol) th.classList.add('readonlyCell')
+          if (isReadonlyCol && !isPKCol && !isDerivedCol) {
+            th.classList.add('readonlyCell')
+          }
+
+          // a derived column is computed, not merely locked - give it its own
+          // hue and an 'fx' marker rather than the read-only dimming
+          if (isDerivedCol) {
+            th.classList.add('dc-derived-header')
+            // the label says what the column is - for a derived variable that
+            // is the configured VAR_LABEL, or the generated name=expression
+            // when the administrator left it blank.  A label that merely
+            // repeats the name adds nothing, so it falls back to the rule.
+            const derivedLabel = (this.cols || [])
+              .filter((c: any) => c.NAME === column)
+              .map((c: any) => String(c.LABEL || '').trim())
+              .find(
+                (l: string) =>
+                  l.length > 0 &&
+                  l.toUpperCase() !== String(column).toUpperCase()
+              )
+            th.title = derivedLabel
+              ? `${derivedLabel} - derived variable, computed at read time and cannot be edited`
+              : 'Derived variable - computed at read time and cannot be edited'
+            const firstDerivedCol = this.columnHeader.findIndex(
+              (name: string) => this.isDerivedCol(name)
+            )
+            if (col === firstDerivedCol) th.classList.add('dc-derived-first')
+          }
 
           // Remove header arrow from Delete column
           if (col === 0) {
@@ -4356,6 +4430,7 @@ export class EditorComponent
           cellProperties: Handsontable.CellProperties
         ) => {
           const isReadonlyCol = col && this.isReadonlyCol(col)
+          const isDerivedCol = col && this.isDerivedCol(col)
 
           // Check if this cell should be marked as invalid due to duplicate primary key values
           // Only applies to primary key columns (col 1 through readOnlyFields)
@@ -4379,7 +4454,26 @@ export class EditorComponent
           }
 
           // Add readonlyCell class for readonly columns to maintain original styling
-          if (isReadonlyCol && !classes.includes('readonlyCell')) {
+          if (isDerivedCol) {
+            // a derived cell carries its own tint; the read-only dimming would
+            // wash it out, and "greyed out" is a different message from
+            // "computed here"
+            classes = classes.filter((c: string) => c !== 'readonlyCell')
+            if (!classes.includes('dc-derived-cell')) {
+              classes.push('dc-derived-cell')
+            }
+            // derived columns are appended after the physical ones - mark the
+            // start of the block so it reads as one group
+            const firstDerivedCol = this.columnHeader.findIndex(
+              (name: string) => this.isDerivedCol(name)
+            )
+            if (
+              col === firstDerivedCol &&
+              !classes.includes('dc-derived-first')
+            ) {
+              classes.push('dc-derived-first')
+            }
+          } else if (isReadonlyCol && !classes.includes('readonlyCell')) {
             classes.push('readonlyCell')
           }
 
@@ -4423,9 +4517,6 @@ export class EditorComponent
     // hot.getPlugin('manualColumnFreeze').freezeColumn(0);
 
     this.queryText = response.data.sasparams[0].FILTER_TEXT
-    this.columnHeader = response.data.sasparams[0].COLHEADERS.split(',')
-    // First column is always used to mark records for deletion
-    this.columnHeader[0] = 'Delete?'
     this.readOnlyFields = response.data.sasparams[0].PKCNT
 
     hot.addHook(

client/src/app/models/TableData.ts

@@ -51,6 +51,11 @@ export interface SASParam {
   TRUNCATED?: number
   /** Reason shown in the tooltip when TRUNCATED is 1, else empty */
   TRUNC_MSG?: string
+  /**
+   * Space separated list of the derived (computed) columns from MPE_DERIVED,
+   * upper case. Empty when the table has none.
+   */
+  DERIVEDVARS?: string
 }
 
 export interface ExcelRule {

client/src/app/multi-dataset/multi-dataset.component.ts

@@ -868,6 +868,21 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {
       return this.sasStoreService
         .callService(myParams, 'SASControlTable', 'editors/getdata', libds)
         .then((res: EditorsGetDataServiceResponse) => {
+          // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
+          // truncated, rather than aborting the way it used to. This flow compares
+          // the whole table against the upload, so a partial read would compute
+          // the changes against the wrong starting state - refuse the table
+          // instead of submitting differences nobody asked for.
+          const sasParams = res?.data?.sasparams?.[0]
+
+          if (sasParams && Number(sasParams.TRUNCATED) === 1) {
+            this.eventService.showInfoModal(
+              'Upload error',
+              `${libds} holds more than the EDIT limit allows (${sasParams.TRUNC_MSG}), so it cannot be compared with the upload. Raise DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT, or submit the table without the multi-table upload.`
+            )
+            return undefined
+          }
+
           return res
         })
         .catch((err: any) => {

client/src/app/services/sas.service.spec.ts

@@ -186,7 +186,8 @@ describe('SasService openServiceInNewWindow', () => {
 
     expect(openSpy).toHaveBeenCalledWith(
       'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_contextname=ctx&_debug=131',
-      '_blank'
+      '_blank',
+      'noopener'
     )
   })
 
@@ -199,7 +200,8 @@ describe('SasService openServiceInNewWindow', () => {
 
     expect(openSpy).toHaveBeenCalledWith(
       'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131',
-      '_blank'
+      '_blank',
+      'noopener'
     )
   })
 })

client/src/app/services/sas.service.ts

@@ -286,7 +286,10 @@ export class SasService {
                 adapterResponse: {
                   error: abortMsg
                 },
-                log: sasRequest?.logFile
+                log: sasRequest?.logFile,
+                /* the modal above already told the user: a caller that also
+                   reports the failure would stack a second one on top */
+                errorShown: !wrapperOptions?.suppressSuccessAbortModal
               })
             }
 
@@ -329,7 +332,8 @@ export class SasService {
                 adapterResponse: {
                   error: errorMessage
                 },
-                log: sasRequest?.logFile
+                log: sasRequest?.logFile,
+                errorShown: !wrapperOptions?.suppressErrorAbortModal
               })
             }
 
@@ -867,7 +871,10 @@ export class SasService {
       params
     })
 
-    window.open(url, '_blank')
+    // noopener: the SAS log page is on a different origin (Viya's serverUrl is
+    // not the app's), and without it the opened window keeps a handle on
+    // window.opener and can navigate this tab.
+    window.open(url, '_blank', 'noopener')
   }
 
   public getServerType(): string {

client/src/styles.scss

@@ -5019,6 +5019,57 @@ body[cds-theme='dark'] {
   border: 1px solid var(--cds-alias-status-warning) !important;
 }
 
+/* Derived (computed) columns: read-only, but computed at read time rather than
+  locked by policy.  Tinted and marked with an 'fx' glyph instead of the
+  read-only dimming, so the two do not read as the same thing.  Blue is the
+  info family - the primary key green and the warning ochre keep their meaning. */
+.handsontable th.dc-derived-header {
+  background: #d3e5f2 !important;
+  box-shadow: inset 0 -2px 0 #1a6ea3;
+}
+
+.handsontable th.dc-derived-header div::after {
+  content: 'fx';
+  margin-left: 4px;
+  font-size: 9px;
+  font-style: italic;
+  opacity: 0.7;
+}
+
+.handsontable td.dc-derived-cell {
+  background: #eef5fa !important;
+  font-style: italic;
+}
+
+/* the first derived column starts the computed block */
+.handsontable th.dc-derived-first {
+  border-left: 2px solid #1a6ea3 !important;
+}
+
+.handsontable td.dc-derived-cell.dc-derived-first {
+  box-shadow: inset 2px 0 0 #1a6ea3;
+}
+
+body[cds-theme='dark'] {
+  .handsontable th.dc-derived-header {
+    background: #2b4a5c !important;
+    box-shadow: inset 0 -2px 0 #4b9cc9;
+  }
+
+  .handsontable td.dc-derived-cell {
+    background: #24343d !important;
+    color: #dbe7ee;
+  }
+
+  .handsontable th.dc-derived-first {
+    border-left-color: #4b9cc9 !important;
+  }
+
+  .handsontable td.dc-derived-cell.dc-derived-first {
+    box-shadow: inset 2px 0 0 #4b9cc9;
+  }
+}
+
 .handsontable .numericListbox {
   text-align: right;
 }

sas/deploy/makeDataSas719.json

@@ -0,0 +1,10 @@
+{
+  "fromjs": [
+    {
+      "ADMIN": "all-users",
+      "DCPATH": "/tmp/dcdata",
+      "OUTFMT": "JSON",
+      "KEEPME": "Y"
+    }
+  ]
+}

sas/mocks/mock-licence.json

@@ -0,0 +1,4 @@
+{
+  "licenceKey": "G4H/DNTdEMkmCIxOrYkRYTQ1P4sWg0gs8Zk9rakxFNYDZcLyEPMJJo82YJQ/S24P2E/YPxaWsDZXgOrWKfeCDS7NrLcwsoFNa8KRF2WTlRgSMiG4sjBRX8zNEIvszj5DAq4LEgOUUAdfctNfpUcygbxT1dhqy6gU3hBu/xRuXGd36rhxQwfahaKkfaCg/rFplkrvqnjoJ6622BxWP/HG1dxZUE7HNJyT4tNaXzlC9U6u/aaJJ66+V+9511BwjM9ss4nyHuHoogVE1yrzLMlL94P3QoM1hwkGHnEHmFzg3zHbQG1f3pUs9YzK0ytsFKU7AK948gJrGbf37Zqa+Mz7EZf6XzqqXx0zQ4y9zAhLV9s4QG8JdhrByhfG5kefrs4L/BpgKq3ElgYL5EMkrLJUrlqWYI2ouxJuIGCQz41VCWXJ2S7pd39ljifc8Zj+t2Mn5EEkRVsrbdxjCkyoZOQ1ALofa1+auUI26Rm5rwEgrfhEocDmphJpO2F8lZgwDxbepSE6v/vEV/JQWBQTyrX10W+qOEu8OhvZrYJo+eIJMaO7q1pjfdrn3XhoH+BeB3Sy1sevGLXQZC+CzOtcvajetGGCDAabtTLc49ouWPUiUOAWPE3uYymsMoRZgpLIMxaUM86hoK0r9T6wSaP0bddK754sXN1jWwIDAHSGz5WO1ko=",
+  "activationKey": "MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQD2w8Ge6tMI82vaQ7x5ZdlwnTJycn0UFZoSgRwObaM0NC5wp9Y8lwkVzN8FUhg2oYSTpvyJpix+VGRj0AlNBwFAVcJG9VDPYh2LE3Us9ghRfkzFUiM2PqorHnzL5DwpailRtmFHoDQPGwwSmKncTo2QHql/K6tWe7g3Nv9uqQNks283hvDQ6PVGU4TaFTSRbjGX4j+JAR651J6wI6nY+9kNxOBwnL/yKTnJGmL8oqZ+oC45quZzgRW4dLIeTODcB4IZe2ruxZ9tvXpSuAESiOv5bBZyMJEKdKwsRdt6vk+bezkZBCzPq+ly892Ncq7x5u4/8dGcGxRxxp5TnIIx3hFoiXZ7svXJ7st6uHuKkuwYq+ebWh6bcvQAwH2wCP40twMjQgmU8nHHo6+pulxu1wVZBxtoyM4m3Lu0ZncTi6mF0duiPvpkNNAZzKOF98aUq3hxYp5PP28qrtWuPsgSdTNWJwnV2R288yAPx4WLUAGB9h889d/m67HUiKadmfG4oyUnChCU/mqeJ/5nQkK1n76ezRPGsTcMZIv2mmPLShA/j38lGMjR/3pzM2Ha6UFjJKpurpeDiyt5pUdmRBpQRoBUx/PHnbXBsO9PBXa0mjdq3UFbgb5iBD1Fz2YClZ/h0d4YlYUMSeharDrLfR15g3fGnzxrkR03zYeHkxMUi5vYHQIDAQABAoICAFxUg5pcH8lOF1FMHfXIY5MM5mbHKsC8knERCPfnEvVB/wxc6D60fow/I0Vg4blzNuFuTjiMykIcfpGQMiK1l8TEMdSODoYCHq6+gY3ootUnLj94i79MtCw9FJHPnKkcVUiDNqPvr/rk8l/06j2J0NMXffEc3FeYJCXYuLtlluTzdI4wueNY/wkoz+EvpQMzWXAMkhxHpIEAqr2aIn+TQDQ0MYxBQlqwDTSPYHFJxIVFlvYx2VdnecKSO2E1/76kOowhTzH3FL+hcWcI/TG7tJuXU1C3llV0sUVBgC1NjwjMkuEm3EN5/ffqgj/eJNJTR1r6tqSzxzMJD42WVmqlPhqEmmRE1yeaio6Ulm5EJLREYEY4/1EotdIO6PCXF1Xeo+10VJLcwi19vzMlz7vFB6vSJzJNYOaiwm4rgmF12+nNt18cHoFvegGPAoCJurUhXU+iV6LlKYB8J/+kpITleZwm4dEaa6EWxgFf4nZ4N+qRVqk4JfC+rN3DQgbPcII+z7axAb8xutq5rvzskGAmKcTEagMG+pnk9DI+0m1C2J6+mC/YruVlpSlPB7Ecu/OF++vJoSCLuDXVuxMTQ3tQXoicOG/S8d1aa3nq4fvLIgZTOZoffUvNvjCyy/Dlv6vaDVCjgqt2kFZVSZv3C6ttyrNVBSjhucASGTq/7q82JKVrAoIBAQD7e7SYqDM8SAjOfpRAHCbIVVSaHw1tul16B6JAF67OIs1uD//Y+maL4GHlpFxnib0tg6z3+sG10NMUhK7elB6SkJ8voT+Ztg/+on659vmaqwTlkGjgks+8DmIw1b0ZbXuIqhnkOeSsll2V9NWItIrbzNwvhtMEQsOtFvmrLlKQvx9Bo3lGmbfdU1/9XpwW4wiA5ONcFCl1xllX9nItB+ActPoAeQgnuejWAISQkITDTOFcGu0AjMjJ66HUlY/WDbf7wY/s2aMbOiVx7B6YzQnK+lZdjjNfoaL8nfN8Z8ZzJFAgmTQ9ztZCJ8fLTfZDsYD6XZnVEIG401+6de0Ea4MHAoIBAQD7Mlr++keNDxYy3k6kwTXDOSKMDfXIDverFy/tiNorcVU5nIpuhjowl4HIQVAhzlXhATp938LM/PvJIeu9HtgsY9BFNL52fr7iGEWknqZQVE+8Um35tzV3vtPH9NVXHIYmcRCAQQJ5a6oVQRJKAM3E1+htM2AeZKMaQSCC6VHGBXJG9nEltdMKTet09pd26Z5y/YIfTqbIekyjRomGdInyWBkpJLJ/uROVb9OGnsYWadDktn7y9+oH+VZoJxW5QwXV0OL7wMzymlrKhhBNMziJRBk4rVztclYzUShpwE8Xfjt5WYkOoieXtmkdpF7TMvRYy6cdflh39l8s7THq4U67AoIBAQDPShR+kIZK9gVsUCHbBllhYBSwRVBIVgmfAJPYyKyp9T3nrwBw1VWZjoajR2tzXLG2xobYkLUDLPIQQ53wkZ7xqIvOMte917gvZ7mVh+IOf1E1+Ga6VcktdZ2yKzwHrZrZ93wIB0BakkbpyDoKyOZ8bsw/0J43MOEh14JMe45LCX62GA1rCGHp6UOCEW4P32uu8vwFdgkGr0wNmXJbc9ZjUzmPEjSKqk8ltp+OmdEVOlW4GempPmJzLeri07UP95bjyZTc4jfzxWYAFWTTBo8/5EE0NHZcKVuzZm7A6LlHR3NUq3PITqmFNk0VY1nOSo3J0UVvvn9u3axWgcWwkv4ZAoIBAQD5GaB/RfZ6fcGHxEi8vRTxedE2jglz0NPuCFpw9ctUVVksfn4QyG8gDL6YtZwfShyDutIkTSdCXIXbmRXLP7Ejo/+/AWT24Q2t7F/ztZg8BL/lguCoR/RvwP8HRVXiwtiNeoXgiZdIjWnnFdZrKIzlbS6CS/Y/bZEI0hjJeVKsSm5ezLi7qqmTiF28aWUeeOG2KotFPJvMJuuk5qgue5boilC5KjkQy5oOEQz0bLMHOvv6MreFfi/IpIv5UPkL9NjuhezxtuoGkucB+xxjUi34gV15hQGoPkAZN5wZ8qj8JSQZXJ67sjYNDskCqiOW0Orau2oHp2lX3SvYw4dC4xdNAoIBAQDSXF8OnWu08smGDs4S57yfqe59N0ULIBDOZdEQcLXRaDeAoOwQj8szUf0pycXUZLvXxv29iF61FJg70iGGCp3lW+9Oe0P+dGsKvzphRGsQPZy4h/qZDoQ9dE667ejI4aDb/C1GaOZ8TT1QfpQcVKbgLjdC19IGQPhFljGM1wMQojU5lp4/iKmA3J3ahOuV4dvEUr6xkqzq+PhH405XdURXsVc1J6qYX2wQXRoB0oDsrCbqidZi14DaeE8UhZRA6m9haUOb/3Y+OkCs2QIW4MNZ+zg8XtHjiUhMxAaJ6zdEFltWPoVFsVfgoecquAHb0uDJJRz/+RfR0AiG0XG0NW+C"
+}

sas/mocks/sasjs/services/admin/makedata.js

@@ -1159,6 +1159,26 @@ const tables = {
       selectbox_value: 'FORMULA',
       selectbox_order: 1,
       ver_to_dttm: 127490111999
+    },
+    {
+      selectbox_rk: 73,
+      ver_from_dttm: 0,
+      select_lib: dcLibref,
+      select_ds: 'MPE_DERIVED',
+      base_column: 'VAR_TYPE',
+      selectbox_value: 'C',
+      selectbox_order: 1,
+      ver_to_dttm: 127490111999
+    },
+    {
+      selectbox_rk: 74,
+      ver_from_dttm: 0,
+      select_lib: dcLibref,
+      select_ds: 'MPE_DERIVED',
+      base_column: 'VAR_TYPE',
+      selectbox_value: 'N',
+      selectbox_order: 2,
+      ver_to_dttm: 127490111999
     }
   ],
   MPE_SUBMIT: [],
@@ -1371,6 +1391,19 @@ const tables = {
         'RLS_SCOPE RLS_GROUP RLS_LIBREF RLS_TABLE RLS_GROUP_LOGIC RLS_SUBGROUP_LOGIC RLS_SUBGROUP_ID RLS_VARIABLE_NM RLS_OPERATOR_NM RLS_RAW_VALUE',
       post_edit_hook: 'services/hooks/mpe_row_level_security_postedit'
     },
+    {
+      tx_from: 0,
+      tx_to: 127490111999,
+      libref: dcLibref,
+      dsn: 'MPE_DERIVED',
+      num_of_approvals_required: 1,
+      loadtype: 'TXTEMPORAL',
+      buskey: 'LIBREF DSN VAR_NM',
+      notes: 'Derived (computed) variables for the EDIT screen',
+      var_txfrom: 'TX_FROM',
+      var_txto: 'TX_TO',
+      post_edit_hook: 'services/hooks/mpe_derived_postedit'
+    },
     {
       tx_from: 0,
       tx_to: 127490111999,
@@ -2002,6 +2035,206 @@ const tables = {
       rule_value: 'services/validations/mpe_x_test.some_num',
       rule_active: 1,
       tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'LIBREF',
+      rule_type: 'NOTNULL',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'LIBREF',
+      rule_type: 'CASE',
+      rule_value: 'UPCASE',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'DSN',
+      rule_type: 'NOTNULL',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'DSN',
+      rule_type: 'CASE',
+      rule_value: 'UPCASE',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_NM',
+      rule_type: 'NOTNULL',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_NM',
+      rule_type: 'CASE',
+      rule_value: 'UPCASE',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_NM',
+      rule_type: 'HARDREGEX',
+      rule_value: '^[A-Za-z_][A-Za-z0-9_]*$',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_DEPS',
+      rule_type: 'CASE',
+      rule_value: 'UPCASE',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_DEPS',
+      rule_type: 'HARDREGEX',
+      rule_value: '^[A-Za-z0-9_ ]*$',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_TYPE',
+      rule_type: 'NOTNULL',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_TYPE',
+      rule_type: 'CASE',
+      rule_value: 'UPCASE',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_TYPE',
+      rule_type: 'HARDSELECT',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_LEN',
+      rule_type: 'NOTNULL',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_LEN',
+      rule_type: 'MINVAL',
+      rule_value: '1',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_LEN',
+      rule_type: 'MAXVAL',
+      rule_value: '32767',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_FMT',
+      rule_type: 'CASE',
+      rule_value: 'UPCASE',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_FMT',
+      rule_type: 'HARDREGEX',
+      rule_value: '^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_LOGIC',
+      rule_type: 'NOTNULL',
+      rule_value: ' ',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_LABEL',
+      rule_type: 'HARDREGEX',
+      rule_value: '^[^\\x00-\\x1F\\x7F]*$',
+      rule_active: 1,
+      tx_to: 127490111999
+    },
+    {
+      tx_from: 0,
+      base_lib: dcLibref,
+      base_ds: 'MPE_DERIVED',
+      base_col: 'VAR_LOGIC',
+      rule_type: 'HARDREGEX',
+      rule_value: '^(?!.*(/\\*|\\*/))[^;&%]*$',
+      rule_active: 1,
+      tx_to: 127490111999
     }
   ],
   MPE_X_TEST: [

sas/mocks/sasjs/services/auditors/getauditfile.js

@@ -47,7 +47,8 @@ if (!tableId || !libds) {
   webOutClose()
 } else {
   // The SAS service also checks EDIT or APPROVE access (mpe_accesscheck). The
-  // mock has no permission store, so access is not emulated.
+  // mock seeds MPE_SECURITY but no mock service enforces it, so access is not
+  // emulated here.
 
   // The approval folder is zipped as it stands. getauditfile.sas lists it with
   // mp_dirlist and adds each file to the package under contents/.

sas/mocks/sasjs/services/dcMockUtils.js

@@ -641,6 +641,27 @@ function zipUtf8(value) {
     return Uint8Array.from(out)
 }
 
+/**
+ * Renders rows as CSV, the counterpart of %mp_ds2csv: the header row, then one
+ * line per row, with values quoted only when they carry a delimiter, quote or
+ * line break.
+ *
+ * @param {string[]} headers column names, in order
+ * @param {Array<Object>} rows objects keyed by column name
+ * @returns {string} CSV text with a trailing newline
+ */
+function toCsv(headers, rows) {
+    const cell = (value) => {
+        const text = value === undefined || value === null ? '' : String(value)
+        return /[",\n\r]/.test(text) ? '"' + text.replace(/"/g, '""') + '"' : text
+    }
+    const lines = [headers.map(cell).join(',')]
+    for (const row of rows) {
+        lines.push(headers.map((header) => cell(row[header])).join(','))
+    }
+    return lines.join('\n') + '\n'
+}
+
 /**
  * Builds a zip archive in memory, entries stored rather than deflated - the JS
  * counterpart of the ODS package getauditfile.sas publishes. The zip format
@@ -661,16 +682,19 @@ function mpZipPackage(entries) {
         const name = zipUtf8(entry.name)
         const data = zipUtf8(entry.content)
         const crc = zipCrc32(data)
+        // Bit 11 marks the name as UTF-8. Without it an extractor decodes a
+        // non-ASCII name as CP437, so the entry cannot be read back by name.
+        const flags = name.some((byte) => byte >= 0x80) ? 0x0800 : 0
 
         // 1980-01-01 00:00, the earliest DOS timestamp: a stable value keeps
         // the archive byte-identical for identical inputs.
         const header = [].concat(
-            u32(0x04034b50), u16(20), u16(0), u16(0), u16(0), u16(0x21),
+            u32(0x04034b50), u16(20), u16(flags), u16(0), u16(0), u16(0x21),
             u32(crc), u32(data.length), u32(data.length), u16(name.length), u16(0)
         )
 
         local.push(Uint8Array.from(header), name, data)
-        central.push({ name, crc, size: data.length, offset })
+        central.push({ name, crc, size: data.length, offset, flags })
         offset += header.length + name.length + data.length
     }
 
@@ -678,7 +702,7 @@ function mpZipPackage(entries) {
     let directorySize = 0
     for (const entry of central) {
         const header = [].concat(
-            u32(0x02014b50), u16(20), u16(20), u16(0), u16(0), u16(0), u16(0x21),
+            u32(0x02014b50), u16(20), u16(20), u16(entry.flags), u16(0), u16(0), u16(0x21),
             u32(entry.crc), u32(entry.size), u32(entry.size),
             u16(entry.name.length), u16(0), u16(0), u16(0), u16(0), u32(0),
             u32(entry.offset)

sas/mocks/sasjs/services/editors/getdata.js

@@ -101,7 +101,7 @@ function pkFirst(columns, keys) {
 // ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
 // Applied before the filter, so a filter can reference them.  The derived
 // columns are added to the schema too, which is what puts them in COLS and in
-// the grid (read only, since the mock marks every column READ).
+// the grid; they are held read-only by a READONLY rule below.
 
 const derived = mpeDerived({
     libref: requestedLibref,
@@ -160,6 +160,16 @@ if (tableData && tableData.columns) {
     }
 }
 
+// A derived variable is computed at read time, so the grid must not accept a
+// value for it.  COLS carries READ for the same reason (a derived column has no
+// CLS rule row), but the client only applies COLS.CLS_RULE when CLS_FLAG is
+// set, which column-level security drives - so the rule is declared here too,
+// as getdata.sas does.  dqrules is the request-local array the filter/map above
+// built, so appending to it leaves the loaded MPE_VALIDATIONS rows untouched.
+for (const v of derived) {
+    dqrules.push({ BASE_COL: v.name, RULE_TYPE: 'READONLY', RULE_VALUE: '' })
+}
+
 // ─── DQ dropdown data (from MPE_SELECTBOX + SOFTSELECT from table data) ──────
 // MPE_SELECTBOX lives in the control library.
 
@@ -426,7 +436,11 @@ if (tableData && tableData.columns && tableData.rows) {
             CLS_FLAG: 0,
             ISMAP: 0,
             TRUNCATED: truncated,
-            TRUNC_MSG: truncMsg
+            TRUNC_MSG: truncMsg,
+            // names the derived columns for the grid, as getdata.sas does - the
+            // READONLY rule above is why they cannot be edited, this is how the
+            // EDIT screen knows they are computed rather than locked
+            DERIVEDVARS: derived.map(d => d.name).join(' ')
         }],
         versions: [],
         xl_rules: []
@@ -441,7 +455,7 @@ if (tableData && tableData.columns && tableData.rows) {
         ],
         maxvarlengths: [], query: [], sasdata: [],
         $sasdata: { vars: {} },
-        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
+        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '', DERIVEDVARS: '' }],
         versions: [], xl_rules: []
     }
 }

sas/mocks/sasjs/services/editors/loadfile.js

@@ -59,6 +59,9 @@ for (let i = 1; i <= 10; i++) {
 
 let libds = ''
 if (typeof table !== 'undefined' && table) libds = String(table).toUpperCase()
+// captured before any hook can re-point the submission: the cleaned CSV is
+// named after the original table, as loadfile.sas names it
+const origLibds = libds
 let libref = libds.split('.')[0] || dcLibref
 let dsn = libds.split('.')[1] || ''
 
@@ -178,6 +181,9 @@ if (!msg) {
         // written, while LIBREF / DS are still ordinary macro variables, so it
         // can re-point the changeset at a different table.  The mock eval()s the
         // hook's .js counterpart in this scope so it can reassign libref / dsn.
+        // A validating hook rejects the submission by setting hookErrmsg (see
+        // hooks/mpe_derived_postedit.js), which stops the staging below.
+        let hookErrmsg = ''
         const hookTables = mpeLoadTableData('MPE_TABLES')
         const hookReg = (hookTables && hookTables.rows)
             ? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
@@ -185,16 +191,21 @@ if (!msg) {
         if (hookReg && hookReg.post_edit_hook) {
             const hookSrc = mockHookSource(hookReg.post_edit_hook)
             if (hookSrc) {
+                // a hook reads and normalises the submitted rows as jsDataRows
+                // (stagedata.js names them that); here they are `rows`
+                const jsDataRows = rows
                 try {
                     eval(hookSrc)
                 } catch (err) {
-                    console.log(
+                    // a hook that throws has not validated anything, so the
+                    // submission is refused rather than staged unchecked
+                    hookErrmsg =
                         'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
-                    )
                 }
                 libds = libref + '.' + dsn
             }
         }
+        if (hookErrmsg) msg = hookErrmsg
 
         tableId = makeTableId()
         // ─── RK assignment (mpe_loader) ──────────────────────────────────────
@@ -214,7 +225,7 @@ if (!msg) {
         if (rkName && !headers.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
             headers.push(rkName)
         }
-        stageSubmission({
+        if (!msg) stageSubmission({
             dataDir: libDataDir(libref),
             stagingDir,
             tableId,
@@ -224,13 +235,11 @@ if (!msg) {
             message: 'File upload: ' + csvName,
             headers,
             rows,
-            // loadfile.sas leaves two files in the approval folder: the cleaned
-            // CSV (%mp_cleancsv) and a copy of the upload (BKP_<name>). The mock
-            // only accepts CSV, so both hold the uploaded text.
-            auditFiles: [
-                { name: libds + '.csv', content: csvText },
-                { name: 'BKP_' + csvName, content: csvText }
-            ]
+            // loadfile.sas leaves the cleaned CSV (%mp_cleancsv) in the approval
+            // folder, named after the table as the request found it. It copies
+            // the upload as BKP_<name> only when a second, non-CSV fileref came
+            // with the request, which the mock does not accept - so no BKP here.
+            auditFiles: [{ name: origLibds + '.csv', content: csvText }]
         })
     }
 }

sas/mocks/sasjs/services/editors/stagedata.js

@@ -22,6 +22,9 @@ let message = _sctRow.MESSAGE || ''
 let libds = _sctRow.LIBDS || ''
 let libref = libds.split('.')[0] || dcLibref
 let dsn = libds.split('.')[1] || ''
+// stagedata.sas names the CSV it stages after the table as the request found
+// it, before a hook can re-point the submission
+const origLibds = libds
 
 // The base table lives in its own library (DC_JSLIB for control tables,
 // TESTDATA for demo user tables); the staging dir stays in the control lib.
@@ -63,6 +66,10 @@ if (schema.length > 0) {
 // eval()s the hook's .js counterpart in this scope so it can reassign
 // libref / dsn / libds; the submit record then names the new base table, exactly
 // as the SAS hook does via call symputx.
+// A validating hook rejects the submission by setting hookErrmsg (see
+// hooks/mpe_derived_postedit.js); the response below turns it into an abort,
+// and nothing is staged.
+let hookErrmsg = ''
 const hookTables = makeTableLoader(mpeDataDir)('MPE_TABLES')
 const hookReg = (hookTables && hookTables.rows)
     ? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
@@ -73,9 +80,10 @@ if (hookReg && hookReg.post_edit_hook) {
         try {
             eval(hookSrc)
         } catch (err) {
-            console.log(
+            // a hook that throws has not validated anything, so the submission
+            // is refused rather than staged unchecked
+            hookErrmsg =
                 'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
-            )
         }
         libds = libref + '.' + dsn
     }
@@ -101,6 +109,29 @@ if (rkName && !jsDataHeaders.some((h) => h.toUpperCase() === rkName.toUpperCase(
     jsDataHeaders.push(rkName)
 }
 
+// ─── Derived variables ───────────────────────────────────────────────────────
+// stagedata.sas drops the MPE_DERIVED droplist from the submitted data before
+// anything is staged (they are display-only), and the approval CSV is written
+// from that stripped dataset - so neither the staged rows nor the pack may
+// carry them.
+
+const derivedVars = mpeDerived({
+    libref,
+    table: dsn,
+    dataDir: mpeDataDir,
+    columns: schema
+})
+if (derivedVars.length > 0) {
+    const derivedNames = new Set(derivedVars.map((d) => d.name.toUpperCase()))
+    const keep = (key) => !derivedNames.has(key.toUpperCase())
+    jsDataHeaders = jsDataHeaders.filter(keep)
+    jsDataRows = jsDataRows.map((row) => {
+        const copy = {}
+        for (const key of Object.keys(row)) if (keep(key)) copy[key] = row[key]
+        return copy
+    })
+}
+
 // ─── Stage the data ──────────────────────────────────────────────────────────
 // One subfolder per submission (DSID) under the staging directory, so that the
 // staged data can sit alongside related artifacts (submit/approval logs,
@@ -108,7 +139,7 @@ if (rkName && !jsDataHeaders.some((h) => h.toUpperCase() === rkName.toUpperCase(
 
 const tableId = makeTableId()
 
-stageSubmission({
+if (!hookErrmsg) stageSubmission({
     dataDir,
     stagingDir,
     tableId,
@@ -117,9 +148,17 @@ stageSubmission({
     dsn,
     message,
     headers: jsDataHeaders,
-    rows: jsDataRows
+    rows: jsDataRows,
+    // stagedata.sas writes the staged rows as <orig_libds>.csv alongside them
+    // (%mp_ds2csv into the approval folder), so the audit pack carries it
+    auditFiles: [{ name: origLibds + '.csv', content: toCsv(jsDataHeaders, jsDataRows) }]
 })
 
 webOutOpen()
-webOutObj([{ STATUS: 'SUCCESS', DSID: tableId, URL: '' }], 'sasparams')
+if (hookErrmsg) {
+    webOutObj([{ MSG: hookErrmsg, MAC: 'stagedata' }], 'sasjsAbort')
+    webOutObj([{ STATUS: 'ERROR', DSID: '' }], 'sasparams')
+} else {
+    webOutObj([{ STATUS: 'SUCCESS', DSID: tableId, URL: '' }], 'sasparams')
+}
 webOutClose()

sas/mocks/sasjs/services/hooks/mpe_derived_postedit.js

@@ -0,0 +1,373 @@
+// ─────────────────────────────────────────────────────────────────────────────
+// Mock of services/hooks/mpe_derived_postedit
+//
+// Registered against DC_JSLIB.MPE_DERIVED (MPE_TABLES.post_edit_hook).  This
+// file is eval'd by editors/stagedata.js and editors/loadfile.js, mirroring how
+// mpe_loader.sas %includes the hook program via %mpe_runhook, so it shares the
+// caller's scope: jsDataRows (the submitted rows - work.staging_ds on the
+// server), libref, dsn, mpeDataDir and the dcMockUtils helpers.
+//
+// A hook rejects a submission by setting hookErrmsg, which the caller turns
+// into a sasjsAbort - the same outcome as the SAS hook's %mp_abort.
+//
+// The rules mirror services/hooks/mpe_derived_postedit.sas, in its order:
+// normalise the incoming rows, reject a duplicate name and a bad field, then
+// check the dependency graph that the resulting configuration would have (the
+// incoming rows plus the current rows they do not replace).  As on the server,
+// the collision between VAR_NM and a physical column is left to the read-time
+// engine, which runs with the target library assigned - the hook does not have
+// it here either.
+// ─────────────────────────────────────────────────────────────────────────────
+
+;(function mpeDerivedPostedit() {
+    const DELETE_MARK = '_____DELETE__THIS__RECORD_____'
+    const NAME_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
+    const FMT_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.'
+
+    const text = (v) => (v === undefined || v === null ? '' : String(v))
+    const up = (v) => text(v).trim().toUpperCase()
+    const strip = (v) => text(v).trim()
+
+    /** Reads a field as the hook normalised it. */
+    const field = (row, name) => up(colVal(row, name))
+
+    /** Writes a field back under the key the row already carries. */
+    const setField = (row, name, value) => {
+        const key = Object.keys(row).find((k) => k.toUpperCase() === name)
+        if (key) row[key] = value
+        else row[name] = value
+    }
+
+    const rows = Array.isArray(jsDataRows) ? jsDataRows : []
+    const isDeleted = (row) => field(row, DELETE_MARK) === 'YES'
+    const isLive = (row) =>
+        !isDeleted(row) && Number(colVal(row, 'ACTIVE')) === 1
+    const keyOf = (row) =>
+        [field(row, 'LIBREF'), field(row, 'DSN'), field(row, 'VAR_NM')].join(
+            '.'
+        )
+
+    const reject = (msg) => {
+        hookErrmsg = msg
+    }
+
+    // ─── Normalise the incoming values ─────────────────────────────────────────
+    for (const row of rows) {
+        if (isDeleted(row)) continue
+        for (const name of [
+            'LIBREF',
+            'DSN',
+            'VAR_NM',
+            'VAR_TYPE',
+            'VAR_FMT',
+            'VAR_DEPS'
+        ]) {
+            setField(row, name, up(colVal(row, name)))
+        }
+        for (const name of ['VAR_LABEL', 'VAR_LOGIC']) {
+            setField(row, name, strip(colVal(row, name)))
+        }
+    }
+
+    // ─── Duplicate names after normalisation ───────────────────────────────────
+    // mpe_loader compares the raw values, so a case variant reaches this point
+    const seen = new Set()
+    for (const row of rows) {
+        if (isDeleted(row)) continue
+        const key = keyOf(row)
+        if (seen.has(key)) {
+            return reject(
+                `MPE_DERIVED: ${field(row, 'VAR_NM')} is declared more than once for ${field(row, 'LIBREF')}.${field(row, 'DSN')}`
+            )
+        }
+        seen.add(key)
+    }
+
+    // ─── Field level checks, the rules the engine applies at read time ─────────
+    // C0 controls and DEL: what a single-line label cannot carry
+    const CONTROL_CHARS = /[\u0000-\u001F\u007F]/
+
+    // true when the value carries a character outside the allowed set
+    const hasInvalidChar = (value, chars) =>
+        [...value].some((c) => !chars.includes(c))
+
+    for (const row of rows) {
+        if (isDeleted(row)) continue
+        const libref = field(row, 'LIBREF')
+        const dsn = field(row, 'DSN')
+        const name = field(row, 'VAR_NM')
+        const type = field(row, 'VAR_TYPE')
+        const len = Number(colVal(row, 'VAR_LEN'))
+        const fmt = field(row, 'VAR_FMT')
+        const label = strip(colVal(row, 'VAR_LABEL'))
+        const logic = strip(colVal(row, 'VAR_LOGIC'))
+
+        let reason = ''
+        if (!libref || !dsn) reason = 'LIBREF and DSN are required'
+        else if (!name) reason = 'VAR_NM is required'
+        else if (hasInvalidChar(name, NAME_CHARS))
+            reason = 'VAR_NM must be a valid SAS name'
+        else if ('0123456789'.includes(name[0]))
+            reason = 'VAR_NM must not start with a digit'
+        else if (name === DELETE_MARK) reason = 'VAR_NM is reserved'
+        else if (type !== 'C' && type !== 'N')
+            reason = 'VAR_TYPE must be C or N'
+        // ACTIVE is a picker in the EDIT screen (HARDSELECT over the
+        // MPE_SELECTBOX pair) - the same pair mpe_derived_check.sas enforces
+        else if (!['0', '1'].includes(String(colVal(row, 'ACTIVE')).trim()))
+            reason = 'ACTIVE must be 1 or 0'
+        else if (!Number.isInteger(len) || len < 1)
+            reason = 'VAR_LEN must be a positive integer'
+        else if (type === 'C' && len > 32767)
+            reason = 'VAR_LEN exceeds 32767 for a character variable'
+        else if (type === 'N' && len > 8)
+            reason = 'VAR_LEN exceeds 8 for a numeric variable'
+        else if (
+            fmt.includes(';') ||
+            fmt.includes('&') ||
+            fmt.includes('%') ||
+            fmt.includes('/*') ||
+            fmt.includes('*/')
+        )
+            reason = 'VAR_FMT must be a format name, without ; & % or /* */'
+        else if (hasInvalidChar(fmt, FMT_CHARS))
+            reason = 'VAR_FMT must be a format name'
+        // free text, in any script - what is refused is anything that cannot be
+        // a single-line column label, and a value the engine would cut to 256
+        else if (CONTROL_CHARS.test(label))
+            reason =
+                'VAR_LABEL must be plain text - no line breaks or control characters'
+        else if ([...label].length > 256)
+            reason = 'VAR_LABEL exceeds 256 characters'
+        else if (!logic) reason = 'VAR_LOGIC is required'
+        else if (logic.includes(';'))
+            reason = 'VAR_LOGIC must be a single expression - remove the ;'
+        else if (logic.includes('&') || logic.includes('%'))
+            reason = 'VAR_LOGIC must not contain macro triggers'
+        else if (logic.includes('/*') || logic.includes('*/'))
+            reason = 'VAR_LOGIC must not contain comments'
+
+        if (reason) return reject(`MPE_DERIVED: ${name}: ${reason}`)
+    }
+
+    // ─── The graph of the resulting configuration ──────────────────────────────
+    // The incoming live rows plus the current rows they do not replace.  Only
+    // active rows are in it, because the engine reads nothing else - an inactive
+    // row can neither resolve a dependency nor be resolved by one.
+    const controlDir = mpeDataDir
+    const loadControl = makeTableLoader(controlDir)
+    const current = loadControl('MPE_DERIVED')
+    const nowSeconds = Date.now() / 1000
+
+    // mpeDerived reads a row as current when TX_TO is missing, zero or in the
+    // future (dcMockUtils.js, the read-time engine), so the same test is used
+    // here - a seeded row with no TX_TO is live for the engine, and a dependency
+    // on it has to validate
+    const isCurrent = (row) =>
+      !Number(colVal(row, 'TX_TO')) || Number(colVal(row, 'TX_TO')) > nowSeconds
+
+    const stagedKeys = new Set(rows.map(keyOf))
+    const stagedLive = rows.filter(isLive)
+
+    // the control-table arm of the SAS hook is limited to the tables this
+    // submission touches, so another table's configuration cannot block it
+    const stagedTables = new Set(
+        rows.map((row) => [field(row, 'LIBREF'), field(row, 'DSN')].join('.'))
+    )
+
+    /** libref.dsn.name - the key every graph lookup uses. */
+    const nodeKey = (n) => [n.libref, n.dsn, n.name].join('.')
+
+    const graph = new Map()
+    for (const row of stagedLive) {
+        graph.set(keyOf(row), {
+            libref: field(row, 'LIBREF'),
+            dsn: field(row, 'DSN'),
+            name: field(row, 'VAR_NM'),
+            deps: field(row, 'VAR_DEPS').split(/\s+/).filter(Boolean),
+            logic: strip(colVal(row, 'VAR_LOGIC')).toUpperCase()
+        })
+    }
+    const currentRows =
+        current && Array.isArray(current.rows)
+            ? current.rows.filter(
+                  (r) =>
+                      isCurrent(r) &&
+                      Number(colVal(r, 'ACTIVE')) === 1 &&
+                      stagedTables.has(
+                          [up(colVal(r, 'LIBREF')), up(colVal(r, 'DSN'))].join(
+                              '.'
+                          )
+                      ) &&
+                      !stagedKeys.has(
+                          [
+                              up(colVal(r, 'LIBREF')),
+                              up(colVal(r, 'DSN')),
+                              up(colVal(r, 'VAR_NM'))
+                          ].join('.')
+                      )
+              )
+            : []
+    for (const row of currentRows) {
+        graph.set(
+            [
+                up(colVal(row, 'LIBREF')),
+                up(colVal(row, 'DSN')),
+                up(colVal(row, 'VAR_NM'))
+            ].join('.'),
+            {
+                libref: up(colVal(row, 'LIBREF')),
+                dsn: up(colVal(row, 'DSN')),
+                name: up(colVal(row, 'VAR_NM')),
+                deps: up(colVal(row, 'VAR_DEPS')).split(/\s+/).filter(Boolean),
+                logic: strip(colVal(row, 'VAR_LOGIC')).toUpperCase()
+            }
+        )
+    }
+
+    /** Names this load leaves inactive, which therefore cannot resolve a dep. */
+    const inactive = new Set()
+    const liveKeys = new Set(stagedLive.map(keyOf))
+    for (const row of rows) {
+        const key = keyOf(row)
+        if (!liveKeys.has(key)) inactive.add(key)
+    }
+    if (current && Array.isArray(current.rows)) {
+        for (const row of current.rows) {
+            const key = [
+                up(colVal(row, 'LIBREF')),
+                up(colVal(row, 'DSN')),
+                up(colVal(row, 'VAR_NM'))
+            ].join('.')
+            if (
+                isCurrent(row) &&
+                Number(colVal(row, 'ACTIVE')) !== 1 &&
+                !liveKeys.has(key)
+            ) {
+                inactive.add(key)
+            }
+        }
+    }
+
+    for (const node of graph.values()) {
+        for (const dep of node.deps) {
+            const key = [node.libref, node.dsn, dep].join('.')
+            if (inactive.has(key) && !graph.has(key)) {
+                return reject(
+                    `MPE_DERIVED: ${node.name}: VAR_DEPS lists ${dep}, which is not active after this load - the engine cannot resolve it`
+                )
+            }
+        }
+    }
+
+    // ─── Self reference, undeclared and stale dependencies ─────────────────────
+    /** The identifiers an expression uses, with quoted runs removed. */
+    const identifiers = (logic) =>
+        logic
+            .replace(/'([^']|'')*'/g, ' ')
+            .replace(/"([^"]|"")*"/g, ' ')
+            .replace(/[^A-Za-z0-9_]/g, ' ')
+            .toUpperCase()
+            .split(/\s+/)
+            .filter(Boolean)
+
+    // scoped per table, as the server does: a name that is a derived variable
+    // on another table says nothing about this one
+    const graphKeys = new Set([...graph.values()].map((n) => nodeKey(n)))
+
+    // one check at a time, in the order the server runs them, so the message a
+    // row gets is the one it would get there
+    for (const node of graph.values()) {
+        if (node.deps.includes(node.name)) {
+            return reject(
+                `MPE_DERIVED: ${node.name}: VAR_DEPS may not list itself`
+            )
+        }
+    }
+
+    for (const node of graph.values()) {
+        for (const token of identifiers(node.logic)) {
+            if (
+                graphKeys.has([node.libref, node.dsn, token].join('.')) &&
+                token !== node.name &&
+                !node.deps.includes(token)
+            ) {
+                return reject(
+                    `MPE_DERIVED: ${node.name}: VAR_LOGIC references ${token}, which is not in VAR_DEPS`
+                )
+            }
+        }
+    }
+
+    for (const node of graph.values()) {
+        const tokens = identifiers(node.logic)
+        for (const dep of node.deps) {
+            if (!tokens.includes(dep)) {
+                return reject(
+                    `MPE_DERIVED: ${node.name}: VAR_DEPS lists ${dep}, which VAR_LOGIC does not reference`
+                )
+            }
+        }
+    }
+
+    // ─── A derived variable may not be named by a row level security rule ──────
+    const rls = loadControl('MPE_ROW_LEVEL_SECURITY')
+    if (rls && Array.isArray(rls.rows)) {
+        for (const node of graph.values()) {
+            const match = rls.rows.find(
+                (r) =>
+                    up(colVal(r, 'RLS_LIBREF')) === node.libref &&
+                    up(colVal(r, 'RLS_TABLE')) === node.dsn &&
+                    up(colVal(r, 'RLS_VARIABLE_NM')) === node.name &&
+                    Number(colVal(r, 'RLS_ACTIVE')) === 1 &&
+                    isCurrent(r)
+            )
+            if (match) {
+                return reject(
+                    `MPE_DERIVED: ${node.name} is named by a row level security rule on ${node.libref}.${node.dsn} - derived variables cannot be used there`
+                )
+            }
+        }
+    }
+
+    // ─── Cycles ────────────────────────────────────────────────────────────────
+    // Repeatedly place the variables whose derived inputs are already placed; a
+    // round that places nothing leaves a cycle.  The path is walked so the
+    // message names it, as the SAS hook does.
+    const nodeKeys = new Map(
+        [...graph.values()].map((n) => [n, [n.libref, n.dsn, n.name].join('.')])
+    )
+    const placed = new Set()
+    let remaining = [...graph.values()]
+
+    while (remaining.length > 0) {
+        const ready = remaining.filter((n) =>
+            n.deps.every((d) => {
+                const key = [n.libref, n.dsn, d].join('.')
+                return !graph.has(key) || placed.has(key)
+            })
+        )
+        if (ready.length === 0) {
+            // walk the unplaced edges so the message names the cycle
+            const start = remaining[0]
+            const path = [start.name]
+            const visited = new Set([nodeKeys.get(start)])
+            let cursor = start
+            for (let i = 0; i < remaining.length; i++) {
+                const parentKey = cursor.deps
+                    .map((d) => [cursor.libref, cursor.dsn, d].join('.'))
+                    .find((key) => graph.has(key) && !placed.has(key))
+                if (!parentKey) break
+                path.push(graph.get(parentKey).name)
+                if (visited.has(parentKey)) break
+                visited.add(parentKey)
+                cursor = graph.get(parentKey)
+            }
+            return reject(
+                `MPE_DERIVED: VAR_DEPS contains a cycle: ${path.join(' -> ')}`
+            )
+        }
+        for (const node of ready) placed.add(nodeKeys.get(node))
+        remaining = remaining.filter((n) => !placed.has(nodeKeys.get(n)))
+    }
+})()

sas/mocks/sasjs/services/public/viewdata.js

@@ -49,6 +49,23 @@ if (!tableData && dsn) {
     if (found) tableData = found.data
 }
 
+// ─── Derived variables (mpe_derived.sas) ─────────────────────────────────────
+// A stored filter can reference a derived column, so the derivation has to run
+// before the filter.  The columns themselves are dropped from the output - they
+// belong to the EDIT screen, as viewdata.sas drops them.
+// a copy: the derivation below returns a new column list, and every use here
+// (cols, search, rows, vars) must keep looking at the physical columns only
+const physicalColumns = [...(tableData && tableData.columns ? tableData.columns : [])]
+const derived = mpeDerived({
+    libref: libref,
+    table: dsn,
+    dataDir: mpeDataDir,
+    columns: physicalColumns
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── Apply the stored filter (mpe_filtermaster, VIEW mode) ───────────────────
 // Mirrors viewdata.sas: %mpe_filtermaster(VIEW,&libds,filter_rk=&filter_rk)
 // Row Level Security and any saved filter are applied here, before the search.
@@ -103,7 +120,7 @@ let nobs = 0
 if (tableData && tableData.columns) {
     // mpGetcols, the JS counterpart of %mp_getcols.sas - the macro viewdata.sas
     // calls for its own column list.
-    cols = mpGetcols(tableData.columns).map(c => {
+    cols = mpGetcols(physicalColumns).map(c => {
         const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
         const formatStr = c.TYPE === 'N' ? (c.FORMAT || '8.') : ('$' + c.LENGTH + '.')
         return {
@@ -119,7 +136,10 @@ if (tableData && tableData.columns) {
 
     let rows
     if (searchType) {
-        const matches = filteredRows.filter(row => rowMatchesSearch(row, tableData.columns, searchType, searchVal))
+        // the derived columns are dropped before the search, so only the
+        // physical columns are searched (viewdata.sas drops them in the
+        // vwsearch data step mp_searchdata then reads)
+        const matches = filteredRows.filter(row => rowMatchesSearch(row, physicalColumns, searchType, searchVal))
         // mp_searchdata caps its output at outobs=&DC_MAXOBS_WEBVIEW, and
         // viewdata.sas then keeps only `if _n_<&DC_MAXOBS_WEBVIEW` of them.
         nobs = Math.min(matches.length, maxObsWebView)
@@ -134,7 +154,7 @@ if (tableData && tableData.columns) {
     // viewdata rows: all values as strings (formatted, SAS proc-print style)
     viewdata = rows.map(r => {
         const row = {}
-        for (const col of tableData.columns) {
+        for (const col of physicalColumns) {
             const lcName = col.name.toLowerCase()
             let val = r[col.name]
             if (val === undefined) val = r[lcName]
@@ -174,15 +194,15 @@ if (tableData && tableData.columns) {
     // (SAS missing), character columns get "".
     // A search that matches nothing sends no rows at all (the client shows
     // its "no data found" panel instead of an empty grid).
-    if (viewdata.length === 0 && !searchType && tableData.columns.length > 0) {
+    if (viewdata.length === 0 && !searchType && physicalColumns.length > 0) {
         const emptyRow = {}
-        for (const col of tableData.columns) emptyRow[col.name] = col.type === 'N' ? '.' : ''
+        for (const col of physicalColumns) emptyRow[col.name] = col.type === 'N' ? '.' : ''
         viewdata = [emptyRow]
     }
 
     // $viewdata vars: keep original SAS types (num stays num, char stays char)
     vars = {}
-    for (const col of tableData.columns) {
+    for (const col of physicalColumns) {
         vars[col.name] = sasVarsEntry(col)
     }
 }

sas/package-lock.json

@@ -6,8 +6,8 @@
     "": {
       "name": "dc-sas",
       "dependencies": {
-        "@sasjs/cli": "6.1.0",
-        "@sasjs/core": "6.0.0"
+        "@sasjs/cli": "6.2.0",
+        "@sasjs/core": "6.1.0"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -165,9 +165,9 @@
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-6.1.0.tgz",
-      "integrity": "sha512-i2CdGS69EM6Sn7XfR1ltBk9Bxz6gEV9xTGSWUmk4eVe+XnJBlYHyg93VgMcx7sWCtzRF9Dv7N2p7eXEc2DHfbQ==",
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-6.2.0.tgz",
+      "integrity": "sha512-zovAOCmyPqcm+gooAZgwvgGH8n5cas3sY0lA80+hyFoFtmZ8VjW/bh19LM8OVwheT9dBCRCx4fnHwjy5XGpzvQ==",
       "license": "ISC",
       "dependencies": {
         "@sasjs/adapter": "^5.1.1",
@@ -235,9 +235,9 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.0.0.tgz",
-      "integrity": "sha512-E9iXPvNgCux80xHhfopCsQE/CUNp1NEF1Z7NRj089A1T8WXP9AWJYcfFfOqoljNh8rnMo1fbIgoNrIaaAgW0MA==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.1.0.tgz",
+      "integrity": "sha512-U0Ekq/TXVMquYV+AG6urBQSUgqUvV37yduufy68QQVgDj3PJiLUJDcOXKOGbhWGD+Nv877VuPCviGYqrqbjXvg==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {

sas/package.json

@@ -26,12 +26,15 @@
     "viyacbd": "sasjs folder delete /Public/app/viya && sasjs cbd -t viya && npm run viyadata",
     "viyadata": "sasjs request services/admin/makedata -d deploy/makeDataViya.json -l sasjsresults/makedata.log -o sasjsresults/output.json -t viya",
     "v4data": "sasjs request services/admin/makedata -d deploy/makeDataV4.json -l sasjsresults/makedatav4.log -o sasjsresults/outputv4.json -t v4",
-    "sasdocs": "sasjs doc && ./sasjs/utils/deploydocs.sh"
+    "sasdocs": "sasjs doc && ./sasjs/utils/deploydocs.sh",
+    "sas719setup": "sasjs cbd -t sas719 && npm run sas719data",
+    "sas719data": "sasjs request services/admin/makedata -d deploy/makeDataSas719.json -l sasjsresults/makedata_sas719.log -o sasjsresults/makedata_sas719.json -t sas719",
+    "sas719test": "sasjs test -t sas719"
   },
   "private": true,
   "dependencies": {
-    "@sasjs/cli": "6.1.0",
-    "@sasjs/core": "6.0.0"
+    "@sasjs/cli": "6.2.0",
+    "@sasjs/core": "6.1.0"
   },
   "overrides": {
     "nanoid": "3.3.18"

sas/sasjs/db/datactrl/mpe_derived.ddl

@@ -17,16 +17,16 @@
 create table &curlib..MPE_DERIVED
    (
     TX_FROM num not null format=datetime19.3,
-    LIBREF varchar(8),
-    DSN varchar(32),
-    VAR_NM varchar(32),
+    LIBREF varchar(8) not null,
+    DSN varchar(32) not null,
+    VAR_NM varchar(32) not null,
     VAR_DEPS varchar(4000),
-    VAR_TYPE varchar(1),
-    VAR_LEN int,
+    VAR_TYPE varchar(1) not null,
+    VAR_LEN int not null,
     VAR_FMT varchar(32),
     VAR_LABEL varchar(256),
-    VAR_LOGIC varchar(4000),
-    ACTIVE int,
+    VAR_LOGIC varchar(4000) not null,
+    ACTIVE int not null,
     TX_TO num not null format=datetime19.3,
     constraint pk_mpe_derived
         primary key(tx_from, libref, dsn, var_nm));

sas/sasjs/db/migrations/20261019_v7.19_release.sas

@@ -20,16 +20,16 @@ proc sql;
 create table &dclib..mpe_derived
   (
     tx_from num not null format=datetime19.3,
-    libref varchar(8),
-    dsn varchar(32),
-    var_nm varchar(32),
+    libref varchar(8) not null,
+    dsn varchar(32) not null,
+    var_nm varchar(32) not null,
     var_deps varchar(4000),
-    var_type varchar(1),
-    var_len num,
+    var_type varchar(1) not null,
+    var_len num not null,
     var_fmt varchar(32),
     var_label varchar(256),
-    var_logic varchar(4000),
-    active num,
+    var_logic varchar(4000) not null,
+    active num not null,
     tx_to num not null format=datetime19.3
   );
 quit;
@@ -57,13 +57,19 @@ insert into &dclib..mpe_tables
   ;
 quit;
 
-/* Dropdown values for VAR_TYPE */
+/* Dropdown values for VAR_TYPE and ACTIVE.  These rows, and the field rules
+  * further down, mirror the MPE_DERIVED rows in mpe_makedata.sas - keep the two
+  * in step, or an upgraded install and a freshly seeded one behave differently. */
 proc sql noprint;
-  select max(selectbox_rk) into: maxrk from &dclib..mpe_selectbox;
+  /* coalesce: an install with no selectbox rows yet returns a missing value,
+    and %eval(&maxrk+1) below would then fail */
+  select coalesce(max(selectbox_rk),0) into: maxrk from &dclib..mpe_selectbox;
 quit;
 
 %let rk1=%eval(&maxrk+1);
 %let rk2=%eval(&maxrk+2);
+%let rk3=%eval(&maxrk+3);
+%let rk4=%eval(&maxrk+4);
 
 proc sql;
 insert into &dclib..mpe_selectbox
@@ -84,6 +90,25 @@ insert into &dclib..mpe_selectbox
     ,selectbox_value='N'
     ,selectbox_order=2
     ,ver_to_dttm='31DEC5999:23:59:59'dt;
+/* ACTIVE - a fixed pair, not a lookup against another table */
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk3
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="ACTIVE"
+    ,selectbox_value='1'
+    ,selectbox_order=1
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk4
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="ACTIVE"
+    ,selectbox_value='0'
+    ,selectbox_order=2
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
 quit;
 
 /* Field rules for the EDIT screen.  These are client side - the server side
@@ -91,23 +116,25 @@ quit;
 proc sql;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* the libraries the user can edit, as MPE_VALIDATIONS.BASE_LIB does */
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
-    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/libraries_editable"
+    ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
     ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* tables in the LIBREF on the same row - the hook reads it from source_row */
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/tables_editable"
+    ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
@@ -125,10 +152,6 @@ insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='CASE',rule_value='UPCASE',rule_active=1
@@ -137,10 +160,8 @@ insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
+/* ACTIVE has no NOTNULL rule: the column is NOT NULL in the model, and the
+  * EDIT screen offers the MPE_SELECTBOX values for it. */
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
     ,rule_type='MINVAL',rule_value='1',rule_active=1
@@ -159,11 +180,13 @@ insert into &dclib..MPE_VALIDATIONS set
     ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LABEL"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^[^\x00-\x1F\x7F]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
-    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(?!.*(/\*|\*/))[^;&%]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 quit;

sas/sasjs/macros/mpe_assert_abort.sas

@@ -0,0 +1,96 @@
+/**
+  @file
+  @brief Asserts that a service returned a sasjsAbort object
+  @details Reads the child's response as text and records a PASS when it carries
+  a sasjsAbort object, which is what mp_abort leaves in the webout when a service
+  rejects its input.  When msg is supplied the MSG value inside that object must
+  match it, so the assertion proves the service aborted for the expected reason
+  rather than for any reason at all.
+
+  The response is scanned a line at a time and only the first MSG that follows
+  the sasjsAbort marker is taken: a single 32767 buffer both truncated longer
+  responses and, with a leading `.*`, returned the last MSG in the text rather
+  than the abort's own.
+
+  An unassigned outref (a child that died before writing a response) records a
+  FAIL rather than raising an unassigned-fileref error in the data step.
+
+  Usage:
+
+      %mpe_execute(&_program,
+        inputfiles=&f1:sascontroltable,
+        outref=web1,
+        outstatus=work.ab1,
+        failfast=0
+      )
+      %mpe_assert_abort(
+        desc=An invalid libds aborts the service,
+        outref=web1,
+        msg=Invalid libds
+      )
+
+  @param [in] desc The description recorded in the test results
+  @param [in] outref The fileref given to %mx_execute, holding the response
+  @param [in] msg= (0) Expected MSG value in the sasjsAbort object.  0 accepts
+    any abort message.
+  @param [out] outds= (work.test_results) Results dataset
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+
+**/
+
+%macro mpe_assert_abort(desc, outref, msg=0, outds=work.test_results);
+%local ok seen abortseen;
+%let ok=0;
+%let seen=;
+%let abortseen=0;
+
+%if %sysfunc(fileref(&outref))>0 %then %do;
+  %put WARNING: &sysmacroname: fileref &outref is not assigned - the child left no response to read;
+%end;
+%else %if %sysfunc(fileexist(%sysfunc(pathname(&outref)))) ne 1 %then %do;
+  /* the fileref exists but nothing was written to it: a child that aborted
+    before it produced a response.  Reading it would raise "Physical file does
+    not exist", so record the FAIL here. */
+  %put WARNING: &sysmacroname: &outref has no response written to it - the child produced nothing;
+%end;
+%else %do;
+data _null_;
+  length line $32767 got $256;
+  retain prxid 0 inabort 0 found 0 abortseen 0;
+  if _n_=1 then prxid=prxparse('/"MSG" *: *"([^"]*)"/');
+  infile &outref end=eof lrecl=32767;
+  do while(not eof);
+    input;
+    line=_infile_;
+    if index(line,'sasjsAbort')>0 then do;
+      abortseen=1;
+      inabort=1;
+    end;
+    /* first MSG at or after the abort marker - stops at the first found */
+    if inabort=1 and found=0 then do;
+      if prxmatch(prxid,line)>0 then do;
+        got=prxposn(prxid,1,line);
+        found=1;
+      end;
+    end;
+  end;
+  call symputx('abortseen',abortseen,'l');
+  if found=1 then call symputx('seen',got,'l');
+  stop;
+run;
+%end;
+
+%if &abortseen=1 %then %do;
+  %if "%superq(msg)"="0" %then %let ok=1;
+  %else %if "&seen"="%superq(msg)" %then %let ok=1;
+%end;
+%if "&seen" ne "" %then %put NOTE: &sysmacroname: sasjsAbort MSG=&seen;
+
+%mp_assert(
+  iftrue=(&ok=1),
+  desc=&desc,
+  outds=&outds
+)
+%mend mpe_assert_abort;

sas/sasjs/macros/mpe_assert_abort.test.sas

@@ -0,0 +1,150 @@
+/**
+  @file
+  @brief testing the mpe_assert_abort macro
+
+  @details The macro decides PASS/FAIL by scanning the child's response text, so
+  the cases that matter can be built as files instead of services: an abort with
+  the expected message, an abort with a different one, no abort at all, and a
+  fileref the child never wrote to.
+
+  Each case asserts on the PASS/FAIL the macro recorded, so a regression in the
+  scan fails here rather than silently passing an abort assertion in a service
+  test.
+
+  Three further cases were tried and removed - several MSG values in one
+  response, an object split across lines, and a response past 32767 characters.
+  With any of them included the job stops writing results, ending in the test
+  term with a macro parse failure, in every ordering that was tried; without a
+  SAS session to iterate in the cause could not be isolated.  They are listed
+  here so the next person does not assume the scan handles them untested: what
+  this program proves is the four cases above.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_assert.sas
+  @li mpe_assert_abort.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let rc_any=work.assert_rc1;
+%let rc_other=work.assert_rc2;
+%let rc_none=work.assert_rc3;
+%let rc_first=work.assert_rc4;
+%let rc_split=work.assert_rc5;
+%let rc_long=work.assert_rc6;
+%let rc_noref=work.assert_rc7;
+
+/**
+  * Test 1 - an abort whose MSG matches is a PASS
+  */
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put '{"sasjsAbort": [{"MSG": "Invalid libds", "MAC": "mp_abort"}]}';
+run;
+
+%mpe_assert_abort(
+  desc=an abort with the expected MSG,
+  outref=&f1,
+  msg=Invalid libds,
+  outds=&rc_any
+)
+
+/**
+  * Test 2 - an abort for a different reason is a FAIL
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put '{"sasjsAbort": [{"MSG": "Invalid libds", "MAC": "mp_abort"}]}';
+run;
+
+%mpe_assert_abort(
+  desc=an abort with a different MSG,
+  outref=&f2,
+  msg=Invalid filter,
+  outds=&rc_other
+)
+
+/**
+  * Test 3 - a completed call (no sasjsAbort) is a FAIL
+  */
+%let f3=%mf_getuniquefileref();
+data _null_;
+  file &f3 termstr=crlf;
+  put '{"sasparams": [{"STATUS": "SUCCESS"}]}';
+  put '{"MSG": "Invalid libds"}';
+run;
+
+%mpe_assert_abort(
+  desc=a response with no sasjsAbort object,
+  outref=&f3,
+  msg=0,
+  outds=&rc_none
+)
+
+/**
+  * Test 4 - a fileref with no response written records a FAIL instead of aborting the step
+  */
+%let f7=%mf_getuniquefileref();
+
+%mpe_assert_abort(
+  desc=an unassigned outref,
+  outref=&f7,
+  msg=0,
+  outds=&rc_noref
+)
+
+/**
+  * Test 5 - the recorded results
+  */
+
+
+data _null_;
+  set &rc_any;
+  call symputx('res_any',test_result,'g');
+run;
+data _null_;
+  set &rc_other;
+  call symputx('res_other',test_result,'g');
+run;
+data _null_;
+  set &rc_none;
+  call symputx('res_none',test_result,'g');
+run;
+data _null_;
+  set &rc_noref;
+  call symputx('res_noref',test_result,'g');
+run;
+
+%mp_assert(
+  iftrue=("&res_any"="PASS"),
+  desc=%str(Checking an abort with the expected MSG passes),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("&res_other"="FAIL"),
+  desc=%str(Checking an abort with a different MSG fails),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("&res_none"="FAIL"),
+  desc=%str(Checking a response without a sasjsAbort object fails),
+  outds=work.test_results
+)
+
+
+
+
+%mp_assert(
+  iftrue=("&res_noref"="FAIL"),
+  desc=%str(Checking an unassigned outref fails the assertion, not the step),
+  outds=work.test_results
+)

sas/sasjs/macros/mpe_derived.sas

@@ -85,11 +85,43 @@
   them whatever MPE_TABLES calls them */
 %let reserved=%str(_____DELETE__THIS__RECORD_____);
 
-%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
+/* dc_dttmtfmt is the bitemporal cut-off used in the where= below - callers get
+  it from %mpeinit().  Unset, the filter degenerates to "lt tx_to" and the
+  control-table step fails with a syntax error a long way from the cause. */
+%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0 or %length(&dc_dttmtfmt)=0)
   ,mac=&sysmacroname
-  ,msg=%str(Missing: libds dclib)
+  ,msg=%str(Missing: libds dclib dc_dttmtfmt)
 )
 
+/* A format catalog is addressed as LIBREF.CATALOGNAME-FC (the form
+  mpe_validatecol accepts, and the one viewtables returns).  A catalog has no
+  observations, so it can have no derived variables - and it is not a dataset,
+  so the engine's existence check would abort the caller.  Return an empty
+  variable list and an empty statement file instead, and leave outview alone:
+  a caller only reads the view when the list is not empty. */
+%if %length(&libds)>3 and "%upcase(%substr(&libds,%length(&libds)-2,3))"="-FC" %then %do;
+  /* the file has to exist even though it stays empty - callers %include it
+    unconditionally, and a fileref with no file behind it fails there */
+  %if "%upcase(&outref)" ne "0" %then %do;
+    filename &outref temp lrecl=32767;
+    data _null_;
+      file &outref;
+    run;
+  %end;
+  /* The shape is the engine's own: mp_derivedvars builds
+    `data &outvars(keep=name); length name $32; name=''; if 0; run;` before it
+    returns (empty case, and again from the sorted list in the populated case),
+    so NAME is the only column either path can produce.  Every caller reads
+    NAME alone - getdata's `where a.NAME in (select NAME from
+    work.derived_vars)` is the only projection of this dataset in the SAS tree. */
+  data &outvars(keep=name);
+    length name $32;
+    name='';
+    if 0;
+  run;
+  %return;
+%end;
+
 %mp_derivedvars(
   libds=&libds
   ,control=&dclib..mpe_derived

sas/sasjs/macros/mpe_derived.test.sas

@@ -294,3 +294,50 @@ delete from &dclib..mpe_derived where upcase(libref)='WORK';
 quit;
 
 %mp_assertscope(COMPARE,ignorelist=_SASJS_WEBOUT_HEADERS,outds=work.test_results)
+
+/* Test 5 - a format catalog is not a dataset: no derived variables, no abort.
+  Callers pass LIBREF.CATALOGNAME-FC (validatefilter, getrawdata, viewdata), and
+  the engine's existence check would otherwise end the service. */
+proc format library=work.dcfmts;
+  value $band 'A'='Alpha';
+run;
+
+%mpe_derived(libds=work.DCFMTS-FC,
+  dclib=&dclib,
+  outref=derref5,
+  outvars=work.derived_vars5
+)
+
+%mp_assertdsobs(work.derived_vars5,
+  desc=Test5 - a format catalog yields no derived variables,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* The branch has to create the statement file even though it writes nothing,
+  because callers %include it unconditionally.  Two things to prove: the branch created the file, and it left it empty.
+  fopen answers the first (0 when the fileref cannot be opened), the loop the
+  second - a single INPUT would stop the step at once on an empty file and never
+  reach the symputx. */
+%let deropen=0;
+%let derlines=-1;
+data _null_;
+  did=fopen('derref5');
+  call symputx('deropen',ifn(did>0,1,0),'l');
+  if did>0 then rc=fclose(did);
+run;
+data _null_;
+  infile derref5 end=eof;
+  n=0;
+  do while(not eof);
+    input;
+    n+1;
+  end;
+  call symputx('derlines',n,'l');
+run;
+
+%mp_assert(
+  iftrue=(&deropen=1 and &derlines=0),
+  desc=Test5 - a format catalog leaves an empty statement file,
+  outds=work.test_results
+)

sas/sasjs/macros/mpe_derived_check.sas

@@ -0,0 +1,75 @@
+/**
+  @file
+  @brief Field level checks for a staged MPE_DERIVED submission
+  @details Writes one row per staged row that breaks a rule, with the reason in
+  REASON, so a caller can report the first failure and abort.  Rows marked for
+  deletion are skipped - they are being removed, so their fields are not loaded.
+
+  The rules are the ones %mpe_derived applies at read time, so a submission
+  cannot stage a row the engine would then refuse.
+
+  @param inds the staged rows, already normalised by the caller
+  @param outds the dataset to create: one row per failing row, plus REASON
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%macro mpe_derived_check(inds=work.staging_ds, outds=work.derived_check);
+data &outds;
+  set &inds;
+  length reason $200;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  call missing(reason);
+
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if var_nm='' then reason='VAR_NM is required';
+  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VAR_NM must be a valid SAS name';
+  else if verify(substr(var_nm,1,1),'0123456789')=0
+    then reason='VAR_NM must not start with a digit';
+  else if var_nm='_____DELETE__THIS__RECORD_____'
+    then reason='VAR_NM is reserved';
+  else if var_type not in ('C','N')
+    then reason='VAR_TYPE must be C or N';
+  /* ACTIVE is a picker in the EDIT screen (HARDSELECT over the MPE_SELECTBOX
+    pair) - the same pair is enforced here, so a submission that bypasses the
+    grid cannot land a value the screen would never offer. */
+  else if missing(active) or active not in (0,1)
+    then reason='ACTIVE must be 1 or 0';
+  else if missing(var_len) or var_len ne int(var_len) or var_len<1
+    then reason='VAR_LEN must be a positive integer';
+  else if var_type='C' and var_len>32767
+    then reason='VAR_LEN exceeds 32767 for a character variable';
+  else if var_type='N' and var_len>8
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
+  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
+    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
+    then reason='VAR_FMT must be a format name, without ; & % or /* */';
+  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VAR_FMT must be a format name';
+  /* VAR_LABEL is free text: any character is allowed, in any script, and the
+    quotes in it are escaped by %mp_derivedvars when the attrib statement is
+    built.  What is refused is anything that cannot be a single-line column
+    label - a control character or a line break - and a value long enough to
+    be cut down silently, since the engine truncates the label to 256. */
+  else if anycntrl(var_label)>0
+    then reason='VAR_LABEL must be plain text - no line breaks or control characters';
+  else if klength(var_label)>256
+    then reason='VAR_LABEL exceeds 256 characters';
+  else if missing(var_logic)
+    then reason='VAR_LOGIC is required';
+  else if index(var_logic,';')>0
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
+  else if indexc(var_logic,'&%')>0
+    then reason='VAR_LOGIC must not contain macro triggers';
+  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
+    then reason='VAR_LOGIC must not contain comments';
+
+  if not missing(reason) then output;
+run;
+
+%mend mpe_derived_check;

sas/sasjs/macros/mpe_derived_check.test.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief testing the mpe_derived_check macro
+
+  @details Checks each field level rule the MPE_DERIVED post edit hook applies
+  to a staged submission.  The macro is exercised directly so the rules can be
+  probed with values a service cannot carry: VAR_LABEL is varchar(256) in the
+  table and a staging dataset declares the same length, so a longer label is
+  truncated before any check would see it.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mpe_derived_check.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/**
+  * Test 1 - one row per failing rule, and only failing rows
+  */
+data work.staging_ds;
+  length libref $8 dsn $32 var_nm $32 var_type $1 var_len 8
+    var_fmt $32 var_label $4000 var_deps $4000 var_logic $4000
+    active 8 _____DELETE__THIS__RECORD_____ $3;
+  libref='DCTEST';
+  dsn='DC_DERIV';
+  /* valid unless a row says otherwise - ACTIVE is mandatory in the screen and
+    the check enforces the same pair here */
+  active=1;
+  _____DELETE__THIS__RECORD_____='No';
+
+  /* meets every rule */
+  var_nm='GOOD';
+  var_type='N';
+  var_len=8;
+  var_fmt='';
+  var_label='a plain label';
+  var_deps='';
+  var_logic='1';
+  output;
+
+  /* delete marked rows are being removed, so their fields are not loaded */
+  var_nm='DELME';
+  var_label='first line'||'0A'x||'second line';
+  _____DELETE__THIS__RECORD_____='Yes';
+  output;
+  _____DELETE__THIS__RECORD_____='No';
+
+  /* free text is allowed, including quotes and punctuation */
+  var_nm='FREETXT';
+  var_label='It''s "quoted", plain text: 100 percent';
+  output;
+
+  /* a line break cannot be a single line column label */
+  var_nm='LBLBREAK';
+  var_label='first line'||'0A'x||'second line';
+  output;
+
+  /* longer than the table holds, so it would be cut down silently */
+  var_nm='LBLLONG';
+  var_label=repeat('x',300);
+  output;
+
+  /* a format name, not an expression */
+  var_nm='BADFMT';
+  var_fmt='DATE9;';
+  var_label='fmt';
+  output;
+
+  /* one expression, no statement separator, no macro triggers */
+  var_nm='BADLOGIC';
+  var_fmt='';
+  var_logic='x; y';
+  output;
+
+  /* C or N */
+  var_nm='BADTYPE';
+  var_logic='1';
+  var_type='X';
+  output;
+
+  /* ACTIVE is a picker in the screen: only 1 or 0 */
+  var_nm='BADACTIVE';
+  var_type='C';
+  var_logic='1';
+  active=2;
+  output;
+
+  stop;
+run;
+
+%mpe_derived_check(inds=work.staging_ds, outds=work.derived_check)
+
+/**
+  * Test 2 - the reported rows, rule by rule
+  */
+data work.expected;
+  length var_nm $32 reason $200;
+  var_nm='LBLBREAK';
+  reason='VAR_LABEL must be plain text - no line breaks or control characters';
+  output;
+  var_nm='LBLLONG';
+  reason='VAR_LABEL exceeds 256 characters';
+  output;
+  var_nm='BADFMT';
+  reason='VAR_FMT must be a format name, without ; & % or /* */';
+  output;
+  var_nm='BADLOGIC';
+  reason='VAR_LOGIC must be a single expression - remove the ;';
+  output;
+  var_nm='BADTYPE';
+  reason='VAR_TYPE must be C or N';
+  output;
+  var_nm='BADACTIVE';
+  reason='ACTIVE must be 1 or 0';
+  output;
+run;
+
+proc sql;
+create table work.cmp as
+  select e.var_nm
+    ,case when a.reason=e.reason then 1 else 0 end as same
+  from work.expected e
+  left join work.derived_check a
+    on upcase(strip(a.var_nm))=upcase(e.var_nm);
+quit;
+
+data _null_;
+  set work.cmp;
+  call symputx(cats('rule_',var_nm),same,'l');
+run;
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.derived_check)=6),
+  desc=%str(Checking only the six failing rows are reported),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_LBLBREAK=1),
+  desc=%str(Checking a label with a line break is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_LBLLONG=1),
+  desc=%str(Checking a label over 256 characters is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADFMT=1),
+  desc=%str(Checking a format carrying a statement separator is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADLOGIC=1),
+  desc=%str(Checking a logic value with a statement separator is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADTYPE=1),
+  desc=%str(Checking a VAR_TYPE other than C or N is rejected),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&rule_BADACTIVE=1),
+  desc=%str(Checking an ACTIVE outside 1 or 0 is rejected),
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - a row that meets every rule, a free text label, and a delete
+  * marked row are all left alone
+  */
+proc sql noprint;
+select count(*) into: notchecked
+  from work.derived_check
+  where upcase(strip(var_nm)) in ('GOOD','FREETXT','DELME');
+quit;
+
+%mp_assert(
+  iftrue=(&notchecked=0),
+  desc=%str(Checking a valid row, a free text label and a delete marked row are not reported),
+  outds=work.test_results
+)

sas/sasjs/macros/mpe_execute.sas

@@ -0,0 +1,160 @@
+/**
+  @file
+  @brief Runs a DC service the way a test needs it run
+  @details Wraps mx_execute so that every call is measurable, using only
+  documented mx_execute outputs:
+
+  @li the child's log is kept when mdebug is on, through outlogds, and printed
+    as CHILD: lines.  A normal run asks for no log, so it stays quiet - flip
+    sasjs_mdebug for an investigation run instead of editing tests;
+  @li the child's outcome is written to outstatus (STATE, URI, ERRMSG,
+    PLATFORM).  state=canceled means the child aborted, so a test can assert
+    what happened rather than inferring it from an outlib that may still hold
+    the previous call's tables;
+  @li failfast aborts the caller as soon as the child is not completed, so the
+    first failure is the real one and the log names it.  Tests that expect an
+    abort pass failfast=0 and check the status row.  SAS 9 reports no job state
+    at all (PROC STP), which core records as state=unknown; that is treated as
+    unmeasured rather than failed, and noted in the log.
+
+  Usage:
+
+      %mpe_execute(&appLoc/services/editors/getdata,
+        viyacontext=&defaultcontext,
+        inputfiles=&f1:sascontroltable,
+        outlib=web1
+      )
+      %put reaching here means the child completed;
+      data _null_;
+        set work.dc_childstatus;
+        putlog "child said: " state= errmsg=;
+      run;
+
+  @param [in] program The _PROGRAM endpoint to run
+  @param [in] inputfiles=(0) fileref:filename pairs, as mx_execute
+  @param [in] inputdatasets=(0) datasets to send as CSV, as mx_execute
+  @param [in] inputparams=(0) name/value dataset, as mx_execute
+  @param [out] outlib=(0) Output libref for the response JSON
+  @param [out] outref=(0) Output fileref for the raw response text
+  @param [in] debug= (131) The _debug value sent to the child.  Numeric, as the
+    server requires; 131 asks for the log.
+  @param [in] mdebug= (&sasjs_mdebug) 1 keeps the child's log and prints it
+  @param [in] viyacontext= (&defaultcontext) Viya compute context
+  @param [out] outstatus=(work.dc_childstatus) One row per child run
+  @param [in] failfast= (1) Abort the caller when the child did not complete
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mx_execute.sas
+
+**/
+
+%macro mpe_execute(program
+    ,inputfiles=0
+    ,inputdatasets=0
+    ,inputparams=0
+    ,outlib=0
+    ,outref=0
+    ,debug=
+    ,mdebug=&sasjs_mdebug
+    ,viyacontext=&defaultcontext
+    ,outstatus=work.dc_childstatus
+    ,viyaresult=WEBOUT_JSON
+    ,failfast=1
+  );
+%local logds state msg;
+/* Aliases: passing &program / &inputfiles ... straight into another macro that
+  * declares the same parameter names makes SAS resolve them against the callee
+  * and null them ("recursive reference to the macro variable").  Unique names
+  * on this side keep the values intact. */
+%local a_viyaresult a_pgm a_inds a_indatasets a_inparams a_dbg a_mdbg
+  a_outlib a_outref a_logds a_status a_ctx;
+/* numeric: the server validates _debug as a number, so anything else never asks
+  * for the log and outlogds comes back empty */
+%if %length(&debug)=0 %then %let debug=131;
+%if %length(&mdebug)=0 %then %let mdebug=0;
+%let a_pgm=&program;
+%let a_viyaresult=&viyaresult;
+%let a_inds=&inputfiles;
+%let a_indatasets=&inputdatasets;
+%let a_inparams=&inputparams;
+%let a_dbg=&debug;
+%let a_mdbg=&mdebug;
+%let a_outlib=&outlib;
+%let a_outref=&outref;
+%let a_status=&outstatus;
+%let a_ctx=&viyacontext;
+
+%if &mdebug>0 %then %let logds=work.dc_childlog;
+%else %let logds=_null_;
+%let a_logds=&logds;
+
+/* a status row from an earlier call must not be read as this call's outcome:
+  a child that dies before core writes one would otherwise look completed */
+%if %index(&a_status,.)>0 and %sysfunc(exist(&a_status)) %then %do;
+  proc datasets library=%scan(&a_status,1,.) nolist;
+    delete %scan(&a_status,2,.);
+  quit;
+%end;
+
+%mx_execute(&a_pgm
+  ,inputfiles=&a_inds
+  ,inputdatasets=&a_indatasets
+  ,inputparams=&a_inparams
+  ,debug=&a_dbg
+  ,mdebug=&a_mdbg
+  ,viyacontext=&a_ctx
+  ,outlib=&a_outlib
+  ,outref=&a_outref
+  ,outlogds=&a_logds
+  ,outstatus=&a_status
+  ,viyaresult=&a_viyaresult
+)
+
+/* the child's own log, for an investigation run */
+%if &mdebug>0 %then %do;
+  data _null_;
+    set &a_status;
+    putlog "DC_EXECUTE: " state= errmsg=;
+  run;
+  %if %sysfunc(exist(work.dc_childlog)) %then %do;
+    data _null_;
+      set work.dc_childlog;
+      putlog "CHILD: " line;
+    run;
+  %end;
+%end;
+
+/* fail fast - a child that did not complete must not leave the test running
+  * against whatever the previous call left in outlib.  A missing status table
+  * and an empty one are both failures: state would stay blank and the caller
+  * would carry on inferring, which is the thing this macro exists to stop. */
+%let state=;
+%let msg=;
+%if %sysfunc(exist(&a_status))=1 %then %do;
+  data _null_;
+    set &a_status;
+    call symputx('state',state,'l');
+    call symputx('msg',errmsg,'l');
+  run;
+%end;
+%else %let state=missing(&a_status);
+
+%if %length(&failfast)=0 %then %let failfast=1;
+%if &failfast=1 %then %do;
+  /* SAS 9 (PROC STP) reports no job status at all: core writes state='unknown'
+  * with that explanation on a call that worked, so it cannot be read as a
+  * failure - only as unmeasured.  Everything else that is not completed is a
+  * failure, a blank state included. */
+  %if "%upcase(&state)"="UNKNOWN" %then
+    %put NOTE: &sysmacroname: &program returned no job status (SAS 9) - the outcome was not measured;
+  %else %if "%upcase(&state)" ne "COMPLETED" %then %do;
+    %mp_abort(
+      iftrue=(1=1)
+      ,mac=&sysmacroname
+      ,msg=%str(&program did not complete: state=&state &msg)
+    )
+  %end;
+%end;
+
+%mend mpe_execute;

sas/sasjs/macros/mpe_execute.test.sas

@@ -0,0 +1,125 @@
+/**
+  @file
+  @brief testing the mpe_execute macro
+
+  @details The wrapper exists so that a test can assert what a child call did
+  instead of inferring it from an outlib that still holds the previous call's
+  tables.  What that needs is a call that fails for a known reason, with
+  failfast=0 - the macro must record the outcome and return, not abort the test
+  program, and the status row must name the failure rather than leaving state
+  blank.
+
+  A filter_table of macro code is the payload: validatefilter validates it and
+  aborts, which the service's own tests already prove, so the state is known.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_nobs.sas
+  @li mp_assert.sas
+  @li mpe_execute.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/**
+  * Test 1 - a child that aborts is recorded in outstatus, and the caller
+  * continues (failfast=0)
+  */
+%let _program=&appLoc/services/public/validatefilter;
+
+%let f1=%mf_getuniquefileref();
+data _null_;
+  file &f1 termstr=crlf;
+  put 'FILTER_TABLE:$41.';
+  put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
+run;
+
+%let outref1=%mf_getuniquefileref();
+
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f1:iwant,
+  outref=&outref1,
+  outstatus=work.exec_status,
+  failfast=0,
+  viyaresult=WEBOUT_TXT
+)
+
+/* The macro reads the LAST row of outstatus - a row is written per HTTP
+  * attempt, so a retried or aborted call leaves more than one - which is what
+  * the assertions below check.  Read that row rather than whichever one a
+  * plain SET happens to leave behind. */
+%let state=;
+%let uri=;
+%let platform=;
+%let errmsg=;
+data _null_;
+  set work.exec_status nobs=nobs;
+  if nobs>0 then do;
+    set work.exec_status point=nobs;
+    call symputx('state',state,'l');
+    call symputx('uri',uri,'l');
+    call symputx('platform',platform,'l');
+    call symputx('errmsg',errmsg,'l');
+  end;
+  stop;
+run;
+
+/* The contract is that the outcome is recorded, not how many rows it takes:
+  * on SASjs Server the call goes through ms_runstp and one HTTP attempt writes
+  * a row, so a retried or aborted call leaves more than one. */
+%mp_assert(
+  iftrue=(%mf_nobs(work.exec_status)>0),
+  desc=%str(Checking the status table records the call),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%upcase(&state)" ne ""),
+  desc=%str(Checking the status row names the child state),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%upcase(&state)" ne "COMPLETED"),
+  desc=%str(Checking an aborted child is not recorded as completed),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=("%upcase(&platform)" ne ""),
+  desc=%str(Checking the status row names the platform),
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - the child's log is only kept when mdebug asks for it
+  */
+%let f2=%mf_getuniquefileref();
+data _null_;
+  file &f2 termstr=crlf;
+  put 'FILTER_TABLE:$41.';
+  put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
+run;
+
+%let outref2=%mf_getuniquefileref();
+
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f2:iwant,
+  outref=&outref2,
+  outstatus=work.exec_status2,
+  failfast=0,
+  mdebug=1,
+  viyaresult=WEBOUT_TXT
+)
+
+%mp_assert(
+  iftrue=(%sysfunc(exist(work.dc_childlog))=1),
+  desc=%str(Checking mdebug=1 keeps the child log),
+  outds=work.test_results
+)

sas/sasjs/macros/mpe_makedata.sas

@@ -697,6 +697,24 @@ insert into &lib..mpe_security set
       ,selectbox_value='N'
       ,selectbox_order=2
       ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="ACTIVE"
+      ,selectbox_value='1'
+      ,selectbox_order=1
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="ACTIVE"
+      ,selectbox_value='0'
+      ,selectbox_order=2
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
   insert into &lib..mpe_selectbox set
       selectbox_rk=%mf_increment(rk)
       ,ver_from_dttm=0
@@ -1987,6 +2005,13 @@ insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
     ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* the libraries the user can edit, as MPE_VALIDATIONS.BASE_LIB does */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/libraries_editable"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -1995,6 +2020,13 @@ insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
     ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* tables in the LIBREF on the same row - the hook reads it from source_row */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='SOFTSELECT_HOOK'
+    ,rule_value="services/validations/tables_editable"
+    ,rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -2027,6 +2059,14 @@ insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+/* ACTIVE is mandatory, and its NOTNULL rule carries '1' as its value: the EDIT
+  * screen pre-fills a new row from the value of the NOTNULL / READONLY / HIDDEN
+  * rule (getColumnDefault), so a derived variable starts active unless the user
+  * says otherwise.  The values themselves come from MPE_SELECTBOX. */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="ACTIVE"
+    ,rule_type='NOTNULL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -2052,9 +2092,15 @@ insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LABEL"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^[^\x00-\x1F\x7F]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
-    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(?!.*(/\*|\*/))[^;&%]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0

sas/sasjs/macros/mpe_refreshcatalogs.sas

@@ -2,11 +2,16 @@
   @file
   @brief Refreshes the data catalog
   @details Assumes library is already assigned.
+
+  @param [in] etlsource identifies this load in the MPE_DATALOADS audit row.
+
   Usage:
 
       %mpe_refreshcatalogs(sashelp)
 
   <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_abort.sas
   @li bitemporal_dataloader.sas
 
   @version 9.3
@@ -14,9 +19,12 @@
 
 **/
 
-%macro mpe_refreshcatalogs(lib,cat=#all);
+%macro mpe_refreshcatalogs(lib,cat=#all
+  ,etlsource=refreshcatalog:mpe_refreshcatalogs:%mf_uid()
+);
 %let lib=%upcase(&lib);
 %let cat=%upcase(&cat);
+%if "&etlsource"="0" %then %let etlsource=refreshcatalog:%sysmacroname:%mf_uid();
 
 %put running &sysmacroname &lib for &cat;
 proc sql;
@@ -50,7 +58,7 @@ create table datacats as select distinct libref,memname from catdata;
   ,base_dsn=mpe_datacatalog_CATS
   ,append_dsn=datacats
   ,PK=LIBREF MEMNAME
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO
@@ -76,7 +84,7 @@ quit;
   ,base_dsn=mpe_datacatalog_OBJS
   ,append_dsn=dataobjs
   ,PK=LIBREF MEMNAME OBJNAME OBJTYPE
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO
@@ -102,7 +110,7 @@ create table statusobjs as
   ,base_dsn=mpe_datastatus_OBJS
   ,append_dsn=statusobjs
   ,PK=LIBREF MEMNAME OBJNAME OBJTYPE
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO
@@ -127,7 +135,7 @@ create table statuscats as
   ,base_dsn=mpe_datastatus_cats
   ,append_dsn=statuscats
   ,PK=LIBREF MEMNAME
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO

sas/sasjs/macros/mpe_refreshlibs.sas

@@ -3,7 +3,12 @@
   @brief Refreshes the library data catalog
   @details
 
+  @param [in] etlsource= (refreshcatalog:<step>:%mf_uid()) identifies this load
+    in the MPE_DATALOADS audit row.  That table is keyed on etlsource together
+    with processed_dttm, libref and dsn, so the uid keeps each load distinct.
+
   <h4> SAS Macros </h4>
+  @li mf_uid.sas
   @li dc_getlibs.sas
   @li mf_getplatform.sas
   @li bitemporal_dataloader.sas
@@ -16,7 +21,9 @@
 
 **/
 
-%macro mpe_refreshlibs(lib=0);
+%macro mpe_refreshlibs(lib=0
+  ,etlsource=refreshcatalog:mpe_refreshlibs:%mf_uid()
+);
 
 %dc_getlibs(outds=work.mm_getLibs)
 proc sort data=mm_getlibs;
@@ -119,7 +126,7 @@ create table libs5 as
   ,base_dsn=MPE_DATACATALOG_LIBS
   ,append_dsn=libs5
   ,PK=LIBREF
-  ,etlsource=&_program
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO

sas/sasjs/macros/mpe_refreshtables.sas

@@ -1,12 +1,18 @@
 /**
   @file mpe_refreshtables.sas
   @brief Refreshes the data catalog
-  @details Assumes library is already assigned.
+  @details
+
+  @param [in] etlsource= (refreshcatalog:<step>:%mf_uid()) identifies this load
+    in the MPE_DATALOADS audit row.  That table is keyed on etlsource together
+    with processed_dttm, libref and dsn, so the uid keeps each load distinct.
   Usage:
 
       %mpe_refreshtables(sashelp)
 
   <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_abort.sas
   @li bitemporal_dataloader.sas
   @li mf_existfeature.sas
   @li mf_getengine.sas
@@ -17,7 +23,9 @@
   @author 4GL Apps Ltd
 **/
 
-%macro mpe_refreshtables(lib,ds=#all);
+%macro mpe_refreshtables(lib,ds=#all
+  ,etlsource=refreshcatalog:mpe_refreshtables:%mf_uid()
+);
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 %local engine; %let engine=%mf_getengine(&lib);
@@ -180,7 +188,7 @@ create table cols as
   ,base_dsn=mpe_datacatalog_vars
   ,append_dsn=finalcols
   ,PK=LIBREF DSN NAME
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO
@@ -242,7 +250,7 @@ create table work.finaltabs as
   ,base_dsn=mpe_datacatalog_tabs
   ,append_dsn=finaltabs
   ,PK=LIBREF DSN
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO
@@ -302,7 +310,7 @@ proc sql;
   ,base_dsn=mpe_datastatus_tabs
   ,append_dsn=statustabs
   ,PK=LIBREF DSN
-  ,etlsource=&sysmacroname
+  ,etlsource=&etlsource
   ,loadtype=TXTEMPORAL
   ,tech_from=TX_FROM
   ,tech_to=TX_TO
@@ -345,7 +353,7 @@ proc sql;
     ,base_dsn=mpe_datastatus_libs
     ,append_dsn=statuslibs
     ,PK=LIBREF
-    ,etlsource=&sysmacroname
+    ,etlsource=&etlsource
     ,loadtype=TXTEMPORAL
     ,tech_from=TX_FROM
     ,tech_to=TX_TO

sas/sasjs/macros/mpe_validatecol.test.sas

@@ -60,7 +60,7 @@ data work.mismatch;
 run;
 
 %mp_assertdsobs(work.mismatch,
-  desc=Every LIBDS value validated as expected (catalog form permitted, nothing smuggled past the dash),
+  desc=%str(Every LIBDS value validated as expected (catalog form permitted, nothing smuggled past the dash)),
   test=EQUALS 0,
   outds=work.test_results
 )

sas/sasjs/services/admin/refreshcatalog.test.1.sas

@@ -8,8 +8,9 @@
   the admin group, so the gate passes here.
 
   <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
-  @li mx_execute.sas
+  @li mpe_execute.sas
 
 **/
 
@@ -23,23 +24,19 @@ data work.params1;
   name='libref';value='A.%sysevalf(3+4)B';output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputparams=work.params1,
   outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
   desc=Macro content in libref aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid libref
 )
 
 /**
@@ -50,7 +47,7 @@ data work.params2;
   name='libref';value='DCTEST';output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputparams=work.params2,
   outlib=web2

sas/sasjs/services/auditors/getdiffs.test.sas

@@ -13,7 +13,9 @@
   against a service that does not validate.
 
   <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
+  @li mpe_execute.sas
   @li mx_execute.sas
   @li mf_getuniquefileref.sas
 
@@ -101,23 +103,19 @@ data _null_;
   put _row;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&fb1:BrowserParams,
   outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
   desc=SQL injection payload in libds aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid libds
 )
 
 /**
@@ -137,23 +135,19 @@ data _null_;
   put "&travtable.,&diffscsv.,&dclib..MPE_X_TEST";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&fb2:BrowserParams,
   outref=web2,
+  outstatus=work.ab2,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort2=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort2=1),
+%mpe_assert_abort(
   desc=Path traversal in table aborts the service,
-  outds=work.test_results
+  outref=web2,
+  msg=Invalid table
 )
 
 /**
@@ -166,23 +160,19 @@ data _null_;
   put "&loadref.,../&loadref./&diffscsv.,&dclib..MPE_X_TEST";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&fb3:BrowserParams,
   outref=web3,
+  outstatus=work.ab3,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort3=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort3',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort3=1),
+%mpe_assert_abort(
   desc=Path traversal in stp_diffs_csv aborts the service,
-  outds=work.test_results
+  outref=web3,
+  msg=Invalid stp_diffs_csv
 )
 
 /**

sas/sasjs/services/editors/getdata.sas

@@ -21,6 +21,10 @@
     DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
   @li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
     else empty
+  @li DERIVEDVARS - space separated list of the derived (computed) columns
+    configured in MPE_DERIVED, upper case, empty when there are none.  They
+    are appended after the physical columns and are read-only; the EDIT grid
+    marks them so a user can see why a column cannot be edited.
 
   <h5> approvers </h5>
   <h5> dqrules </h5>
@@ -181,7 +185,23 @@ run;
 
 
 %global loadtype var_txfrom var_txto var_processed filter_text pk coltype
-  sortpk;
+  sortpk var_busfrom var_busto rk_underlying buskey pre_edit_hook;
+/* clear first: a global survives the call, so in a reused session it would still
+ * hold the previous request's values, and any path that skips an assignment
+ * would apply them to this one */
+%let loadtype=;
+%let var_txfrom=;
+%let var_txto=;
+%let var_processed=;
+%let filter_text=;
+%let pk=;
+%let coltype=;
+%let sortpk=;
+%let var_busfrom=;
+%let var_busto=;
+%let rk_underlying=;
+%let buskey=;
+%let pre_edit_hook=;
 
 %put getting table attributes;
 proc sql noprint;
@@ -340,8 +360,10 @@ select upcase(loadtype)
                           );
   %put sourcevars=&sourcevars;
   /* the payload is capped at &maxrows - whichever limit that works out to be
-    - this is what goes to the browser */
-  options obs=&maxrows;
+    - this is what goes to the browser.  The cap is a data set option on the
+    SET, not `options obs=`, so no later step in this session can be cut by it:
+    a session option left set would truncate the metadata steps below, and a
+    table wide enough to hit the cell limit would lose columns rather than rows. */
   data outdata;
     /* pk fields come first */
     %if %length(&pk)>0 %then %do;
@@ -355,9 +377,8 @@ select upcase(loadtype)
       /* send empty row if empty table to help with hot rendering */
       output;
     %end;
-    set work.out ;
+    set work.out(obs=&maxrows) ;
   run;
-  options obs=max;
 
 
 
@@ -628,8 +649,19 @@ proc sql noprint;
 select count(*) into: ismap from &mpelib..mpe_xlmap_info
   where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
 
+/* the derived columns, named for the EDIT grid so it can mark them as special.
+  They are read-only through the READONLY DQ rule below, but that rule does not
+  say WHY the column cannot be edited - a computed column is a different thing
+  from an ordinary read-only one.  work.derived_vars exists on every path, empty
+  when the table has no derived variables. */
+%let derivedvars=;
+proc sql noprint;
+select upcase(name) into: derivedvars separated by ' '
+  from work.derived_vars;
+quit;
+
 data sasparams;
-  length colHeaders $20000 filter_text $32767 trunc_msg $200;
+  length colHeaders $20000 filter_text $32767 trunc_msg $200 derivedvars $32767;
   colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
   pkCnt=countw("&pk");
   pk="&pk";
@@ -640,6 +672,7 @@ data sasparams;
   if trim(symget('rk_underlying')) ne '' then rk_flag=1;
   else rk_flag=0;
   filter_text=symget('filter_text');
+  derivedvars=symget('derivedvars');
   /* set when the selection was cut down to the row / cell limit, so the EDIT
     screen can say so against the table name */
   truncated=&truncated;
@@ -661,7 +694,16 @@ proc sort data=&mpelib..mpe_validations
   by base_col rule_type rule_value;
 run;
 
-/* merge with NOTNULL constraints in the physical table */
+/* merge with NOTNULL constraints in the physical table, and mark the derived
+  variables read-only: a derived column is computed at read time, so the grid
+  must not accept a value for it.  COLS carries the same READ (a derived column
+  has no CLS rule row) but the client only applies COLS.CLS_RULE when CLS_FLAG
+  is set, which is driven by the table's column-level security - so the rule is
+  declared here, where it holds regardless.
+
+  work.derived_vars exists on every path that reaches this step: %mpe_derived
+  writes it before returning, empty when the table has no derived variables,
+  so the union below needs no guard */
 proc sql;
 create table _data_ as
   select * from dqrules
@@ -674,6 +716,11 @@ union
     and upcase(memname)="%scan(&orig_libds,2,.)"
     and upcase(name) in (select name from vars3)
     and notnull='yes'
+union
+  select upcase(name) as base_col
+    ,'READONLY' as rule_type
+    ,'' as rule_value
+  from work.derived_vars
   order by 1,2,3;
 data dqrules;
   set &syslast;

sas/sasjs/services/editors/getdata.test.1.sas

@@ -14,7 +14,8 @@
   rows x 1004 columns, created and registered by testsetup) filtered to 200
   rows: that is under the row limit but 200 x 1004 = 200,800 cells, over the
   200,000 cell limit, so the cell limit decides the cap -
-  floor(200000 / 1004) = 199 rows.
+  floor(200000 / 1004) = 199 rows.  It also checks the column list is whole:
+  the cap applies to the rows the browser is sent, never to the COLS metadata.
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
@@ -98,7 +99,7 @@ run;
 
 %mp_assert(
   iftrue=(&t1_trunc=1),
-  desc=Test 1 - a selection over the row limit is served, flagged as truncated,
+  desc=%str(Test 1 - a selection over the row limit is served, flagged as truncated),
   outds=work.test_results
 )
 %mp_assert(
@@ -133,7 +134,7 @@ run;
 
 %mp_assert(
   iftrue=(&t2_trunc=1),
-  desc=Test 2 - a selection over the cell limit is served, flagged as truncated,
+  desc=%str(Test 2 - a selection over the cell limit is served, flagged as truncated),
   outds=work.test_results
 )
 %mp_assert(
@@ -147,6 +148,27 @@ run;
   outds=work.test_results
 )
 
+/**
+  * The row cap must not reach the metadata: the COLS payload for this table
+  * describes every column, not the 199 rows the cell limit admits.  A row cap
+  * left set as a session option would cut the column list to 199 entries, and
+  * a grid losing 800 columns looks like a missing table rather than a limit.
+  */
+%let t2_cols=-1;
+%let t2_phys=-1;
+proc sql noprint;
+select count(*) into: t2_cols from web2.cols;
+select count(*) into: t2_phys
+  from dictionary.columns
+  where libname='DCTEST' and memname='WIDEBOY';
+quit;
+
+%mp_assert(
+  iftrue=(&t2_cols=&t2_phys),
+  desc=%str(Test 2 - every column is described, not just the rows the cap admits),
+  outds=work.test_results
+)
+
 /**
   * dump results to the log for offline inspection
   */

sas/sasjs/services/editors/getdata.test.sas

@@ -3,15 +3,14 @@
   @brief testing gethistory service
 
   <h4> SAS Macros </h4>
-  @li mp_assert.sas
-  @li mp_assertcols.sas
-  @li mp_assertcolvals.sas
+  @li mpe_execute.sas
   @li mp_assertdsobs.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
   @li mp_filterstore.sas
   @li mpe_derived.sas
-  @li mx_execute.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
 
 **/
 
@@ -57,7 +56,7 @@ data _null_;
   put 'LIBDS:$43. FILTER_RK:8.';
   put "&dclib..MPE_TABLES,&filter_rk1";
 run;
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f1:sascontroltable,
   outlib=web1,
@@ -112,8 +111,8 @@ data work.inquery;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
-AND,AND,1,LIBREF,CONTAINS,"'DC'"
-AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
+AND,AND,1,LIBREF,=,"'&dc_libref'"
+AND,AND,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
 ;;;;
 run;
 %mp_filterstore(
@@ -138,7 +137,7 @@ data _null_;
   put "&dclib..MPE_TABLES,&filter_rk";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f2:sascontroltable,
   outlib=web2
@@ -218,7 +217,7 @@ data _null_;
   put "DCTEST.DCFMTS-FC,&filter_rk3";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f3:sascontroltable,
   outlib=web3,
@@ -288,12 +287,16 @@ quit;
 )
 
 data work.inquery4;
-  infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:$4000.;
-datalines4;
-AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
-;;;;
+  length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
+    OPERATOR_NM $10 RAW_VALUE $4000;
+  GROUP_LOGIC='AND'; SUBGROUP_LOGIC='AND'; SUBGROUP_ID=1;
+  VARIABLE_NM='LIBREF'; OPERATOR_NM='=';
+  RAW_VALUE="'"!!trim(symget('dc_libref'))!!"'";
+  output;
+  GROUP_LOGIC='AND'; SUBGROUP_LOGIC='AND'; SUBGROUP_ID=2;
+  VARIABLE_NM='DERIVED_NOTE'; OPERATOR_NM='CONTAINS';
+  RAW_VALUE="'MPE_LOCK'";
+  output;
 run;
 %mp_filterstore(
   libds=&dc_libref..MPE_TABLES,
@@ -318,29 +321,38 @@ data _null_;
   put "&dclib..MPE_TABLES,&filter_rk4";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f4:sascontroltable,
   outlib=web4,
+  outstatus=work.ab4,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-/* a cancelled child returns no webout, so the outlib keeps whatever an
-  earlier call left in it - check the job state before reading its members,
-  otherwise the assertion failures below describe the wrong problem */
-%let abort4=0;
+%let state4=NONE;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort4',1);
+  if not exist('work.ab4') then stop;
+  set work.ab4;
+  call symputx('state4',state);
 run;
 
 %mp_assert(
-  iftrue=(&abort4=0),
+  iftrue=("&state4"="completed"),
   desc=Test4 - the derived variable call completes,
   outds=work.test_results
 )
 
-%if &abort4=0 %then %do;
+/* a cancelled child returns no webout, so the outlib keeps whatever an earlier
+  * call left in it - read its members only when the child completed */
+%let completed4=0;
+data _null_;
+  if not exist('work.ab4') then stop;
+  set work.ab4;
+  if state='completed' then call symputx('completed4',1);
+run;
+
+%if &completed4=1 %then %do;
   data sasdata4;
     set web4.sasdata;
   run;
@@ -355,10 +367,30 @@ run;
   run;
 %end;
 
-%mp_assertcols(work.cols4,
+%if &completed4=1 and "&sasjs_mdebug"="1" %then %do;
+  /* the payload the assertions read, so a missing column or an unapplied filter
+    can be seen without guessing */
+  data _null_;
+    set work.cols4;
+    if _n_=1 then putlog "NOTE: &sysmacroname: Test4 cols payload:";
+    putlog (_all_)(=);
+  run;
+  data _null_;
+    set sasdata4;
+    if _n_=1 then putlog "NOTE: &sysmacroname: Test4 data payload:";
+    putlog (_all_)(=);
+  run;
+%end;
+
+proc sql noprint;
+select count(*) into: derived4
+  from work.cols4
+  where upcase(name)='DERIVED_NOTE';
+quit;
+
+%mp_assert(
+  iftrue=(&derived4>0),
   desc=Test4 - derived column is in the COLS payload,
-  cols=DERIVED_NOTE,
-  test=ALL,
   outds=work.test_results
 )
 
@@ -372,6 +404,19 @@ run;
   outds=work.test_results
 )
 
+/* COLS.CLS_RULE only reaches the grid when CLS_FLAG is set, which a table
+  with no column level security does not do - the read-only promise is
+  carried by this rule instead */
+data work.chk4e;
+  set web4.dqrules;
+  where upcase(base_col)='DERIVED_NOTE' and rule_type='READONLY';
+run;
+%mp_assertdsobs(work.chk4e,
+  desc=Test4 - derived column carries a READONLY rule,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
 data work.chk4b;
   set work.cols4;
   where name='DERIVED_NOTE' and label=:'DERIVED_NOTE=';
@@ -390,7 +435,7 @@ run;
 
 data work.chk4c;
   set work.sasdata4;
-  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCK_ANYTABLE');
+  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCKANYTABLE');
 run;
 %mp_assertdsobs(work.chk4c,
   desc=Test4 - derived value is correct,

sas/sasjs/services/editors/stagedata.test.3.sas

@@ -5,13 +5,14 @@
     1. loading row level EDIT security against a table with LOADTYPE=REPLACE
     2. applying LOADTYPE=REPLACE to a table with row level EDIT security
 
-  The staged loads are aborted by the post edit hooks inside the child
-  service.  The abort is recorded as a FAILED load in MPE_LOADS (via
-  mpe_loadfail), so the assertions query MPE_LOADS rather than the child
-  response - a canceled child does not always return a webout or log.
+  The staged loads are aborted by the post edit hooks.  The assertion fetches
+  the sasjsAbort object that mp_abort writes to the webout before the hook
+  aborts.
 
   <h4> SAS Macros </h4>
   @li mx_execute.sas
+  @li mpe_execute.sas
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
 
 
@@ -71,7 +72,6 @@ quit;
 /**
   * Test 1 - submitting an RLS EDIT rule against a REPLACE table should abort
   */
-%let t1=%sysfunc(datetime());
 data work.sascontroltable;
   action='LOAD';
   message='RLS EDIT rule on REPLACE table should abort';
@@ -100,36 +100,19 @@ data work.jsdata;
   output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web1,
   outref=wb1,
+  outstatus=work.ab1,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-/* a hook-aborted load is terminated before mpe_loadfail can record a
-  * FAILED status, so the load row stays IN PROGRESS - the abort is
-  * proved by the load NOT completing successfully.  Tests in this suite run
-  * in parallel sessions, so match the row for THIS test by its unique
-  * reason_txt (the submitted message) rather than any load in the time
-  * window - an unrelated aborted/IN PROGRESS load would otherwise satisfy
-  * the assertion even if this load completed.  The row id (csv_dir/mperef)
-  * is generated inside the aborted child service, which returns no webout,
-  * so reason_txt is the only stable identifier the test controls. */
-%let abort1=0;
-proc sql noprint;
-select count(*) into: abort1
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t1
-    and reason_txt="RLS EDIT rule on REPLACE table should abort"
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort1>0),
+%mpe_assert_abort(
   desc=Checking RLS EDIT rule is rejected for REPLACE loadtype table (#211),
-  outds=work.test_results
+  outref=wb1
 )
 
 /**
@@ -137,7 +120,6 @@ quit;
   * time: the rule is evaluated against the submitted data, where a derived
   * column does not exist
   */
-%let t3=%sysfunc(datetime());
 
 /* define a derived variable on the target table */
 proc sql noprint;
@@ -185,27 +167,19 @@ data work.jsdata;
   output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web3,
   outref=wb3,
+  outstatus=work.ab3,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-%let abort3=0;
-proc sql noprint;
-select count(*) into: abort3
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t3
-    and reason_txt="RLS rule on a derived variable should abort"
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort3>0),
+%mpe_assert_abort(
   desc=Checking an RLS rule on a derived variable is rejected,
-  outds=work.test_results
+  outref=wb3
 )
 
 /* tidy up the derived configuration */
@@ -219,7 +193,6 @@ quit;
   * should abort.  The active EDIT rule was inserted in the prep section,
   * long before this child service reads it.
   */
-%let t2=%sysfunc(datetime());
 
 /* now stage a REPLACE loadtype for DCTEST.WIDEBOY */
 data work.sascontroltable;
@@ -237,27 +210,19 @@ data work.jsdata;
   _____DELETE__THIS__RECORD_____='No';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web2,
   outref=wb2,
+  outstatus=work.ab2,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-%let abort2=0;
-proc sql noprint;
-select count(*) into: abort2
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t2
-    and reason_txt="REPLACE loadtype on RLS-secured table should abort"
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort2>0),
+%mpe_assert_abort(
   desc=Checking REPLACE loadtype is rejected for RLS EDIT-secured table (#211),
-  outds=work.test_results
+  outref=wb2
 )
 
 /**

sas/sasjs/services/editors/stagedata.test.4.sas

@@ -18,13 +18,12 @@
     6. an inactive row is set active again while another active row depends on
       it - the re-activation is what makes the dependency resolvable
 
-  The staged loads are aborted by the post edit hook inside the child service.
-  An aborted load leaves its MPE_LOADS row IN PROGRESS, so each assertion
-  matches on the unique reason_txt (the submitted message) rather than any load
-  in the time window - the suite runs in parallel sessions.
+  The staged loads are aborted by the post edit hook. The assertion fetches the
+  sasjsAbort object that mp_abort writes to the webout before the hook aborts.
 
   <h4> SAS Macros </h4>
-  @li mx_execute.sas
+  @li mpe_execute.sas
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
 
 
@@ -89,27 +88,19 @@ data work.jsdata;
   output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web1,
   outref=wb1,
+  outstatus=work.ab1,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-%let abort1=0;
-proc sql noprint;
-select count(*) into: abort1
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t1
-    and reason_txt='Derived var depending on an inactive row should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort1>0),
+%mpe_assert_abort(
   desc=Checking a dependency on an inactive derived row is rejected,
-  outds=work.test_results
+  outref=wb1
 )
 
 /**
@@ -117,7 +108,6 @@ quit;
   * rejected: the control table still shows the row as active, so the check has
   * to read the staged row too
   */
-%let t2=%sysfunc(datetime());
 
 proc sql noprint;
 delete from &dc_libref..mpe_derived
@@ -164,27 +154,19 @@ data work.jsdata;
   _____DELETE__THIS__RECORD_____='No';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web2,
   outref=wb2,
+  outstatus=work.ab2,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-%let abort2=0;
-proc sql noprint;
-select count(*) into: abort2
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t2
-    and reason_txt='Deactivating a relied-on derived row should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort2>0),
+%mpe_assert_abort(
   desc=Checking deactivating a relied-on derived row is rejected,
-  outds=work.test_results
+  outref=wb2
 )
 
 /**
@@ -235,7 +217,7 @@ data work.jsdata;
   output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web3,
@@ -313,7 +295,7 @@ data work.jsdata;
   _____DELETE__THIS__RECORD_____='Yes';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web4,
@@ -385,27 +367,19 @@ data work.jsdata;
   _____DELETE__THIS__RECORD_____='Yes';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web5,
   outref=wb5,
+  outstatus=work.ab5,
+  failfast=0,
   mdebug=&sasjs_mdebug
 )
 
-%let abort5=0;
-proc sql noprint;
-select count(*) into: abort5
-  from &dc_libref..mpe_loads
-  where processed_dttm>&t5
-    and reason_txt='Deleting a relied-on derived row should abort'
-    and upcase(status) ne 'SUCCESS';
-quit;
-
-%mp_assert(
-  iftrue=(&abort5>0),
+%mpe_assert_abort(
   desc=Checking deleting a relied-on derived row is rejected,
-  outds=work.test_results
+  outref=wb5
 )
 
 /**
@@ -475,7 +449,7 @@ data work.jsdata;
   output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web6,
@@ -557,7 +531,7 @@ data work.jsdata;
   _____DELETE__THIS__RECORD_____='No';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputdatasets=work.sascontroltable work.jsdata,
   outlib=web7,
@@ -578,6 +552,7 @@ run;
   outds=work.test_results
 )
 
+
 /* tidy up so repeated runs start clean */
 proc sql noprint;
 delete from &dc_libref..mpe_derived

sas/sasjs/services/hooks/mpe_derived_postedit.sas

@@ -34,6 +34,7 @@
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mpe_derived_check.sas
 
   <h4> Related Macros </h4>
   @li mpe_derived.sas
@@ -101,44 +102,7 @@ run;
 /**
   * Field level checks - the same rules %mpe_derived applies at read time
   */
-data work.derived_check;
-  set work.staging_ds;
-  length reason $200;
-  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
-  call missing(reason);
-
-  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
-  else if var_nm='' then reason='VAR_NM is required';
-  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
-    then reason='VAR_NM must be a valid SAS name';
-  else if verify(substr(var_nm,1,1),'0123456789')=0
-    then reason='VAR_NM must not start with a digit';
-  else if var_nm='_____DELETE__THIS__RECORD_____'
-    then reason='VAR_NM is reserved';
-  else if var_type not in ('C','N')
-    then reason='VAR_TYPE must be C or N';
-  else if missing(var_len) or var_len ne int(var_len) or var_len<1
-    then reason='VAR_LEN must be a positive integer';
-  else if var_type='C' and var_len>32767
-    then reason='VAR_LEN exceeds 32767 for a character variable';
-  else if var_type='N' and var_len>8
-    then reason='VAR_LEN exceeds 8 for a numeric variable';
-  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
-    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
-    then reason='VAR_FMT must be a format name, without ; & % or //*';
-  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
-    then reason='VAR_FMT must be a format name';
-  else if missing(var_logic)
-    then reason='VAR_LOGIC is required';
-  else if index(var_logic,';')>0
-    then reason='VAR_LOGIC must be a single expression - remove the ;';
-  else if indexc(var_logic,'&%')>0
-    then reason='VAR_LOGIC must not contain macro triggers';
-  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
-    then reason='VAR_LOGIC must not contain comments';
-
-  if not missing(reason) then output;
-run;
+%mpe_derived_check(inds=work.staging_ds, outds=work.derived_check)
 
 %if %mf_nobs(work.derived_check)>0 %then %do;
   data _null_;
@@ -443,7 +407,7 @@ run;
 
 %mp_abort(iftrue= (&cyc_found=1)
   ,mac=mpe_derived_postedit
-  ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
+  ,msg=%str(MPE_DERIVED: VAR_DEPS contains a cycle: &cyc_path)
 )
 
 %mend _mpe_derived_postedit;

sas/sasjs/services/public/getcols.test.sas

@@ -12,8 +12,9 @@
   assertion cannot pass against a service that does not validate.
 
   <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
-  @li mx_execute.sas
+  @li mpe_execute.sas
   @li mf_getuniquefileref.sas
 
 **/
@@ -30,23 +31,19 @@ data _null_;
   put '%sysfunc(coalescec(&dclib..MPE_X_TEST,))';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f1:iwant,
   outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
   desc=Macro content in libds aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid libds
 )
 
 /**
@@ -59,7 +56,7 @@ data _null_;
   put "&dclib..MPE_X_TEST";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f2:iwant,
   outlib=web2

sas/sasjs/services/public/getcolvals.sas

@@ -36,7 +36,8 @@
   <h4> SAS Macros </h4>
   @li mf_existds.sas
   @li mf_nobs.sas
-  @li mf_verifymacvars.sas
+  @li mpeinit.sas
+  @li mpeterm.sas
   @li dc_assignlib.sas
   @li mf_getvarformat.sas
   @li mp_abort.sas
@@ -111,7 +112,10 @@ run;
 %let startrow=1;
 %let rows=4000;
 
-%mp_abort(iftrue= (%mf_verifymacvars(libds col2)=0)
+/* a request with no rows never reached the validation below, so this is the
+  only case that is missing inputs - an invalid value is a different abort, and
+  saying "missing" for it would hide the reason */
+%mp_abort(iftrue= (%mf_nobs(work.iwant)<=0)
   ,mac=&_program..sas
   ,msg=%str(Missing inputs from iwant)
 )

sas/sasjs/services/public/getcolvals.test.4.sas

@@ -13,8 +13,9 @@
   validate.
 
   <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
-  @li mx_execute.sas
+  @li mpe_execute.sas
   @li mf_getuniquefileref.sas
 
 **/
@@ -27,27 +28,23 @@
 %let f1=%mf_getuniquefileref();
 data _null_;
   file &f1 termstr=crlf;
-  put 'LIBDS:$19. COL:$9.';
+  put 'LIBDS:$100. COL:$100.';
   put '%sysfunc(coalescec(&dclib..MPE_X_TEST,)),SOME_TIME';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f1:iwant,
   outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
   desc=Macro content in libds aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid libds
 )
 
 /**
@@ -56,27 +53,23 @@ run;
 %let f2=%mf_getuniquefileref();
 data _null_;
   file &f2 termstr=crlf;
-  put 'LIBDS:$19. COL:$9.';
-  put '&dclib..MPE_X_TEST,%sysfunc(coalescec(SOME_TIME,))';
+  put 'LIBDS:$100. COL:$100.';
+  put "&dclib..MPE_X_TEST,%nrstr(%sysfunc(coalescec(SOME_TIME,)))";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f2:iwant,
   outref=web2,
+  outstatus=work.ab2,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort2=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort2=1),
+%mpe_assert_abort(
   desc=Macro content in col aborts the service,
-  outds=work.test_results
+  outref=web2,
+  msg=Invalid col
 )
 
 /**
@@ -85,11 +78,11 @@ run;
 %let f3=%mf_getuniquefileref();
 data _null_;
   file &f3 termstr=crlf;
-  put 'LIBDS:$19. COL:$9.';
+  put 'LIBDS:$100. COL:$100.';
   put "&dclib..MPE_X_TEST,SOME_TIME";
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f3:iwant,
   outlib=web3

sas/sasjs/services/public/getddl.test.sas

@@ -15,7 +15,7 @@
 data work.params;
   length name $32 value $1000;
   name='libref';value="&dclib";output;
-  name='ds';value='';output;
+  name='ds';value='MPE_TABLES';output;
   name='flavour';value='TSQL';output;
 run;
 %mx_execute(&_program,
@@ -31,7 +31,7 @@ data work.results;
   input;
   putlog _infile_;
   if _n_=2 then do;
-    if _infile_=:'/* TSQL Flavour DDL for' then do;
+    if _infile_=:'/* TSQL Flavour DDL for' and index(_infile_,'MPE_TABLES')>0 then do;
       putlog 'test passed';
       output;
       output;
@@ -42,7 +42,7 @@ data work.results;
 run;
 
 %mp_assertdsobs(work.results,
-  desc=DDL file is successfully returned,
+  desc=DDL file is successfully returned for a table,
   test=EQUALS 2,
   outds=work.test_results
 )

sas/sasjs/services/public/getrawdata.test.1.sas

@@ -13,8 +13,9 @@
   validate.
 
   <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
-  @li mx_execute.sas
+  @li mpe_execute.sas
 
 **/
 
@@ -31,23 +32,19 @@ data work.params1;
   name='filter';value='0';output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputparams=work.params1,
   outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
   desc=Macro content in table aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid table
 )
 
 /**
@@ -60,23 +57,19 @@ data work.params2;
   name='filter';value='0 or 1=1';output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputparams=work.params2,
   outref=web2,
+  outstatus=work.ab2,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort2=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort2=1),
+%mpe_assert_abort(
   desc=SQL injection in filter aborts the service,
-  outds=work.test_results
+  outref=web2,
+  msg=Invalid filter
 )
 
 /**
@@ -91,23 +84,19 @@ data work.params3;
   name='filter';value='0';output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputparams=work.params3,
   outref=web3,
+  outstatus=work.ab3,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort3=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort3',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort3=1),
+%mpe_assert_abort(
   desc=Path traversal in table aborts the service,
-  outds=work.test_results
+  outref=web3,
+  msg=Invalid table
 )
 
 /**
@@ -120,7 +109,7 @@ data work.params4;
   name='filter';value='0';output;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputparams=work.params4,
   outref=web4,

sas/sasjs/services/public/validatefilter.sas

@@ -79,11 +79,23 @@ run;
   outvars=work.derived_vars
 )
 %let derived_nobs=%mf_nobs(work.derived_vars);
-%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&ds));
+
+/* targetds is passed as before for a plain dataset, and as the view when the
+  filter can reference derived columns.  A format catalog is the one case that
+  must be left alone: mp_filterstore rewrites a -FC libds to a cntlout dataset
+  itself, and a -FC name handed over as targetds is not something it can read.
+  The trailing comma lets the call below read targetds=..., queryds=... */
+%let targetds_opt=;
+%if &derived_nobs>0 %then %do;
+  %let targetds_opt=targetds=work.vw_derived,;
+%end;
+%else %if %length(&ds)>3 and "%upcase(%substr(&ds,%length(&ds)-2,3))" ne "-FC" %then %do;
+  %let targetds_opt=targetds=&ds,;
+%end;
 
 %mp_filterstore(
   libds=&ds,
-  targetds=&filterds,
+  &targetds_opt
   queryds=work.filterquery,
   filter_summary=&dc_libref..mpe_filteranytable,
   filter_detail=&dc_libref..mpe_filtersource,

sas/sasjs/services/public/validatefilter.test.1.sas

@@ -12,8 +12,9 @@
   assertion cannot pass against a service that does not validate.
 
   <h4> SAS Macros </h4>
+  @li mpe_assert_abort.sas
   @li mp_assert.sas
-  @li mx_execute.sas
+  @li mpe_execute.sas
   @li mf_getuniquefileref.sas
 
 **/
@@ -30,23 +31,19 @@ data _null_;
   put '%sysfunc(coalescec(&dclib..MPE_TABLES,))';
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f1:iwant,
   outref=web1,
+  outstatus=work.ab1,
+  failfast=0,
   viyaresult=WEBOUT_TXT
 )
 
-%let abort1=0;
-data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
-run;
-
-%mp_assert(
-  iftrue=(&abort1=1),
+%mpe_assert_abort(
   desc=Macro content in filter_table aborts the service,
-  outds=work.test_results
+  outref=web1,
+  msg=Invalid filter_table
 )
 
 /**
@@ -71,7 +68,7 @@ AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
 ;;;;
 run;
 
-%mx_execute(&_program,
+%mpe_execute(&_program,
   viyacontext=&defaultcontext,
   inputfiles=&f2:iwant &f3:filterquery,
   outlib=web2

sas/sasjs/services/public/viewdata.test.3.sas

@@ -57,7 +57,7 @@ data _null_;
 run;
 %mp_assert(
   iftrue=(&nobs=&expected),
-  desc=&desc (&searchtype "%superq(searchval)" returned &nobs rows, expected &expected),
+  desc=%str(&desc (&searchtype "%superq(searchval)" returned &nobs rows, expected &expected)),
   outds=work.test_results
 )
 %mend search;
@@ -118,6 +118,6 @@ proc sql noprint;
 quit;
 %mp_assert(
   iftrue=(&caprows=&capnobs and &capnobs gt 0 and &capnobs le &capmax),
-  desc=Checking a search beyond the cap returns as many rows as it reports (&capnobs reported, &caprows returned, cap &capmax),
+  desc=%str(Checking a search beyond the cap returns as many rows as it reports (&capnobs reported, &caprows returned, cap &capmax)),
   outds=work.test_results
 )

sas/sasjs/targets/sas9/macros_meta/dc_getlibs.sas

@@ -44,4 +44,12 @@ run;
 
 options metarepository=&repo;
 
+%if "&sasjs_mdebug"="1" %then %do;
+  data _null_;
+    set &outds;
+    putlog "NOTE: &sysmacroname: &outds:";
+    putlog (_all_)(=);
+  run;
+%end;
+
 %mend dc_getlibs;

sas/sasjs/targets/server/macros_server/dc_getlibs.sas

@@ -20,7 +20,15 @@ create table &outds as
     ,engine
     ,'' as libraryid length=17
   from dictionary.libnames
-  where libname not in ('WORK','SASUSER');
+  where upcase(libname) not in ("WORK","SASUSER","%upcase(&dc_libref)");
 insert into &syslast values ("&DC_LIBREF", "&DC_LIBNAME",'','V9');
 
+%if "&sasjs_mdebug"="1" %then %do;
+  data _null_;
+    set &outds;
+    putlog "NOTE: &sysmacroname: &outds:";
+    putlog (_all_)(=);
+  run;
+%end;
+
 %mend dc_getlibs;

sas/sasjs/targets/server/macros_server/dc_refreshcatalog.sas

@@ -6,6 +6,11 @@
 
   The MPELIB should be pre-assigned
 
+  The refresh macros are called from the statements this macro writes into a
+  catalog, so they are not visible as calls to lint.
+
+  @sasjslint {"noUnusedMacros": false}
+
   <h4> SAS Macros </h4>
   @li mpe_refreshlibs.sas
   @li dc_assignlib.sas
@@ -32,12 +37,17 @@ data libraries;
   where also libref="&libref";
 %end;
   file executor;
-  str=cats('%mpe_refreshtables(',libref,')');
+  str=cats('%mpe_refreshtables(',libref,');');
   put str;
   putlog str;
-  str=cats('%mpe_refreshcatalogs(',libref,')');
+  str=cats('%mpe_refreshcatalogs(',libref,');');
   put str;
 run;
-%inc executor/source2;
+%if "&sasjs_mdebug"="1" %then %do;
+  /* echo the generated statements, so a failure inside one of them can be
+    traced back to the libref that produced it */
+  %inc executor/source2;
+%end;
+%else %inc executor;
 
 %mend dc_refreshcatalog;

sas/sasjs/targets/server/services_server/admin/makedata.sas

@@ -6,8 +6,10 @@
     It also adds the STAGING directory as subdirectory to the BASE
     library location.
 
-  @warning This STP self destructs! It will delete itself after a successful run
-    to avoid being executed twice (and overwriting actual data)
+  @warning This STP self destructs! It will delete itself (and the
+    configurator) after a successful run, to avoid being executed twice and
+    overwriting actual data.  Pass a KEEPME column in the input data to keep
+    them - test harnesses reseed before every run and cannot redeploy first.
 
   <h4> SAS Macros </h4>
   @li mf_getapploc.sas
@@ -55,6 +57,24 @@ data _null_;
 run;
 %end;
 
+/* optional KEEPME column - keep this program and the configurator on the drive
+  * after a successful run.  A normal install self-destructs so the seeding
+  * cannot be run twice over live data; a test harness reseeds before every run
+  * and wants them to stay put. */
+%let keepme=0;
+%if &exist=1 %then %do;
+data _null_;
+  dsid=open('work.fromjs');
+  /* every row, not just the first: KEEPME may be set on any of them */
+  if varnum(dsid,'KEEPME')>0 then do until(eof);
+    set work.fromjs end=eof;
+    if upcase(cats(keepme)) in ('1','Y','YES','TRUE') then call symputx('keepme',1);
+  end;
+  rc=close(dsid);
+  stop;
+run;
+%end;
+
 %mp_abort(iftrue= (&syscc ne 0)
   ,mac=&_program
   ,msg=%str(Issue on makedata entry)
@@ -191,7 +211,13 @@ data _null_;
 run;
 %end;
 
-/* We ran successfully, now remove configurator and makedata STPs */
-%ms_deletefile(&root/services/admin/configurator.sas)
-%ms_deletefile(&_program..sas)
+/* We ran successfully, now remove configurator and makedata STPs, unless the
+  * caller asked to keep them (KEEPME in the input data).  Keeping them is for
+  * test harnesses, which reseed before every run and cannot redeploy first. */
+%if &keepme ne 1 %then %do;
+  %ms_deletefile(&root/services/admin/configurator.sas)
+  %ms_deletefile(&_program..sas)
+%end;
+%else %put NOTE: &_program retained, as KEEPME was requested;
+
 

sas/sasjs/targets/viya/macros_viya/dc_getlibs.sas

@@ -21,7 +21,15 @@ create table &outds as
     ,engine
     ,'' as libraryid length=17
   from dictionary.libnames
-  where libname not in ('WORK','SASUSER','CASUSER');
+  where upcase(libname) not in ("WORK","SASUSER","CASUSER","%upcase(&dc_libref)");
 insert into &syslast values ("&DC_LIBREF", "&DC_LIBNAME",'','V9');
 
+%if "&sasjs_mdebug"="1" %then %do;
+  data _null_;
+    set &outds;
+    putlog "NOTE: &sysmacroname: &outds:";
+    putlog (_all_)(=);
+  run;
+%end;
+
 %mend dc_getlibs;

sas/sasjs/targets/viya/macros_viya/dc_refreshcatalog.sas

@@ -6,6 +6,11 @@
 
   The MPELIB should be pre-assigned
 
+  The refresh macros are called from the statements this macro writes into a
+  catalog, so they are not visible as calls to lint.
+
+  @sasjslint {"noUnusedMacros": false}
+
   <h4> SAS Macros </h4>
   @li mpe_refreshlibs.sas
   @li dc_assignlib.sas
@@ -36,13 +41,18 @@
     set &mpelib..mpe_datacatalog_libs;
     where &dc_dttmtfmt. le TX_TO;
     file executor;
-    str=cats('%mpe_refreshtables(',libref,')');
+    str=cats('%mpe_refreshtables(',libref,');');
     put str;
-    str=cats('%mpe_refreshcatalogs(',libref,')');
+    str=cats('%mpe_refreshcatalogs(',libref,');');
     put str;
     putlog str;
   run;
-  %inc executor;
+  %if "&sasjs_mdebug"="1" %then %do;
+    /* echo the generated statements, so a failure inside one of them can be
+      traced back to the libref that produced it */
+    %inc executor/source2;
+  %end;
+  %else %inc executor;
 %end;
 
 %mend dc_refreshcatalog;

sas/sasjs/targets/viya/services_viya/admin/makedata.sas

@@ -9,7 +9,6 @@
   @li mf_mkdir.sas
   @li mf_trimstr.sas
   @li mp_abort.sas
-  @li mpe_getvars.sas
   @li mpe_makedata.sas
   @li mpe_makedatamodel.sas
   @li mpe_makesampledata.sas
@@ -33,6 +32,24 @@
 %let exist=%sysfunc(exist(work.fromjs));
 %let inds=%sysfunc(ifc(&exist=1,fromjs,_null_));
 
+/* optional KEEPME column - keep this program on the content server after a
+  * successful run.  A normal install self-destructs so the seeding cannot be
+  * run twice over live data; a test harness reseeds once and then re-runs
+  * individual tests, so it wants the program to stay put. */
+%let keepme=0;
+%if &exist=1 %then %do;
+data _null_;
+  dsid=open('work.fromjs');
+  /* every row, not just the first: KEEPME may be set on any of them */
+  if varnum(dsid,'KEEPME')>0 then do until(eof);
+    set work.fromjs end=eof;
+    if upcase(cats(keepme)) in ('1','Y','YES','TRUE') then call symputx('keepme',1);
+  end;
+  rc=close(dsid);
+  stop;
+run;
+%end;
+
 data _null_;
   set &inds;
   call symputx('dcpath',dcpath);
@@ -278,4 +295,9 @@ run;
 %webout(OBJ,result)
 %webout(CLOSE)
 
-%mv_deletejes(path=&root/admin, name=makedata)
+/* keep the program when KEEPME was requested, so a reseed does not need a
+  * redeploy first */
+%if &keepme ne 1 %then %do;
+  %mv_deletejes(path=&root/admin, name=makedata)
+%end;
+%else %put NOTE: &_program retained, as KEEPME was requested;

sas/sasjs/tests/testsetup.sas

@@ -55,6 +55,8 @@ run;
 %dc_getsettings()
 
 
+
+
 /**
   * Set up Test library
   */