Merge pull request 'fix(mocks): key rk_underlying tables on their business key' (#357 ) from fix/mock-rk-underlying-pk into feat/derived-vars

Reviewed-on: #357
Merge branch 'feat/derived-vars' into fix/mock-rk-underlying-pk
2026-10-24 09:24:39 +00:00 · 2026-10-24 08:40:44 +00:00 · 2026-10-24 00:32:25 +00:00 · 2026-10-23 23:53:15 +00:00 · 2026-10-23 23:30:57 +00:00 · 2026-10-23 19:58:09 +00:00
71 changed files with 5881 additions and 2191 deletions
@@ -198,7 +198,7 @@ jobs:
          # Start frontend and run cypress
          # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
          # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"

      - name: Zip Cypress videos
        if: always()
@@ -380,8 +380,12 @@ jobs:
          # ahead of the installation footer. The section text lives in a
          # dedicated script below (.gitea/scripts/verify-section.sh) so this
          # YAML scalar stays clean.
+          # `jq -r` is required: without it jq emits the result as a JSON string
+          # literal (quoted, with \n escapes), and the PATCH below then encodes
+          # that literal a second time, so the release body renders as one
+          # escaped line instead of markdown.
          VERIFY_SECTION=$(./.gitea/scripts/verify-section.sh)
-          NEW_BODY=$(jq -n --arg body "$RELEASE_BODY" --arg verify "$VERIFY_SECTION" \
+          NEW_BODY=$(jq -nr --arg body "$RELEASE_BODY" --arg verify "$VERIFY_SECTION" \
            '$body + "\n" + $verify + "\n\nFor installation instructions, please visit https://docs.datacontroller.io/"')
          curl -k --fail-with-body -sS -X PATCH \
            -H "$AUTH_HEADER" \
@@ -1,3 +1,27 @@
+# [7.18.0](https://git.datacontroller.io/dc/dc/compare/v7.17.0...v7.18.0) (2026-10-19)
+
+
+### Bug Fixes
+
+* **deps:** bump @sasjs/core to 5.4.0 ([19fb032](https://git.datacontroller.io/dc/dc/commit/19fb032f0dbdd612651ca4ff3b55d1aefa4850d5))
+* **deps:** pin @sasjs/utils and drop the unused jsrsasign ([05d2d1e](https://git.datacontroller.io/dc/dc/commit/05d2d1e08e2c3082f0dd5aa4629d91907c0401fc))
+* **editor:** own the delete-record column client-side ([56025cc](https://git.datacontroller.io/dc/dc/commit/56025cc0524d34d3589992e9f1059c13fb5581ed))
+* **editor:** surface the reserved-column refusal on every getdata path ([b26f557](https://git.datacontroller.io/dc/dc/commit/b26f5571b66b65166c1158817f2df3027ae45cdc))
+* **filters:** do not turn a fractional temporal value into a literal ([f710d87](https://git.datacontroller.io/dc/dc/commit/f710d871fffc0fa451fd8cc4d037a26414725654))
+* **filters:** fix the dynamic where clause deref and map its temporal values ([ece67d3](https://git.datacontroller.io/dc/dc/commit/ece67d37ba2c4e2037dea626b9730a692388f025))
+* **filters:** keep the range field readable and cover a BETWEEN in the rewrite ([be79f3a](https://git.datacontroller.io/dc/dc/commit/be79f3a1ea5e4c4a157396d276f2c6654f615d39))
+* **filters:** map an IN entry to its unformatted value before the literal ([8f98ebe](https://git.datacontroller.io/dc/dc/commit/8f98ebe326978b8da58d8ac895ec946e562b5de4))
+* **filters:** stop the temporal rewrite corrupting a typed value, and bind the range dropdown to the unformatted value ([a5f61d4](https://git.datacontroller.io/dc/dc/commit/a5f61d4a73051237b671695f67122c237770c95a))
+* **filters:** use the unformatted value in every IN branch ([cc01f54](https://git.datacontroller.io/dc/dc/commit/cc01f540c29d3d5f7dd86943b6eed82b332ce868))
+* **makedata:** shorten the DC_MAXCELLS_WEBEDIT description to fit var_desc ([a51d9e5](https://git.datacontroller.io/dc/dc/commit/a51d9e5574383e5d1e1943c95261fbce63e8788d))
+* **multi-dataset:** surface the reserved-column guard, use the name constant ([2b8a2fa](https://git.datacontroller.io/dc/dc/commit/2b8a2fa1039da3ab888af8c3ffae7e1b848b250a))
+* **release:** emit the release body as raw text, not a JSON string ([a05554e](https://git.datacontroller.io/dc/dc/commit/a05554e67a8d44e32e2d9f5e4a959a9d0592be5e))
+
+
+### Features
+
+* **filters:** show dates as date literals in the filter query ([662d000](https://git.datacontroller.io/dc/dc/commit/662d00032c27634bbc2d82c392e7875a2e174eff))
+
 # [7.17.0](https://git.datacontroller.io/dc/dc/compare/v7.16.0...v7.17.0) (2026-10-15)


@@ -25,11 +25,13 @@ const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
 //   2. Ordinary columns (e.g. SOME_DROPDOWN) still get their rule via the
 //      new per-row cols[].COLTYPE path — test 2 proves the migration didn't
 //      silently drop real columns' rules either.
-// Column positions below come from COLHEADERS in
-// sas/mocks/sasjs/services/editors/getdata.js:
-// "_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,..."
-// — childNodes[0] on a body <tr> is the row-header <th>, so childNodes[N+1]
-// is the Nth (0-indexed) column in that list.
+// Column positions below come from the mock's COLHEADERS
+// (sas/mocks/sasjs/services/editors/getdata.js):
+// "PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,..."
+// — the client-owned delete column is prepended by the editor itself (see
+// deleteRecordColumn.ts), so on screen it is still column 0. childNodes[0] on
+// a body <tr> is the row-header <th>, so childNodes[N+1] is the Nth
+// (0-indexed) column in that list.
 context('coltype vertical-array regression tests: ', function () {
  this.beforeAll(() => {
    cy.visit(`${hostUrl}/SASLogon/logout`)
@@ -0,0 +1,162 @@
+export {}
+//
+// Derived (computed) variables are configured per table in MPE_DERIVED and
+// created at read time, ahead of the filter, so that they can be filtered on.
+//
+// TESTDATA.DERIVED_DEMO carries two of them, neither of which is a physical
+// column of the table:
+//   REGION      ifc(STATE in ('CA','OR','WA'),'West','East')
+//   LINE_VALUE  QTY * PRICE
+//
+// The mock implements the derivation in getdata.js / getcolvals.js (see
+// mpeDerived in dcMockUtils.js), so the EDIT screen can be exercised without a
+// SAS server. The real evaluation is done by SAS.
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('derived variables: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.viewport(1440, 900)
+    bootApp()
+  })
+
+  it('1 | derived columns are shown in the EDIT grid with computed values', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    getCellByHeaderAndRow(0, 'REGION').should('have.text', 'West')
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'East')
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '21')
+    getCellByHeaderAndRow(1, 'LINE_VALUE').should('have.text', '99')
+    // REGION_LABEL is built on REGION, so it only resolves if the derived
+    // statements run in VARIABLE_ORDER sequence
+    getCellByHeaderAndRow(0, 'REGION_LABEL').should('have.text', 'West-CA')
+    getCellByHeaderAndRow(1, 'REGION_LABEL').should('have.text', 'East-NY')
+
+    cy.screenshot('derived-1-grid', { capture: 'viewport' })
+  })
+
+  it('2 | filters on a character derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('REGION')
+    setFilterValue('West')
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', "REGION = 'West'")
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 2)
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'West')
+
+    cy.screenshot('derived-2-filter-region', { capture: 'viewport' })
+  })
+
+  it('3 | filters on a numeric derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('LINE_VALUE')
+    setFilterOperator('>')
+    setFilterValue('50')
+    submitFilter()
+
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 1)
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '99')
+
+    cy.screenshot('derived-3-filter-value', { capture: 'viewport' })
+  })
+})
+
+/**
+ * Cypress clears cookies between tests, which drops the SASjs Server session
+ * and lands the app back on the evaluation agreement card. Accept it when it is
+ * shown - a no-op when the session survived.
+ */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: longerCommandTimeout })
+  cy.get('body').then(($body: any) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Opens a table straight in the EDIT screen and waits for the grid. */
+const openEditor = (libMem: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/editor/${libMem}`, {
+    timeout: longerCommandTimeout
+  })
+  cy.get('.ht_clone_top .htCore thead tr th', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+  cy.wait(2500)
+}
+
+const openFilterPopup = () => {
+  cy.contains('button.btnView', 'Filter', {
+    timeout: longerCommandTimeout
+  }).click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/**
+ * The soft-select inputs drop a transparent .overlay click-catcher over the
+ * modal while their suggestion list is open, so they need force.
+ */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterOperator = (operator: string) => {
+  cy.get('.filter-modal .operator-col select').first().select(operator)
+  cy.wait(500)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+const getCellByHeaderAndRow = (rowIndex: number, headerText: string) => {
+  return cy
+    .get('.ht_clone_top .htCore thead tr th')
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => (th as HTMLElement).innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => (th as HTMLElement).innerText.trim() === headerText
+      )
+
+      return cy
+        .get('.ht_master tbody tr')
+        .then((rows: any) => rows[rowIndex].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}
@@ -399,6 +399,26 @@ context('excel tests: ', function () {
    })
  })

+  it('25 | Keeps decimals when the upload preview becomes a manual edit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    // Entering manual edit mode discards the source file but keeps the values
+    // parsed from it. SOME_NUM holds 0.901045951946194 in the first data row
+    // of the fixture, so a truncated or zeroed cell fails the assertion.
+    attachExcelFile('regular_excel.xlsx', () => {
+      getFirstRowCells().then((previewCells) => {
+        expect(previewCells).to.include(decimalSomeNum)
+
+        cy.get('.btn-upload-preview.btn-primary').click()
+        cy.get('.modal-footer .btn-success-outline').click()
+
+        getFirstRowCells().then((editCells) => {
+          expect(editCells).to.include(decimalSomeNum)
+        })
+      })
+    })
+  })
+
  // Large files break Cypress

  // it ('? | Uploads Excel with size of 5MB', (done) => {
@@ -476,6 +496,19 @@ const attachExcelFile = (excelFilename: string, callback?: any) => {
    })
 }

+const decimalSomeNum = '0.901045951946194'
+
+// The row's cell texts, without depending on column positions: the preview
+// grid renders row-status and delete cells in an order that varies.
+const getFirstRowCells = () =>
+  cy
+    .get('#hotTable', { timeout: longerCommandTimeout })
+    .find('div.ht_master.handsontable tbody tr')
+    .eq(0)
+    .then(($row) =>
+      Array.from($row[0].children).map((cell: any) => cell.innerText.trim())
+    )
+
 const submitExcel = (callback?: any) => {
  cy.get('.buttonBar button.preview-submit', { timeout: longerCommandTimeout })
    .click()
@@ -11,6 +11,18 @@ context('filtering tests: ', function () {
  this.beforeAll(() => {
    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
    cy.loginAndUpdateValidKey()
+
+    /**
+     * The evaluation agreement is per-origin localStorage, so a fresh browser
+     * profile lands on it and the nav tree never renders.  Guarded, so it is a
+     * no-op once an earlier spec in the run has accepted it.
+     */
+    cy.get('body').then(($body) => {
+      if ($body.find('#TCS input[type="checkbox"]').length) {
+        cy.get('#TCS input[type="checkbox"]').check({ force: true })
+        cy.wait(4000)
+      }
+    })
  })

  this.beforeEach(() => {
@@ -51,9 +63,13 @@ context('filtering tests: ', function () {

    openFilterPopup(() => {
      setFilterWithValue('SOME_TIME', '00:00:42', 'time', () => {
-        checkInfoBarIncludes(`(( SOME_TIME = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_TIME = '00:00:42't ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(3, () => done())
+          }
+        )
      })
    })
  })
@@ -63,9 +79,13 @@ context('filtering tests: ', function () {

    openFilterPopup(() => {
      setFilterWithValue('SOME_TIME', '42', 'value', () => {
-        checkInfoBarIncludes(`(( SOME_TIME = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_TIME = '00:00:42't ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(3, () => done())
+          }
+        )
      })
    }, false)
  })
@@ -75,9 +95,13 @@ context('filtering tests: ', function () {

    openFilterPopup(() => {
      setFilterWithValue('SOME_DATE', '12/02/1960', 'date', () => {
-        checkInfoBarIncludes(`(( SOME_DATE = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_DATE = '12FEB1960'd ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
      })
    })
  })
@@ -87,9 +111,13 @@ context('filtering tests: ', function () {

    openFilterPopup(() => {
      setFilterWithValue('SOME_DATE', '42', 'value', () => {
-        checkInfoBarIncludes(`(( SOME_DATE = 42 ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_DATE = '12FEB1960'd ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
      })
    }, false)
  })
@@ -104,9 +132,10 @@ context('filtering tests: ', function () {
        'datetime',
        () => {
          checkInfoBarIncludes(
-            `(( SOME_DATETIME = 42 ))`,
+            `(( SOME_DATETIME = '01JAN1960:00:00:42'dt ))`,
            (includes: boolean) => {
-              if (includes) done()
+              if (!includes) return
+              assertGridRowCount(5, () => done())
            }
          )
        }
@@ -120,9 +149,10 @@ context('filtering tests: ', function () {
    openFilterPopup(() => {
      setFilterWithValue('SOME_DATETIME', '42', 'value', () => {
        checkInfoBarIncludes(
-          `(( SOME_DATETIME = 42 ))`,
+          `(( SOME_DATETIME = '01JAN1960:00:00:42'dt ))`,
          (includes: boolean) => {
-            if (includes) done()
+            if (!includes) return
+            assertGridRowCount(5, () => done())
          }
        )
      })
@@ -134,9 +164,13 @@ context('filtering tests: ', function () {

    openFilterPopup(() => {
      setFilterWithValue('SOME_DATE', '', 'in', () => {
-        checkInfoBarIncludes(`(( SOME_DATE IN (42) ))`, (includes: boolean) => {
-          if (includes) done()
-        })
+        checkInfoBarIncludes(
+          `(( SOME_DATE IN ('12FEB1960'd) ))`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
      })
    })
  })
@@ -156,6 +190,34 @@ context('filtering tests: ', function () {
  //     })
  //   })
  // })
+
+  /**
+   * With the pickers off, a temporal BETWEEN takes both of its values from a
+   * soft-select over the column's values, so the value is chosen from that list
+   * rather than typed.  The range dropdown binds the formatted label, as the
+   * single-value dropdown's list does, so the field reads the date and the
+   * clause composition maps it back to the internal day / second count before
+   * the literal is built - a display string that reached sasTemporalLiteral as
+   * a non-number would come back null and be passed through to RAW_VALUE
+   * unquoted, which mp_filtercheck rejects on a numeric column and aborts the
+   * save.  The test asserts the applied clause and the row count, so it pins
+   * the mapping end to end.
+   */
+  it('7 | Non picker - filter date field BETWEEN from the value list', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    openFilterPopup(() => {
+      setTemporalRangeFromValueList('SOME_DATE', '12FEB1960', () => {
+        checkInfoBarIncludes(
+          `BETWEEN '12FEB1960'd AND '12FEB1960'd`,
+          (includes: boolean) => {
+            if (!includes) return
+            assertGridRowCount(5, () => done())
+          }
+        )
+      })
+    }, false)
+  })
 })

 const checkInfoBarIncludes = (text: string, callback: any) => {
@@ -166,6 +228,21 @@ const checkInfoBarIncludes = (text: string, callback: any) => {
  })
 }

+/**
+ * The temporal filter tests assert the applied filter still selects the same
+ * rows as the internal value did - a readable clause that filtered differently
+ * would be a worse bug than the one being fixed.
+ */
+const assertGridRowCount = (expected: number, callback: any) => {
+  cy.get('#hotTable .ht_master tbody tr', {
+    timeout: longerCommandTimeout
+  })
+    .should('have.length', expected)
+    .then(() => {
+      if (callback) callback()
+    })
+}
+
 const openFilterPopup = (
  callback?: any,
  usePickers: boolean = true,
@@ -331,6 +408,54 @@ const setFilterWithValue = (
  if (callback) callback()
 }

+/**
+ * Drives a temporal BETWEEN with the pickers off, choosing both values from the
+ * range soft-select's list rather than typing them - typing bypasses the
+ * binding this test exists to pin.
+ */
+const setTemporalRangeFromValueList = (
+  variableValue: string,
+  formatted: string,
+  callback?: any
+) => {
+  cy.wait(600)
+
+  cy.focused().type(variableValue)
+  cy.wait(100)
+  cy.get('.variable-col .autocomplete-wrapper', { withinSubject: null })
+    .first()
+    .trigger('keydown', { key: 'ArrowDown' })
+  cy.get('.variable-col .autocomplete-wrapper', {
+    withinSubject: null
+  }).trigger('keydown', { key: 'Enter' })
+  cy.focused().tab()
+  cy.wait(100)
+
+  cy.focused().select('BETWEEN').trigger('change')
+  cy.wait(400)
+
+  const chooseFromList = (range: 'start' | 'end') => {
+    const inputId = `vals-${range}_0_0`
+
+    cy.get(`#${inputId}`)
+      .clear({ force: true })
+      .type(formatted, { force: true })
+    cy.get(`#datalist_${inputId} option`)
+      .contains(formatted)
+      .click({ force: true })
+    cy.get(`#${inputId}`).trigger('keyup', { key: 'Escape', force: true })
+    cy.wait(300)
+  }
+
+  chooseFromList('start')
+  chooseFromList('end')
+
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+
+  if (callback) callback()
+}
+
 const openTableFromTree = (libNameIncludes: string, tablename: string) => {
  cy.get('.app-loading', { timeout: longerCommandTimeout })
    .should('not.exist')
@@ -1,13 +1,15 @@
-// The EDIT screen refuses a selection that is too large - DC_MAXOBS_WEBEDIT
-// rows, or DC_MAXCELLS_WEBEDIT cells (rows x columns), whichever is reached
-// first - and the VIEW screen caps at DC_MAXOBS_WEBVIEW rows.
+// The EDIT screen is bounded by DC_MAXOBS_WEBEDIT rows and DC_MAXCELLS_WEBEDIT
+// cells (rows x columns), whichever is reached first, and the VIEW screen caps
+// at DC_MAXOBS_WEBVIEW rows.
 //
+// A selection over an EDIT limit is served TRUNCATED - the grid carries the
+// first rows that fit, and a warning triangle against the table name says why.
 // This spec pins those limits against the mock, which mirrors the shipped
 // defaults (250 rows / 200000 cells for EDIT, 2000 rows for VIEW).  It uses
 // two existing fixtures rather than new ones:
 //   MPE_X_SEARCH - 1000 rows x 9 cols, so it trips the ROW limit
-//   MPE_X_WIDE   - 200 rows x 1001 cols (200,400 cells), under the row limit
-//                  but over the CELL limit
+//   MPE_X_WIDE   - 200 rows x 1001 cols (200,200 cells), under the row limit
+//                  but over the CELL limit, so the row cap is 199
 export {}

 const hostUrl = Cypress.env('hosturl')
@@ -60,27 +62,51 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
    })
  })

-  it('2 | EDIT refuses a table over the row limit', () => {
+  it('2 | EDIT serves a table over the row limit, truncated, with a warning', () => {
    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
    openEdit()

-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 1000 rows x 9 columns is over the 250 row limit, so the grid carries the
+    // first 250 rows and the header says how many it is showing.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
      'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
+      '250 rows'
    )
+    cy.get('#hotTable .ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length.at.least', 1)
+
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXOBS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
  })

-  it('3 | EDIT refuses a table over the cell limit', () => {
+  it('3 | EDIT serves a table over the cell limit, truncated, with a warning', () => {
    openTableFromTree(libraryToOpenIncludes, 'mpe_x_wide')
    openEdit()

-    // 200 rows x 1002 columns (the 1001 stored columns plus the housekeeping
-    // delete column) = 200,400 cells, over the 200,000 cell limit, while the
-    // row count is under the 250 row limit.
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 200 rows x 1001 stored columns = 200,200 cells, over the 200,000 cell
+    // limit, while the row count is under the 250 row limit - so the cell
+    // limit sets the cap: floor(200000 / 1001) = 199 rows.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
      'contain.text',
-      'Selection is too wide (200 rows x 1002 cols) - please filter and try again!'
+      '199 rows'
    )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
  })

  it('4 | the EDIT limits follow MPE_CONFIG, not the shipped defaults', () => {
@@ -90,20 +116,29 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
    // all.  And MPE_CONFIG lives in the control library, so reading it from the
    // table's own library would fail for this table in particular.
    //
-    // The cell limit is the one to move: 7 rows x 18 columns is 126 cells, so
-    // at 110 the TESTDATA.MPE_X_NEW edit must be refused, where a hard-coded
-    // 200000 - or a config read that fails and falls back to it - lets it
-    // through.  110 is still above MPE_CONFIG's own 17 x 6 = 102, so the option
-    // can be put back afterwards.
+    // The cell limit is the one to move: 7 rows x 17 columns is 119 cells, so
+    // at 110 the TESTDATA.MPE_X_NEW edit must be truncated to
+    // floor(110 / 17) = 6 rows, where a hard-coded 200000 - or a config read
+    // that fails and falls back to it - lets it through.  110 is still above
+    // MPE_CONFIG's own 6 x 17 = 102, so the option can be put back afterwards.
    setConfigValue('DC_MAXCELLS_WEBEDIT', '110', () => {
      visitPage('view/data/TESTDATA.MPE_X_NEW')
      closeAbortModalIfPresent()
      openEdit()

-      cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
        'contain.text',
-        'Selection is too wide (7 rows x 18 cols) - please filter and try again!'
+        '6 rows'
      )
+      cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+        .should('be.visible')
+        .realHover()
+      cy.get('.editor-title clr-tooltip-content', {
+        timeout: longerCommandTimeout
+      })
+        .should('be.visible')
+        .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+        .should(($el) => expect($el.css('opacity')).to.eq('1'))
    })

    // Put the option back, so the rest of the estate sees the shipped default.
@@ -112,28 +147,31 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
      closeAbortModalIfPresent()
      openEdit()

-      // 126 cells under a 200000 cell limit: the editor loads, no abort.
-      cy.get('#hotTable .ht_master .htCore tbody tr', {
-        timeout: longerCommandTimeout
-      }).should('exist')
+      // 119 cells under a 200000 cell limit: all 7 rows load, no warning.
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+        'contain.text',
+        '7 rows'
+      )
+      cy.get('.truncation-warning').should('not.exist')
    })
  })

-  it('5 | the filter picker still lists the variables after the abort', () => {
+  it('5 | the warning says to filter, and filtering narrows the grid', () => {
    openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
    openEdit()

-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
-      'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
-    )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'Filter the table to work with the rest')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))

-    closeAbortModal()
-
-    // The abort returns no table, and the picker's variable list normally comes
-    // from the table's own columns - so without the public/getcols fallback the
-    // "please filter and try again" advice has nothing to pick from.  Every
-    // column of MPE_X_SEARCH must be on offer.
+    // The advice has to be actionable: the picker must offer the table's
+    // columns and the filter must reload the grid.
    openFilterPopup()

    filterVariables().should(($options) => {
@@ -155,20 +193,20 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
      ])
    })

-    // and the suggestion list is usable, not just populated
    setFilterVariable('SITE_NAME')
-
-    // Submitting is what exercises TYPE.  The picker keys its operator set and
-    // its value quoting on it, and the fallback list never passes through
-    // mergeColsRules, so without TYPE on this payload a character value would be
-    // submitted unquoted and the backend's %mp_filtercheck would reject it.
-    setFilterValue('Bristol')
+    // 100 of the 1000 rows: under the limit, so nothing is truncated.
+    setFilterValue('Nereid Trench')
    submitFilter()

    cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
      'contain.text',
-      "SITE_NAME = 'Bristol'"
+      "SITE_NAME = 'Nereid Trench'"
    )
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      '100 rows'
+    )
+    cy.get('.truncation-warning').should('not.exist')
  })
 })

@@ -185,13 +223,6 @@ const openEdit = () => {
  }).click()
 }

-/** The abort modal offers Close, and the filter button sits behind it. */
-const closeAbortModal = () => {
-  cy.get('app-info-modal', { timeout: longerCommandTimeout })
-    .contains('button', 'Close')
-    .click()
-}
-
 /**
 * The EDIT screen's Filter button opens the same .filter-modal the viewer uses.
 * It only renders once the editor is in its read-only (non-editing) state,
@@ -0,0 +1,81 @@
+// Marks this file as an ES module so its top-level consts don't collide with
+// other spec files under the TS type-checker.
+export {}
+//
+// When the startup service fails without a usable response, the error modal
+// offers a button that runs the same service in a new browser window, where the
+// SAS log is returned instead. This spec forces a startup failure against the
+// mock backend and checks the button and the URL it opens.
+const hostUrl = Cypress.env('hosturl')
+
+const STARTUP_PROGRAM = '/Public/app/dc/services/public/startupservice'
+
+/** The body a Viya Compute task returns when the run outlives the
+ * execution-task timeout: a bare string, so the adapter has no webout and no
+ * log to report. */
+const rawJobError = [
+  'Job error',
+  'The Compute service could not execute the task c74a707a-8b88-46a5-8f28-9a7694ad5e13 because the context is not reusable.',
+  'path: /compute/tasks',
+  'correlator: dc1a9fda-8f65-4f32-b4d4-2a70e02179b1;1bcb6a4c-8a16-4053-a790-292544c25a03'
+].join('\n')
+
+context('startupservice launcher: ', function () {
+  it('1 | runs the startupservice in a new window when its response is malformed', () => {
+    // Report an empty admin folder so the app loads the startup service: on a
+    // freshly seeded mock estate makedata is still present, which sends the app
+    // to the deploy screen instead.
+    cy.intercept('GET', '**/drive/folder**', (req) => {
+      // Guarded parse: a lone '%' in the URL would make decodeURIComponent
+      // throw, and an exception here aborts the request.
+      let url = req.url
+      try {
+        url = decodeURIComponent(req.url)
+      } catch (e) {
+        url = req.url
+      }
+
+      if (url.includes('services/admin')) {
+        req.reply({ files: [], folders: [] })
+        return
+      }
+      req.continue()
+    })
+
+    cy.intercept('**/stp/execute/**', (req) => {
+      if (req.url.includes('public/startupservice')) {
+        // The adapter cannot use a bare string as a response payload, so the
+        // startup request fails with nothing to show.
+        req.reply({
+          statusCode: 200,
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify(rawJobError)
+        })
+        return
+      }
+      req.continue()
+    })
+
+    // The launcher must not actually open a tab under test.
+    cy.visit(hostUrl, {
+      timeout: 60000,
+      onBeforeLoad(win) {
+        cy.stub(win, 'open').as('windowOpen')
+      }
+    })
+
+    cy.get('.clr-abort-modal', { timeout: 60000 }).should('be.visible')
+    cy.get('.clr-abort-modal').last().should('contain.text', 'startupservice')
+    cy.get('.clr-abort-modal').last().contains('button', 'Show log').click()
+
+    cy.get('@windowOpen').should('have.been.calledOnce')
+    cy.get('@windowOpen').then((stub: any) => {
+      const url = stub.firstCall.args[0]
+
+      expect(url).to.contain(STARTUP_PROGRAM)
+      expect(url).to.contain('_debug=131')
+      // The browser route is what makes the log available: no execution task.
+      expect(url).not.to.contain('_executionTasks')
+    })
+  })
+})
@@ -41,22 +41,22 @@
  },
  "private": true,
  "dependencies": {
-    "@angular/animations": "20.3.33",
+    "@angular/animations": "20.3.34",
    "@angular/cdk": "^20.2.14",
-    "@angular/common": "20.3.33",
-    "@angular/compiler": "20.3.33",
-    "@angular/core": "20.3.33",
-    "@angular/forms": "20.3.33",
-    "@angular/platform-browser": "20.3.33",
-    "@angular/platform-browser-dynamic": "20.3.33",
-    "@angular/router": "20.3.33",
+    "@angular/common": "20.3.34",
+    "@angular/compiler": "20.3.34",
+    "@angular/core": "20.3.34",
+    "@angular/forms": "20.3.34",
+    "@angular/platform-browser": "20.3.34",
+    "@angular/platform-browser-dynamic": "20.3.34",
+    "@angular/router": "20.3.34",
    "@cds/core": "^6.15.1",
    "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
    "@clr/icons": "^13.0.2",
    "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
    "@handsontable/angular-wrapper": "18.1.1",
-    "@sasjs/adapter": "4.19.1",
-    "@sasjs/utils": "^3.6.0",
+    "@sasjs/adapter": "5.1.2",
+    "@sasjs/utils": "4.0.0",
    "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
    "@types/d3-graphviz": "^2.6.7",
    "@types/text-encoding": "0.0.35",
@@ -74,7 +74,6 @@
    "hyperformula": "^2.5.0",
    "iconv-lite": "^0.5.0",
    "jquery-datetimepicker": "^2.5.21",
-    "jsrsasign": "11.1.1",
    "marked": "^5.0.0",
    "moment": "^2.30.1",
    "ngx-clipboard": "^16.0.0",
@@ -100,7 +99,7 @@
    "@angular-eslint/schematics": "19.8.1",
    "@angular-eslint/template-parser": "19.8.1",
    "@angular/cli": "20.3.37",
-    "@angular/compiler-cli": "20.3.33",
+    "@angular/compiler-cli": "20.3.34",
    "@babel/plugin-proposal-private-methods": "^7.18.6",
    "@compodoc/compodoc": "^2.0.0",
    "@cypress/webpack-preprocessor": "^5.17.1",
@@ -148,7 +147,7 @@
    "ajv": "8.18.0",
    "uuid": "11.1.1",
    "lighthouse": "13.4.0",
-    "fast-uri": "3.1.8",
+    "fast-uri": "4.2.3",
    "readdir-glob": {
      "brace-expansion": "^5.0.9"
    },
@@ -245,6 +245,25 @@
              }
            </clr-tooltip>

+            @if (truncationNote) {
+              <clr-tooltip>
+                <clr-icon
+                  clrTooltipTrigger
+                  shape="warning"
+                  aria-label="The selection was truncated"
+                  class="color-orange truncation-warning"
+                  size="20"
+                ></clr-icon>
+                <clr-tooltip-content
+                  clrPosition="right"
+                  clrSize="lg"
+                  *clrIfOpen
+                >
+                  {{ truncationNote }}
+                </clr-tooltip-content>
+              </clr-tooltip>
+            }
+
            @if (dataSource) {
              @if (!zeroFilterRows) {
                ({{ dataSource.length | thousandSeparator: ',' }}
@@ -89,6 +89,11 @@ import { withoutEditStatus } from './utils/withoutEditStatus'
 import { getEditStatusSymbol } from './utils/getEditStatusSymbol'
 import { normalizeSortConfig } from './utils/normalizeSortConfig'
 import { EDIT_STATUS_COLUMN_NAME } from '../shared/dc-validator/utils/editStatusColumnRule'
+import { DELETE_RECORD_COLUMN_NAME } from '../shared/dc-validator/utils/deleteRecordColumnRule'
+import {
+  prepareDeleteRecordColumn,
+  seedDeleteRecordColumn
+} from '../shared/dc-validator/utils/deleteRecordColumn'
 import {
  errorRenderer,
  noSpinnerRenderer,
@@ -155,6 +160,9 @@ export class EditorComponent
  dsmeta: DSMeta[] = []
  versions: Version[] = []
  dsNote = ''
+  /** Set when getdata served a truncated selection - shown as a warning
+   * against the table name, with the reason in a tooltip. */
+  truncationNote = ''

  viewboxes = false

@@ -1355,8 +1363,14 @@ export class EditorComponent
            colSpecs.TYPE === 'num' &&
            !colSpecs.DDTYPE.includes('TIME') &&
            !colSpecs.DDTYPE.includes('DATE')
-          )
-            row[colKey] = parseInt(row[colKey])
+          ) {
+            const numericValue = parseFloat(row[colKey])
+
+            // A cell the file parser could not read as a number stays as it
+            // was: the editor's numeric validation then flags it as text,
+            // where a NaN would render as an empty cell.
+            if (!Number.isNaN(numericValue)) row[colKey] = numericValue
+          }
        }
      }
    }
@@ -3519,14 +3533,24 @@ export class EditorComponent
          // A synchronous throw inside initSetup (called from the .then()
          // above) lands here too, not just a network failure - log it, or
          // the table just renders hidden with no clue why.
-          // eslint-disable-next-line no-console
-          console.error('editors/getdata failed:', err)
-          this.getdataError = true
-          this.tableTrue = true
+          this.handleGetdataFailure(err)
        })
    }
  }

+  /**
+   * Puts the page into its getdata error state. Shared by the callService
+   * .catch and initSetup's own retry branch: a throw from the retry runs on a
+   * bare setTimeout, outside that promise chain, so it would otherwise never
+   * reach the .catch and the user would be left on the loading spinner.
+   */
+  private handleGetdataFailure(err: any) {
+    // eslint-disable-next-line no-console
+    console.error('editors/getdata failed:', err)
+    this.getdataError = true
+    this.tableTrue = true
+  }
+
  /**
   * VA mode, initial (unfiltered) route only: briefly wait for VA's buffered
   * first message and, if it already carries a filter, save that filter and
@@ -3875,13 +3899,28 @@ export class EditorComponent

    this.hotInstance = this.hotTableComponent?.hotInstance!
    if (!this.hotInstance) {
-      // Retry init, don't permanently abort
+      // Retry init, don't permanently abort. A throw from the retry runs on a
+      // bare setTimeout rather than in the caller's promise chain, so it would
+      // never reach that .catch - surface it through the same handler here.
      if (attempt < 60) {
-        setTimeout(() => this.initSetup(response, attempt + 1), 50)
+        setTimeout(() => {
+          try {
+            this.initSetup(response, attempt + 1)
+          } catch (err) {
+            this.handleGetdataFailure(err)
+          }
+        }, 50)
      }
      return
    }

+    // The delete column is client-owned (see deleteRecordColumnRule.ts):
+    // this asserts getdata did not send it, then prepends it to COLHEADERS so
+    // it is the grid's first column. A throw here lands in the caller's
+    // .catch - or, on a retry, the branch above - which sets getdataError and
+    // shows the error state.
+    prepareDeleteRecordColumn(response)
+
    this.cols = response.data.cols
    this.dsmeta = response.data.dsmeta
    this.versions = response.data.versions || []
@@ -3900,6 +3939,11 @@ export class EditorComponent
      this.dsNote = ''
    }

+    // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
+    // truncated to the first rows within the limit, and carries the reason
+    // here.  Empty means the whole selection is in the grid.
+    this.truncationNote = response.data.sasparams[0].TRUNC_MSG || ''
+
    const hot = this.hotInstance

    const approvers: Approver[] = response.data.approvers
@@ -3931,9 +3975,9 @@ export class EditorComponent

    this.checkEmptyRowWhenFilter()

-    if (this.headerColumns.indexOf('_____DELETE__THIS__RECORD_____') !== -1) {
+    if (this.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME) !== -1) {
      this.headerColumns[
-        this.headerColumns.indexOf('_____DELETE__THIS__RECORD_____')
+        this.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME)
      ] = 'Delete?'
    }

@@ -3998,6 +4042,10 @@ export class EditorComponent
      row[EDIT_STATUS_COLUMN_NAME] = 'U'
    }

+    // The delete flag is client-owned too (see deleteRecordColumnRule.ts) -
+    // getdata no longer sends the column, so every row starts unmarked.
+    seedDeleteRecordColumn(this.dataSource)
+
    // Seed HARDFORMULA/SOFTFORMULA columns with their computed formula
    // string per row. dataSourceUnchanged isn't populated yet this early in
    // a fresh load (only the excel-preview/add-row flows set it) - at this
@@ -47,6 +47,10 @@ export interface SASParam {
  PKCNT: number
  RK_FLAG: number
  TMVARS: string
+  /** 1 when getdata cut the selection down to the row / cell limit */
+  TRUNCATED?: number
+  /** Reason shown in the tooltip when TRUNCATED is 1, else empty */
+  TRUNC_MSG?: string
 }

 export interface ExcelRule {
@@ -18,6 +18,11 @@ import {
 import { globals } from '../_globals'
 import { EditorsGetDataServiceResponse } from '../models/sas/editors-getdata.model'
 import { DcValidator } from '../shared/dc-validator/dc-validator'
+import {
+  prepareDeleteRecordColumn,
+  seedDeleteRecordColumn
+} from '../shared/dc-validator/utils/deleteRecordColumn'
+import { DELETE_RECORD_COLUMN_NAME } from '../shared/dc-validator/utils/deleteRecordColumnRule'
 import { ExcelRule } from '../models/TableData'
 import { HotTableInterface } from '../models/HotTable.interface'
 import { Col } from '../shared/dc-validator/models/col.model'
@@ -360,7 +365,21 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {

      this.uploadLoading = false

-      const datasetObjects = this.buildDatasetsObjects(datasets)
+      // buildDatasetsObjects throws when a getdata response carries the
+      // reserved delete-record column (see deleteRecordColumn.ts). This chain
+      // has no .catch, so an uncaught throw here would become an unhandled
+      // rejection that aborts the loop with nothing shown. Surface it on
+      // screen, the way the editor flow refuses the same response, rather
+      // than leaving the user on an empty table selection with the file
+      // still attached.
+      let datasetObjects: DatasetsObject[] = []
+      try {
+        datasetObjects = this.buildDatasetsObjects(datasets)
+      } catch (err) {
+        console.warn('Error building dataset objects', err)
+        this.eventService.showInfoModal('Upload error', (err as Error).message)
+        return
+      }

      datasetObjects.forEach((datasetObject) => {
        this.parsedDatasets.push({
@@ -1074,6 +1093,12 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {

    for (let response of responses) {
      if (response.data) {
+        // The delete column is client-owned (see deleteRecordColumnRule.ts):
+        // asserts getdata did not send it, then prepends it to COLHEADERS so
+        // it is index 0 for everything below.
+        prepareDeleteRecordColumn(response)
+        seedDeleteRecordColumn(response.data.sasdata)
+
        const datasetObject: DatasetsObject = {
          ...response,
          hotTable: {
@@ -1097,14 +1122,10 @@ export class MultiDatasetComponent implements OnInit, AfterViewInit {
        datasetObject.headerPks = response.data.sasparams[0].PK.split(' ')

        if (
-          datasetObject.headerColumns.indexOf(
-            '_____DELETE__THIS__RECORD_____'
-          ) !== -1
+          datasetObject.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME) !== -1
        ) {
          datasetObject.headerColumns[
-            datasetObject.headerColumns.indexOf(
-              '_____DELETE__THIS__RECORD_____'
-            )
+            datasetObject.headerColumns.indexOf(DELETE_RECORD_COLUMN_NAME)
          ] = 'Delete?'
        }

@@ -0,0 +1,110 @@
+import { QueryComponent } from './query.component'
+import { HelperService } from '../services/helper.service'
+
+/**
+ * dynamicWhereClauseCreator builds the prefilter that getcolvals sends when the
+ * dynamic where clause is on, and it composes a temporal value into the same
+ * SAS literal the applied filter carries.  The value it prefilters with comes
+ * from the *other* clauses in the query, so the clause under test sits at index
+ * 1 and index 0 is skipped.
+ */
+describe('QueryComponent - dynamic where clause', () => {
+  const buildComponent = (elements: any[]) => {
+    const component = new QueryComponent(
+      {} as any,
+      { catchResponseError: () => undefined } as any,
+      new HelperService(null as any),
+      { detectChanges: () => undefined } as any
+    )
+
+    component.clauses = {
+      queryObj: [{ clauseLogic: 'AND', elements }],
+      groupLogic: 'AND'
+    } as any
+
+    return component
+  }
+
+  const temporalClause = (over: any = {}) => ({
+    variable: 'SOME_DATE',
+    operator: '=',
+    value: '',
+    values: [],
+    type: 'num',
+    ddtype: 'DATE',
+    valueVariable: false,
+    ...over
+  })
+
+  it('sends an IN entry as the literal for its unformatted value', () => {
+    const component = buildComponent([
+      temporalClause(),
+      temporalClause({
+        operator: 'IN',
+        value: [{ checked: true, val: '12FEB1960' }],
+        values: [{ formatted: '12FEB1960', unformatted: 42 }]
+      })
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table.length).toBe(1)
+    expect(table[0].RAW_VALUE).toBe("('12FEB1960'd)")
+  })
+
+  it('maps both ends of a BETWEEN picked from the value list', () => {
+    const component = buildComponent([
+      temporalClause(),
+      temporalClause({
+        operator: 'BETWEEN',
+        value: ['12FEB1960', '13FEB1960'],
+        values: [
+          { formatted: '12FEB1960', unformatted: 42 },
+          { formatted: '13FEB1960', unformatted: 43 }
+        ]
+      })
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table.length).toBe(1)
+    expect(table[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  it('leaves an already-unformatted BETWEEN alone', () => {
+    const component = buildComponent([
+      temporalClause(),
+      temporalClause({
+        operator: 'BETWEEN',
+        value: [42, 43],
+        values: [
+          { formatted: '12FEB1960', unformatted: 42 },
+          { formatted: '13FEB1960', unformatted: 43 }
+        ]
+      })
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  it('emits the unformatted value in the non-literal IN fallback', () => {
+    const component = buildComponent([
+      temporalClause(),
+      {
+        variable: 'SOME_CHAR',
+        operator: 'IN',
+        value: [{ checked: true, val: 'ABC ' }],
+        values: [{ formatted: 'ABC ', unformatted: 'ABC' }],
+        type: 'char',
+        ddtype: 'CHAR',
+        valueVariable: false
+      }
+    ])
+
+    const table = component.dynamicWhereClauseCreator(0, 0)
+
+    expect(table[0].RAW_VALUE).toBe("('ABC')")
+  })
+})
@@ -425,35 +425,57 @@ export class QueryComponent
            continue

          let rawValue = ''
-          const operator =
-            this.clauses.queryObj[groupInd].elements[clauseInd].operator
-          const variable =
-            this.clauses.queryObj[groupInd].elements[clauseInd].variable
+          const element = this.clauses.queryObj[groupInd].elements[clauseInd]
+          const operator = element.operator
+          const variable = element.variable
+          /* a temporal value is sent as a SAS literal (eg '20APR2042'd), the
+             same way the applied filter carries it */
+          const temporalValue = (value: any) =>
+            element.valueVariable
+              ? null
+              : this.helperService.sasTemporalLiteral(value, element.ddtype)
+          /* the dropdowns and the IN modal hold the *formatted* value (the
+             soft-select option value, and setINValues), so a picked value is
+             mapped back to the unformatted one the filter needs */
+          const unformattedValue = (value: any) =>
+            (element.values || []).find((v: any) => v.formatted === value)
+              ?.unformatted ?? value

          if (operator === 'BETWEEN') {
-            rawValue = `${this.clauses.queryObj[groupInd].elements[clauseInd].value[0]} AND ${this.clauses.queryObj[groupInd][clauseInd].value[1]}`
+            const from = unformattedValue(element.value[0])
+            const to = unformattedValue(element.value[1])
+            rawValue = `${temporalValue(from) ?? from} AND ${
+              temporalValue(to) ?? to
+            }`
          } else if (operator === 'IN' || operator === 'NOT IN') {
-            this.clauses.queryObj[groupInd][clauseInd].value.forEach(
-              (element: any) => {
-                if (element.checked) {
-                  let value = element.val
+            element.value.forEach((item: any) => {
+              if (item.checked) {
+                /* map once, and use the unformatted value in every branch - the
+                   literal is built from it, and so are the fallbacks, so a
+                   formatted label can never reach RAW_VALUE */
+                const value = unformattedValue(item.val)
+                const literal = temporalValue(value)

-                  if (typeof value === 'string' && !isSpecialMissing(value)) {
-                    if (rawValue.length > 0) {
-                      rawValue += `,'${value}'`
-                    } else {
-                      rawValue = `('${value}'`
-                    }
+                if (literal) {
+                  rawValue += (rawValue.length > 0 ? ',' : '(') + literal
+                } else if (
+                  typeof value === 'string' &&
+                  !isSpecialMissing(value)
+                ) {
+                  if (rawValue.length > 0) {
+                    rawValue += `,'${value}'`
                  } else {
-                    if (rawValue.length > 0) {
-                      rawValue += `,${value}`
-                    } else {
-                      rawValue = `(${value}`
-                    }
+                    rawValue = `('${value}'`
+                  }
+                } else {
+                  if (rawValue.length > 0) {
+                    rawValue += `,${value}`
+                  } else {
+                    rawValue = `(${value}`
                  }
                }
              }
-            )
+            })

            if (rawValue.length > 0) {
              rawValue += ')'
@@ -470,11 +492,12 @@ export class QueryComponent
              rawValue = '.'
            }
          } else {
-            if (
-              this.clauses.queryObj[groupInd].elements[clauseInd].type ===
-              'char'
-            ) {
-              rawValue = `'${this.clauses.queryObj[groupInd].elements[clauseInd].value}'`
+            const literal = temporalValue(element.value)
+
+            if (literal) {
+              rawValue = literal
+            } else if (element.type === 'char') {
+              rawValue = `'${element.value}'`
            }
          }

@@ -170,9 +170,11 @@ describe('AppService - startup retry', () => {

    expect(deps.sasService.request).toHaveBeenCalledTimes(1)
    expect(deps.eventService.showInfoModal).toHaveBeenCalledTimes(1)
-    const [, message] = deps.eventService.showInfoModal.calls.mostRecent().args
+    const [, message, sasService] =
+      deps.eventService.showInfoModal.calls.mostRecent().args
    expect(message).toContain(rawJobError)
    expect(message).not.toContain('Globvars, Sasdatasets, Saslibs, XLMaps')
+    expect(sasService).toBe('public/startupservice')
    expect(deps.licenceService.isAppActivated.value).toBeFalse()
  })
 })
@@ -107,7 +107,11 @@ export class AppService {
            malformedMessage
          )
          this.startupCheckService.updateStep(2, 'error', 'Malformed response')
-          this.eventService.showInfoModal('Error', malformedMessage)
+          this.eventService.showInfoModal(
+            'Error',
+            malformedMessage,
+            'public/startupservice'
+          )
          this.licenceService.isAppActivated.next(false)

          return
@@ -139,7 +143,8 @@ export class AppService {
          )
          this.eventService.showInfoModal(
            'Error',
-            `${missingProps.join(', ')} are not present in the startupservice`
+            `${missingProps.join(', ')} are not present in the startupservice`,
+            'public/startupservice'
          )
          this.licenceService.isAppActivated.next(false)

@@ -221,7 +226,8 @@ export class AppService {
        this.startupCheckService.updateStep(2, 'error', 'Request failed')
        this.eventService.showInfoModal(
          'Error',
-          'There is an issue with startupservice response'
+          'There is an issue with startupservice response',
+          'public/startupservice'
        )
        this.licenceService.isAppActivated.next(false)
      })
@@ -42,11 +42,15 @@ export class EventService {
    this.onDemoLimitModalShow.emit(featureName)
  }

-  public showInfoModal(modalTitle: string = 'Info', message: string) {
+  public showInfoModal(
+    modalTitle: string = 'Info',
+    message: string,
+    sasService: string | null = null
+  ) {
    let infoModal: InfoModal = {
      modalTitle,
      message,
-      sasService: '',
+      sasService,
      details: null
    }

@@ -46,3 +46,72 @@ describe('HelperService - convertArrayValues', () => {
    expect(result.length).toBe(3)
  })
 })
+
+describe('HelperService - sasTemporalLiteral', () => {
+  let service: HelperService
+
+  beforeEach(() => {
+    service = new HelperService(null as any)
+  })
+
+  it('renders a SAS date value as a date literal', () => {
+    expect(service.sasTemporalLiteral(42, 'DATE')).toBe("'12FEB1960'd")
+    expect(service.sasTemporalLiteral(0, 'DATE')).toBe("'01JAN1960'd")
+    expect(service.sasTemporalLiteral(34322, 'DATE')).toBe("'20DEC2053'd")
+  })
+
+  it('renders a date before 1960', () => {
+    expect(service.sasTemporalLiteral(-1, 'DATE')).toBe("'31DEC1959'd")
+  })
+
+  it('renders a SAS datetime value as a datetime literal', () => {
+    expect(service.sasTemporalLiteral(42, 'DATETIME')).toBe(
+      "'01JAN1960:00:00:42'dt"
+    )
+    expect(service.sasTemporalLiteral(86400, 'DATETIME')).toBe(
+      "'02JAN1960:00:00:00'dt"
+    )
+    expect(service.sasTemporalLiteral(31536001, 'DATETIME')).toBe(
+      "'31DEC1960:00:00:01'dt"
+    )
+  })
+
+  it('rounds double noise onto the second', () => {
+    expect(service.sasTemporalLiteral(1.9999999999999, 'TIME')).toBe(
+      "'00:00:02't"
+    )
+    expect(service.sasTemporalLiteral(42.000000000000004, 'DATETIME')).toBe(
+      "'01JAN1960:00:00:42'dt"
+    )
+  })
+
+  it('returns null rather than round a genuinely fractional value', () => {
+    expect(service.sasTemporalLiteral(42.5, 'DATETIME')).toBeNull()
+    expect(service.sasTemporalLiteral(42.5, 'TIME')).toBeNull()
+    expect(service.sasTemporalLiteral(42.3, 'TIME')).toBeNull()
+  })
+
+  it('renders a SAS time value as a time literal', () => {
+    expect(service.sasTemporalLiteral(0, 'TIME')).toBe("'00:00:00't")
+    expect(service.sasTemporalLiteral(42, 'TIME')).toBe("'00:00:42't")
+    expect(service.sasTemporalLiteral(3661, 'TIME')).toBe("'01:01:01't")
+  })
+
+  it('accepts a value that arrives as a string', () => {
+    expect(service.sasTemporalLiteral('42', 'DATE')).toBe("'12FEB1960'd")
+  })
+
+  it('returns null when the value cannot be a literal', () => {
+    // not temporal
+    expect(service.sasTemporalLiteral(42, 'N')).toBeNull()
+    expect(service.sasTemporalLiteral(42, 'C')).toBeNull()
+    expect(service.sasTemporalLiteral(42, '')).toBeNull()
+    // not a number
+    expect(service.sasTemporalLiteral('12FEB1960', 'DATE')).toBeNull()
+    expect(service.sasTemporalLiteral('', 'DATE')).toBeNull()
+    expect(service.sasTemporalLiteral(null, 'DATE')).toBeNull()
+    expect(service.sasTemporalLiteral(undefined, 'DATE')).toBeNull()
+    // a fractional number of days is not a SAS date
+    expect(service.sasTemporalLiteral(42.5, 'DATE')).toBeNull()
+  })
+})
@@ -5,6 +5,22 @@ import { SasService } from './sas.service'

 const librariesToShow = 50

+/** Month names used by a SAS date literal, eg 20APR2042. */
+const sasMonths = [
+  'JAN',
+  'FEB',
+  'MAR',
+  'APR',
+  'MAY',
+  'JUN',
+  'JUL',
+  'AUG',
+  'SEP',
+  'OCT',
+  'NOV',
+  'DEC'
+]
+
@Injectable({
  providedIn: 'root'
 })
@@ -226,6 +242,70 @@ export class HelperService {
    })
  }

+  /**
+   * Formats a SAS internal date / datetime / time value as the SAS literal a
+   * WHERE clause can carry, eg:
+   *   34322 (DATE)     -> '20DEC2053'd
+   *   42    (DATETIME) -> '01JAN1960:00:00:42'dt
+   *   42    (TIME)     -> '00:00:42't
+   *
+   * The value is days since 1960-01-01 for a date, and seconds since
+   * 1960-01-01 (datetime) or since midnight (time).  The date part is built
+   * arithmetically in UTC, so the day cannot shift with the browser timezone.
+   *
+   * SAS temporal values are whole days and whole seconds, so the value is
+   * rounded to seconds first - that kills the noise a double picks up on the
+   * way here, where 1.9999999999999 is simply the second 2.  A value that is
+   * genuinely fractional does not land on a second, and rounding it would
+   * silently widen the filter, so it returns null instead.
+   *
+   * Returns null when the value is not a finite number, is not a whole day or
+   * second, or the ddtype is not temporal - callers then send the raw value.
+   */
+  public sasTemporalLiteral(value: any, ddtype: string): string | null {
+    const kind = (ddtype ?? '').toString().toUpperCase()
+    if (kind !== 'DATE' && kind !== 'DATETIME' && kind !== 'TIME') return null
+    if (value === null || value === undefined || value === '') return null
+
+    const num = typeof value === 'number' ? value : Number(String(value).trim())
+    if (!isFinite(num)) return null
+
+    /* Within the precision of a double, is this a whole second? */
+    const seconds = Math.round(num * 1e8) / 1e8
+    if (!Number.isInteger(seconds)) return null
+
+    if (kind === 'TIME') return `'${this.sasTimeFromSeconds(seconds)}'t`
+
+    const days = kind === 'DATETIME' ? Math.floor(seconds / 86400) : seconds
+    const datePart = this.sasDateFromDays(days)
+
+    if (kind === 'DATE') return `'${datePart}'d`
+
+    return `'${datePart}:${this.sasTimeFromSeconds(seconds - days * 86400)}'dt`
+  }
+
+  /** `ddMMMyyyy` for a SAS date value (days since 1960-01-01). */
+  private sasDateFromDays(days: number): string {
+    const date = new Date(Date.UTC(1960, 0, 1) + days * 86400000)
+
+    return (
+      this.addLeadingZero(date.getUTCDate()) +
+      sasMonths[date.getUTCMonth()] +
+      date.getUTCFullYear()
+    )
+  }
+
+  /** `hh:mm:ss` for a SAS time value (whole seconds since midnight). */
+  private sasTimeFromSeconds(seconds: number): string {
+    return (
+      this.addLeadingZero(Math.floor(seconds / 3600)) +
+      ':' +
+      this.addLeadingZero(Math.floor((seconds % 3600) / 60)) +
+      ':' +
+      this.addLeadingZero(seconds % 60)
+    )
+  }
+
  public addLeadingZero(value: string | number): string {
    if (typeof value !== 'string') value = value.toString()

@@ -142,7 +142,8 @@ export class LicenceService {
      this.isAppActivated.next(false)
      this.eventService.showInfoModal(
        'Error',
-        'Some of the globvars are not present in the startupservice'
+        'Some of the globvars are not present in the startupservice',
+        'public/startupservice'
      )
      // this.router.navigateByUrl('/licensing/key?error=missing')
      return
@@ -1,4 +1,5 @@
 import { SasStoreService } from './sas-store.service'
+import { HelperService } from './helper.service'

 /**
 * The filter picker keys its operator set, its variable highlight and its value
@@ -116,3 +117,197 @@ describe('SasStoreService', () => {
    })
  })
 })
+
+/**
+ * The clause shown in the filter panel is composed by whereClauseCreator, and
+ * its temporal rewrite turns the internal day / second count into a SAS
+ * literal.  It may only do that when the composed value really is nothing but
+ * those numbers - a value typed by hand is not, and rewriting the digit runs
+ * inside it produced clauses like '13JAN1960'd/'03JAN1960'd/'14MAY1965'd for
+ * a date typed as 12/02/1960.
+ */
+describe('SasStoreService - temporal clause preview', () => {
+  const buildStore = () =>
+    new SasStoreService({} as any, new HelperService(null as any), {} as any)
+
+  const clauseFor = (
+    value: any,
+    operator: string,
+    ddtype: string,
+    values: any[] = []
+  ) => ({
+    queryObj: [
+      {
+        clauseLogic: '',
+        elements: [
+          {
+            variable: 'SOME_DATE',
+            operator,
+            value,
+            type: 'num',
+            ddtype,
+            valueVariable: false,
+            values
+          }
+        ]
+      }
+    ],
+    groupLogic: ''
+  })
+
+  it('leaves a typed temporal value alone when it is not a number', () => {
+    const store = buildStore()
+
+    const result = store.whereClauseCreator(
+      clauseFor('12/02/1960', '=', 'DATE'),
+      '',
+      'lib.ds'
+    )
+
+    expect(result.whereClause).toContain('12/02/1960')
+    expect(result.whereClause).not.toContain("'d")
+  })
+
+  it('still shows an internal day count as a date literal', () => {
+    const store = buildStore()
+
+    const result = store.whereClauseCreator(
+      clauseFor('42', '=', 'DATE'),
+      '',
+      'lib.ds'
+    )
+
+    expect(result.whereClause).toContain("'12FEB1960'd")
+  })
+
+  /**
+   * A BETWEEN pair is composed as `<from> AND <to>`, so the guard that decides
+   * whether the rewrite may run has to see past the separator - its letters are
+   * not part of the numbers the value is built from.
+   */
+  it('shows a BETWEEN pair picked from the value list as literals', () => {
+    const store = buildStore()
+
+    const result = store.whereClauseCreator(
+      clauseFor(['12FEB1960', '13FEB1960'], 'BETWEEN', 'DATE', [
+        { formatted: '12FEB1960', unformatted: '42' },
+        { formatted: '13FEB1960', unformatted: '43' }
+      ]),
+      '',
+      'lib.ds'
+    )
+
+    expect(result.whereClause).toContain(
+      "BETWEEN '12FEB1960'd AND '13FEB1960'd"
+    )
+  })
+})
+
+/**
+ * createFilterQueryTable builds the RAW_VALUE the backend validates and stores.
+ * The range dropdown binds the formatted label, so a BETWEEN has to map each
+ * end back to its unformatted value before the literal is built - otherwise
+ * sasTemporalLiteral returns null and an unquoted date reaches the backend.
+ */
+describe('SasStoreService - temporal RAW_VALUE', () => {
+  const buildStore = () =>
+    new SasStoreService({} as any, new HelperService(null as any), {} as any)
+
+  const betweenQuery = (value: any[], values: any[]) => ({
+    groupLogic: 'AND',
+    filterGroups: [
+      {
+        clauseLogic: 'AND',
+        filterClauses: [
+          {
+            ddtype: 'DATE',
+            invalidClause: false,
+            logic: '',
+            operator: 'BETWEEN',
+            operators: [],
+            type: 'num',
+            value,
+            valueVariable: false,
+            values,
+            variable: 'SOME_DATE'
+          }
+        ]
+      }
+    ]
+  })
+
+  it('maps a BETWEEN picked from the value list back to the internal value', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      betweenQuery(
+        ['12FEB1960', '13FEB1960'],
+        [
+          { formatted: '12FEB1960', unformatted: 42 },
+          { formatted: '13FEB1960', unformatted: 43 }
+        ]
+      ) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  it('still builds literals when the values are already internal counts', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      betweenQuery(['42', '43'], []) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("'12FEB1960'd AND '13FEB1960'd")
+  })
+
+  const inQuery = (value: any[], values: any[]) => ({
+    groupLogic: 'AND',
+    filterGroups: [
+      {
+        clauseLogic: 'AND',
+        filterClauses: [
+          {
+            ddtype: 'DATE',
+            invalidClause: false,
+            logic: '',
+            operator: 'IN',
+            operators: [],
+            type: 'num',
+            value,
+            valueVariable: false,
+            values,
+            variable: 'SOME_DATE'
+          }
+        ]
+      }
+    ]
+  })
+
+  it('maps an IN entry picked from the value list back to the internal value', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      inQuery(
+        [{ checked: true, val: '12FEB1960' }],
+        [{ formatted: '12FEB1960', unformatted: 42 }]
+      ) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("('12FEB1960'd)")
+  })
+
+  it('passes an already-unformatted IN entry through unchanged', () => {
+    const store = buildStore()
+
+    const clauses = store.createFilterQueryTable(
+      inQuery(
+        [{ checked: true, val: 42 }],
+        [{ formatted: '12FEB1960', unformatted: 42 }]
+      ) as any
+    )
+
+    expect(clauses[0].RAW_VALUE).toBe("('12FEB1960'd)")
+  })
+})
@@ -596,6 +596,27 @@ export class SasStoreService {
        }
        oper = this.checkOperator(oper, value, type).operator

+        /* A temporal value is shown as a SAS literal (eg '20DEC2053'd) rather
+           than the internal day / second count.  Only rewrite when the composed
+           value is nothing but the numbers the clause is built from - a single
+           value, a BETWEEN pair or an IN list.  The BETWEEN separator is
+           removed before the test because ' AND ' is not part of that character
+           set; a typed value that is not a number, such as a date entered by
+           hand, would otherwise have every digit run inside it rewritten into
+           an unrelated literal. */
+        if (
+          !isValueVariable &&
+          ['DATE', 'DATETIME', 'TIME'].includes(query.ddtype) &&
+          /^[-0-9.,()\s]+$/.test(String(value).replace(/\s+AND\s+/gi, ' '))
+        ) {
+          value = String(value).replace(
+            /-?\d+(?:\.\d+)?/g,
+            (match) =>
+              this.helperService.sasTemporalLiteral(match, query.ddtype) ??
+              match
+          )
+        }
+
        if (type === 'char' && oper !== 'IN' && oper !== 'NOT IN') {
          if (typeof value === 'undefined') {
            value = ''
@@ -662,17 +683,53 @@ export class SasStoreService {
    filterQuery.filterGroups.forEach(
      (filterGroup: FilterGroup, groupIndex: number) => {
        filterGroup.filterClauses.forEach((filterClause: FilterClause) => {
+          /* A temporal value is sent as a SAS literal (eg '20APR2042'd) rather
+             than the internal day / second count, so the resolved WHERE clause
+             - and the FILTER_TEXT shown above the grid - carries a date. */
+          const temporalValue = (value: any) =>
+            filterClause.valueVariable
+              ? null
+              : this.helperService.sasTemporalLiteral(
+                  value,
+                  filterClause.ddtype
+                )
+
+          /* The range dropdown binds the formatted label, as the value list
+             and the IN modal do, so each end of a BETWEEN is mapped back to
+             its unformatted value before the literal is built.  This is the
+             same lookup whereClauseCreator applies for the preview - without
+             it the formatted string reaches sasTemporalLiteral, comes back
+             null, and RAW_VALUE carries an unquoted date that the backend
+             rejects on a numeric column. */
+          const unformattedValue = (value: any) => {
+            const match = (filterClause.values || []).find(
+              (entry: any) => entry.formatted === value
+            )
+            return match ? match.unformatted : value
+          }
+
          let rawValue = ''
          if (filterClause.operator === 'BETWEEN') {
-            rawValue = `${filterClause.value[0]} AND ${filterClause.value[1]}`
+            const from = unformattedValue(filterClause.value[0])
+            const to = unformattedValue(filterClause.value[1])
+            rawValue = `${temporalValue(from) ?? from} AND ${
+              temporalValue(to) ?? to
+            }`
          } else if (
            filterClause.operator === 'IN' ||
            filterClause.operator === 'NOT IN'
          ) {
            filterClause.value.forEach((element: any) => {
              if (element.checked) {
-                const value = element.val
-                if (
+                /* Mapped like the BETWEEN ends - whereClauseCreator normally
+                   rewrites element.val to the unformatted count before submit,
+                   so this is a pass-through for the usual path and a guard for
+                   any path that reaches here without the preview having run. */
+                const value = unformattedValue(element.val)
+                const literal = temporalValue(value)
+                if (literal) {
+                  rawValue += (rawValue.length > 0 ? ',' : '(') + literal
+                } else if (
                  typeof value === 'string' &&
                  (!isSpecialMissing(value) || filterClause.type === 'char')
                ) {
@@ -700,7 +757,13 @@ export class SasStoreService {
              rawValue = '.'
            }
          } else {
-            if (filterClause.type === 'char' && !filterClause.valueVariable) {
+            const literal = temporalValue(filterClause.value)
+            if (literal) {
+              rawValue = literal
+            } else if (
+              filterClause.type === 'char' &&
+              !filterClause.valueVariable
+            ) {
              rawValue = `'${filterClause.value.replace(/'/g, "''")}'`
            }
          }
@@ -14,7 +14,12 @@ const buildService = () => {
  const sasjsAdapter: any = {
    request: jasmine.createSpy('request').and.resolveTo({}),
    getSasRequests: () => [],
-    setDebugState: jasmine.createSpy('setDebugState')
+    setDebugState: jasmine.createSpy('setDebugState'),
+    getSasjsConfig: () => ({
+      serverUrl: 'https://sas.example.com',
+      appLoc: '/Public/app/dc',
+      serverType: 'SASVIYA'
+    })
  }
  const loggerService: any = {
    logRequestData: jasmine.createSpy('logRequestData')
@@ -163,3 +168,38 @@ describe('SasService debug state', () => {
    expect(initService(true).getSasjsConfig().debug).toBe(false)
  })
 })
+
+/**
+ * The URL shape the deploy configurator already uses for makedata: the service
+ * is run by the JES / STP web app in a browser tab, so the SAS log comes back
+ * as the page body instead of through the adapter.
+ */
+describe('SasService openServiceInNewWindow', () => {
+  it('opens the service URL built from the live adapter config', () => {
+    const { service } = buildService()
+    spyOn(service, 'getExecutionPath').and.returnValue('/SASJobExecution')
+    const openSpy = spyOn(window, 'open')
+
+    service.openServiceInNewWindow('public/startupservice', {
+      _contextname: 'ctx'
+    })
+
+    expect(openSpy).toHaveBeenCalledWith(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_contextname=ctx&_debug=131',
+      '_blank'
+    )
+  })
+
+  it('opens without parameters when none are given', () => {
+    const { service } = buildService()
+    spyOn(service, 'getExecutionPath').and.returnValue('/SASJobExecution')
+    const openSpy = spyOn(window, 'open')
+
+    service.openServiceInNewWindow('public/startupservice')
+
+    expect(openSpy).toHaveBeenCalledWith(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131',
+      '_blank'
+    )
+  })
+})
@@ -21,6 +21,7 @@ import { SasViyaService } from './sas-viya.service'
 import { ViyaApiFolder } from '../viya-api-explorer/models/viya-api-folder.model'
 import { ViyaApiFolderMembers } from '../viya-api-explorer/models/viya-api-folder-content.model'
 import { parseUserAgent } from '../shared/utils/parse-user-agent'
+import { buildServiceUrl } from '../shared/utils/service-url'
 import { VERSION } from '../../environments/version'

 /**
@@ -842,6 +843,33 @@ export class SasService {
    return this.sasjsAdapter
  }

+  /**
+   * Runs a SAS service in a new browser window, bypassing the adapter.
+   *
+   * The adapter's request path can hide the backend log when a service fails:
+   * on Viya a long startup service may be submitted as an execution task and
+   * time out before it finishes, returning neither webout nor log. A browser
+   * request to the JES / STP web app runs the same `_program` with no task and
+   * returns the SAS log (`_debug=131`), so this is the diagnostic path offered
+   * when a failed request leaves nothing usable to display.
+   */
+  public openServiceInNewWindow(
+    service: string,
+    params: { [name: string]: string | number | null | undefined } = {}
+  ) {
+    const config = this.getSasjsConfig()
+
+    const url = buildServiceUrl({
+      serverUrl: config.serverUrl || '',
+      appLoc: config.appLoc || '',
+      executionPath: this.getExecutionPath() || '',
+      service,
+      params
+    })
+
+    window.open(url, '_blank')
+  }
+
  public getServerType(): string {
    const sasjsConfig = this.sasjsAdapter.getSasjsConfig()

@@ -1,27 +1,6 @@
 import { VaFilterService } from './va-filter.service'
 import { VaMessage } from './va-messaging.service'
-
-/**
- * Minimal stand-in for HelperService.convertJsDateToSasDate using the real
- * epoch math, so date/datetime assertions use the actual SAS internal values
- * without pulling in HelperService's dependency graph (SasService etc.).
- */
-class HelperServiceStub {
-  convertJsDateToSasDate(jsDate: Date, unit = 'days'): number {
-    const ms = Date.UTC(
-      jsDate.getFullYear(),
-      jsDate.getMonth(),
-      jsDate.getDate(),
-      jsDate.getHours(),
-      jsDate.getMinutes(),
-      jsDate.getSeconds()
-    )
-    const sasMs = ms + 315619200000
-    return unit === 'days'
-      ? Math.floor(Math.abs(sasMs / 86400000))
-      : sasMs / 1000
-  }
-}
+import { HelperService } from './helper.service'

 function msg(parameters: any[]): VaMessage {
  return {
@@ -63,7 +42,7 @@ describe('VaFilterService', () => {
  ]

  beforeEach(() => {
-    service = new VaFilterService(new HelperServiceStub() as any)
+    service = new VaFilterService(new HelperService({} as any))
  })

  const labelToVar = () =>
@@ -109,28 +88,30 @@ describe('VaFilterService', () => {
      expect(build([{ label: 'SOME_NUM', value: 'abc' }])).toEqual([])
    })

-    it('converts temporal values to SAS internal numbers', () => {
+    it('renders temporal values as SAS literals', () => {
      expect(
        build([{ label: 'SOME_TIME', value: '0:00:02' }])[0].RAW_VALUE
-      ).toBe('2')
+      ).toBe("'00:00:02't")
      expect(
        build([{ label: 'SOME_DATE', value: '01JUL1960' }])[0].RAW_VALUE
-      ).toBe('182')
+      ).toBe("'01JUL1960'd")
      expect(
        build([{ label: 'SOME_DATETIME', value: '01JAN1960:00:06:05' }])[0]
          .RAW_VALUE
-      ).toBe('365')
+      ).toBe("'01JAN1960:00:06:05'dt")
    })

-    it('passes an already-internal (bare number) temporal value through raw', () => {
+    it('renders an already-internal (bare number) temporal value as a literal', () => {
      // VA sends the raw SAS internal value when the data item has no format.
      expect(build([{ label: 'SOME_DATE', value: '182' }])[0].RAW_VALUE).toBe(
-        '182'
+        "'01JUL1960'd"
      )
      expect(
        build([{ label: 'SOME_DATETIME', value: '365' }])[0].RAW_VALUE
-      ).toBe('365')
-      expect(build([{ label: 'SOME_TIME', value: '2' }])[0].RAW_VALUE).toBe('2')
+      ).toBe("'01JAN1960:00:06:05'dt")
+      expect(build([{ label: 'SOME_TIME', value: '2' }])[0].RAW_VALUE).toBe(
+        "'00:00:02't"
+      )
    })

    it('parses ISO date shapes identically to DATE9 (VA sends no format)', () => {
@@ -232,9 +213,9 @@ describe('VaFilterService', () => {
      expect(clauses.map((c) => [c.VARIABLE_NM, c.RAW_VALUE])).toEqual([
        ['SOME_CHAR', "'x'"],
        ['SOME_NUM', '10'],
-        ['SOME_TIME', '2'],
-        ['SOME_DATE', '182'],
-        ['SOME_DATETIME', '365']
+        ['SOME_TIME', "'00:00:02't"],
+        ['SOME_DATE', "'01JUL1960'd"],
+        ['SOME_DATETIME', "'01JAN1960:00:06:05'dt"]
      ])
    })

@@ -247,9 +228,9 @@ describe('VaFilterService', () => {
        ])
      )
      expect(clauses.map((c) => [c.VARIABLE_NM, c.RAW_VALUE])).toEqual([
-        ['SOME_DATE', '182'],
-        ['SOME_DATE2', '182'],
-        ['SOME_DTTM', '365']
+        ['SOME_DATE', "'01JUL1960'd"],
+        ['SOME_DATE2', "'01JUL1960'd"],
+        ['SOME_DTTM', "'01JAN1960:00:06:05'dt"]
      ])
    })
  })
@@ -223,15 +223,18 @@ export class VaFilterService {

  /**
   * Formats one VA filter value into a DC RAW_VALUE for the column kind, or null
-   * when invalid (dropped). Matches DC's native filters, which send the NUMERIC
-   * INTERNAL value (not SAS literals — those error out in the backend):
-   *  - numeric  -> finite number, raw (else null — injection guard)
-   *  - time     -> seconds since midnight   (VA: "0:00:02" -> 2)
-   *  - date     -> days since 1960-01-01    (VA: "01JUL1960" -> 182)
-   *  - datetime -> seconds since 1960-01-01 (VA: "01JAN1960:00:06:05" -> 365)
+   * when invalid (dropped). Matches DC's native filters, which send a SAS
+   * literal for a temporal column and the raw value otherwise:
+   *  - numeric  -> finite number, raw (else null - injection guard)
+   *  - time     -> seconds since midnight, as a SAS time literal
+   *                (VA: "0:00:02" -> '00:00:02't)
+   *  - date     -> days since 1960-01-01, as a SAS date literal
+   *                (VA: "01JUL1960" -> '01JUL1960'd)
+   *  - datetime -> seconds since 1960-01-01, as a SAS datetime literal
+   *                (VA: "01JAN1960:00:06:05" -> '01JAN1960:00:06:05'dt)
   *  - char     -> single-quote escaped
   * Temporal strings are regex-parsed (digits/letters/colons only), so a value
-   * either yields a finite number or is dropped — also the injection guard.
+   * either yields a finite number or is dropped - also the injection guard.
   * Date/datetime reuse HelperService.convertJsDateToSasDate so the value is
   * byte-identical to what DC's own pickers produce.
   *
@@ -239,7 +242,7 @@ export class VaFilterService {
   * itself (VA sends it unformatted when the data item carries no format / is
   * used as a measure), so it is passed through raw. This is unambiguous because
   * the only date formats in use (DATE9, ISO) always contain a letter or a
-   * separator — a plain integer is never a formatted date.
+   * separator - a plain integer is never a formatted date.
   */
  private formatValueByKind(
    value: any,
@@ -254,23 +257,32 @@ export class VaFilterService {
        return Number.isFinite(num) ? `${num}` : null
      }
      case 'time': {
-        if (this.isBareNumber(s)) return `${Number(s)}`
-        const secs = this.timeToSeconds(s)
-        return secs === null ? null : `${secs}`
+        const secs = this.isBareNumber(s) ? Number(s) : this.timeToSeconds(s)
+        return secs === null
+          ? null
+          : this.helperService.sasTemporalLiteral(secs, 'TIME')
      }
      case 'date': {
-        if (this.isBareNumber(s)) return `${Number(s)}`
-        const d = this.parseDate(s, order)
-        return d
-          ? `${this.helperService.convertJsDateToSasDate(d, 'days')}`
-          : null
+        let days: number | null = null
+        if (this.isBareNumber(s)) days = Number(s)
+        else {
+          const d = this.parseDate(s, order)
+          if (d) days = this.helperService.convertJsDateToSasDate(d, 'days')
+        }
+        return days === null
+          ? null
+          : this.helperService.sasTemporalLiteral(days, 'DATE')
      }
      case 'datetime': {
-        if (this.isBareNumber(s)) return `${Number(s)}`
-        const d = this.parseDatetime(s, order)
-        return d
-          ? `${this.helperService.convertJsDateToSasDate(d, 'seconds')}`
-          : null
+        let secs: number | null = null
+        if (this.isBareNumber(s)) secs = Number(s)
+        else {
+          const d = this.parseDatetime(s, order)
+          if (d) secs = this.helperService.convertJsDateToSasDate(d, 'seconds')
+        }
+        return secs === null
+          ? null
+          : this.helperService.sasTemporalLiteral(secs, 'DATETIME')
      }
      default:
        return `'${s.replace(/'/g, "''")}'`
@@ -28,6 +28,16 @@
    }
  </div>
  <div class="modal-footer">
+    @if (showServiceLauncher(data.sasService)) {
+      <button
+        type="button"
+        class="btn btn-primary"
+        (click)="openServiceInNewWindow()"
+      >
+        Show log
+      </button>
+    }
+
    @if (showConfiguratorButton(data.sasService)) {
      <button
        type="button"
@@ -0,0 +1,71 @@
+import { InfoModalComponent } from './info-modal.component'
+
+describe('InfoModalComponent', () => {
+  let component: InfoModalComponent
+  let sasService: any
+
+  const buildComponent = (sasServiceConfig: any = {}) => {
+    sasService = {
+      getSasjsConfig: () => sasServiceConfig,
+      openServiceInNewWindow: jasmine.createSpy('openServiceInNewWindow')
+    }
+
+    component = new InfoModalComponent(
+      {} as any,
+      {} as any,
+      sasService,
+      {} as any
+    )
+    component.data = {
+      modalTitle: 'Error',
+      message: 'startupservice failed',
+      sasService: 'public/startupservice',
+      details: null
+    }
+    component.ngOnInit()
+  }
+
+  describe('showServiceLauncher', () => {
+    beforeEach(() => buildComponent())
+
+    it('is shown for the startupservice, on every server type', () => {
+      expect(component.showServiceLauncher('public/startupservice')).toBeTrue()
+      expect(
+        component.showServiceLauncher('services/public/startupservice')
+      ).toBeTrue()
+    })
+
+    it('is not shown for other services or a missing service', () => {
+      expect(component.showServiceLauncher('public/getdata')).toBeFalse()
+      expect(component.showServiceLauncher('')).toBeFalse()
+      expect(component.showServiceLauncher(null)).toBeFalse()
+    })
+  })
+
+  describe('openServiceInNewWindow', () => {
+    it('runs the startupservice in a new window', () => {
+      buildComponent({ serverType: 'SASVIYA' })
+
+      component.openServiceInNewWindow()
+
+      expect(sasService.openServiceInNewWindow).toHaveBeenCalledWith(
+        'public/startupservice',
+        {}
+      )
+    })
+
+    it('passes the configured Viya context so the log comes from it', () => {
+      buildComponent({
+        serverType: 'SASVIYA',
+        contextName: 'SAS Job Execution compute context'
+      })
+
+      component.openServiceInNewWindow()
+
+      expect(sasService.openServiceInNewWindow).toHaveBeenCalledWith(
+        'public/startupservice',
+        { _contextname: 'SAS Job Execution compute context' }
+      )
+    })
+  })
+})
@@ -71,6 +71,34 @@ export class InfoModalComponent implements OnInit {
    this.onConfirmModalClick.emit()
  }

+  /**
+   * Whether to show the `Show log` button.
+   *
+   * The startupservice failure modal is a dead end otherwise: the adapter
+   * returned no log, so the only way to see the backend reason is to run the
+   * service through the JES / STP web app in a browser tab. Offered on every
+   * server type - SAS 9, Viya and SASjs Server all accept the `_program` URL.
+   */
+  showServiceLauncher(sasService: string | null) {
+    return !!sasService?.includes('startupservice')
+  }
+
+  /**
+   * Runs the startupservice in a new browser window so the SAS log is
+   * available even when the adapter response carried none (e.g. a Viya
+   * execution task that timed out).
+   */
+  openServiceInNewWindow() {
+    const sasjsConfig = this.sasService.getSasjsConfig()
+    const params: { [name: string]: string } = {}
+
+    if (sasjsConfig.contextName) {
+      params['_contextname'] = sasjsConfig.contextName
+    }
+
+    this.sasService.openServiceInNewWindow(this.data.sasService || '', params)
+  }
+
  /**
   * Only on SAS9, opening a backend configurator/deploy page
   */
@@ -61,13 +61,13 @@ export class DcValidator {
    // injected separately — see its own doc comment for why.
    //
    // COLHEADERS is the authoritative grid order and the only safe key to
-    // build against: it is the variable list of the *output* dataset, where
-    // getdata.sas hoists the delete column and the PK to the front ("order
-    // delete var and pk fields at start of table"), whereas cols[].VARNUM
-    // describes the *source* table and cols[] itself can arrive sorted by
-    // NAME. Those orders diverge whenever the PK isn't already the source's
-    // first column, so rules are keyed by name and emitted in COLHEADERS
-    // order — never paired positionally against cols[] or VARNUM.
+    // build against: it is the variable list of the *output* dataset, and the
+    // editor prepends the client-owned delete column to it (see
+    // editor.component.ts's initSetup) before this constructor runs, whereas
+    // cols[].VARNUM describes the *source* table and cols[] itself can arrive
+    // sorted by NAME. Those orders diverge whenever the PK isn't already the
+    // source's first column, so rules are keyed by name and emitted in
+    // COLHEADERS order — never paired positionally against cols[] or VARNUM.
    //
    // A column with no parseable COLTYPE falls back to a plain text rule
    // rather than being dropped: getRules() is consumed positionally against
@@ -0,0 +1,75 @@
+import {
+  prepareDeleteRecordColumn,
+  seedDeleteRecordColumn
+} from './deleteRecordColumn'
+import { DELETE_RECORD_COLUMN_NAME } from './deleteRecordColumnRule'
+
+const response = (overrides: any = {}) => ({
+  data: {
+    sasparams: [{ COLHEADERS: 'PRIMARY_KEY_FIELD,SOME_CHAR' }],
+    cols: [],
+    $sasdata: { vars: {} },
+    ...overrides
+  }
+})
+
+describe('prepareDeleteRecordColumn', () => {
+  it('prepends the delete column to COLHEADERS', () => {
+    const res = response()
+
+    prepareDeleteRecordColumn(res)
+
+    expect(res.data.sasparams[0].COLHEADERS).toEqual(
+      `${DELETE_RECORD_COLUMN_NAME},PRIMARY_KEY_FIELD,SOME_CHAR`
+    )
+  })
+
+  it('throws when COLHEADERS already carries the delete column', () => {
+    const res = response({
+      sasparams: [{ COLHEADERS: `${DELETE_RECORD_COLUMN_NAME},SOME_CHAR` }]
+    })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError(/reserved column/)
+  })
+
+  it('throws when cols carries the delete column', () => {
+    const res = response({ cols: [{ NAME: DELETE_RECORD_COLUMN_NAME }] })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError(/reserved column/)
+  })
+
+  it('throws when $sasdata vars carries the delete column', () => {
+    const res = response({
+      $sasdata: { vars: { [DELETE_RECORD_COLUMN_NAME]: {} } }
+    })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError(/reserved column/)
+  })
+
+  it('leaves COLHEADERS untouched when it throws', () => {
+    const colHeaders = `${DELETE_RECORD_COLUMN_NAME},SOME_CHAR`
+    const res = response({ sasparams: [{ COLHEADERS: colHeaders }] })
+
+    expect(() => prepareDeleteRecordColumn(res)).toThrowError()
+    expect(res.data.sasparams[0].COLHEADERS).toEqual(colHeaders)
+  })
+})
+
+describe('seedDeleteRecordColumn', () => {
+  it("sets every row's delete flag to 'No'", () => {
+    const rows: any[] = [{ PRIMARY_KEY_FIELD: 1 }, { PRIMARY_KEY_FIELD: 2 }]
+
+    seedDeleteRecordColumn(rows)
+
+    expect(rows[0][DELETE_RECORD_COLUMN_NAME]).toEqual('No')
+    expect(rows[1][DELETE_RECORD_COLUMN_NAME]).toEqual('No')
+  })
+
+  it('overwrites a flag a row already carried', () => {
+    const rows: any[] = [{ [DELETE_RECORD_COLUMN_NAME]: 'Yes' }]
+
+    seedDeleteRecordColumn(rows)
+
+    expect(rows[0][DELETE_RECORD_COLUMN_NAME]).toEqual('No')
+  })
+})
@@ -0,0 +1,64 @@
+import { Col } from '../models/col.model'
+import { DELETE_RECORD_COLUMN_NAME } from './deleteRecordColumnRule'
+
+/**
+ * The parts of a getdata response this module touches. Declared structurally
+ * so the editor and the multi-dataset loader can both use it without either
+ * owning the other's response type.
+ */
+interface DeleteColumnResponse {
+  data: {
+    sasparams: { COLHEADERS: string }[]
+    cols?: Col[]
+    $sasdata?: { vars?: Record<string, unknown> }
+  }
+}
+
+/**
+ * Asserts that getdata did NOT send the client-owned delete column, then
+ * prepends it to COLHEADERS so the grid's first column is the delete flag.
+ *
+ * `_____DELETE__THIS__RECORD_____` is created by the editor itself (see
+ * deleteRecordColumnRule.ts) and getdata must never emit it. Its presence in a
+ * response can only mean a real table column of that name, or a hook that
+ * re-added it - either way it would shadow the delete flag, so refuse the load
+ * rather than render a table whose delete column cannot be trusted. Callers
+ * run this inside a promise chain whose `.catch` surfaces the error (the
+ * editor sets `getdataError`).
+ *
+ * Call it BEFORE constructing a DcValidator or deriving headerArray: rules,
+ * `columnHeader` and `headerArray` are all keyed off COLHEADERS order, and the
+ * delete column is always index 0.
+ */
+export const prepareDeleteRecordColumn = (response: DeleteColumnResponse) => {
+  const sasparams = response.data.sasparams[0]
+
+  const sentByBackend =
+    sasparams.COLHEADERS.split(',').includes(DELETE_RECORD_COLUMN_NAME) ||
+    (response.data.cols || []).some(
+      (col) => col.NAME === DELETE_RECORD_COLUMN_NAME
+    ) ||
+    Object.prototype.hasOwnProperty.call(
+      response.data.$sasdata?.vars || {},
+      DELETE_RECORD_COLUMN_NAME
+    )
+
+  if (sentByBackend) {
+    throw new Error(
+      `getdata returned the reserved column ${DELETE_RECORD_COLUMN_NAME}. ` +
+        'That name is created by the editor itself and must not exist in the target table.'
+    )
+  }
+
+  sasparams.COLHEADERS = `${DELETE_RECORD_COLUMN_NAME},${sasparams.COLHEADERS}`
+}
+
+/**
+ * Seeds every row's delete flag to 'No' - getdata no longer sends the column,
+ * so the editor owns the value too (see deleteRecordColumnRule.ts).
+ */
+export const seedDeleteRecordColumn = (rows: any[]) => {
+  for (const row of rows) {
+    row[DELETE_RECORD_COLUMN_NAME] = 'No'
+  }
+}
@@ -1,16 +1,21 @@
 import { DcValidation } from '../models/dc-validation.model'

 /**
- * `_____DELETE__THIS__RECORD_____` is a client-side UI concept (the delete
- * checkbox column) that getdata.sas synthesizes into vars3/vars4 for display
- * purposes only — it's never a real column of the underlying dataset, so
- * `%mp_getcols` (which builds `cols1`, the left side of the join into
- * `cols`) never has a row for it, and its rule can no longer travel via
- * `cols[].COLTYPE` (see issue #253's vertical-array fix). The rule is fixed
- * and data-independent, so it's injected directly here instead.
+ * `_____DELETE__THIS__RECORD_____` is a purely client-owned column - the
+ * delete checkbox column. getdata.sas does NOT emit it: the editor adds the
+ * name to COLHEADERS itself (see editor.component.ts's initSetup) and seeds
+ * every row to 'No'. A response that carries it is refused, because the name
+ * can then only come from a real table column of that name (or a hook that
+ * re-added it), which would silently shadow the delete flag.
+ *
+ * Its rule can never travel via `cols[].COLTYPE` - `%mp_getcols` (which
+ * builds `cols1`, the left side of the join into `cols`) has no row for it -
+ * so it is injected directly, here and in DcValidator's constructor.
 */
+export const DELETE_RECORD_COLUMN_NAME = '_____DELETE__THIS__RECORD_____'
+
 export const DELETE_RECORD_COLUMN_RULE: DcValidation = {
-  data: '_____DELETE__THIS__RECORD_____',
+  data: DELETE_RECORD_COLUMN_NAME,
  type: 'dropdown',
  source: ['No', 'Yes']
 }
@@ -0,0 +1,86 @@
+import { buildServiceUrl } from './service-url'
+
+describe('buildServiceUrl', () => {
+  const viyaConfig = {
+    serverUrl: 'https://sas.example.com',
+    appLoc: '/Public/app/dc',
+    executionPath: '/SASJobExecution',
+    service: 'public/startupservice'
+  }
+
+  it('builds the Viya JES URL for a service, with the log enabled', () => {
+    expect(buildServiceUrl(viyaConfig)).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('leaves serverUrl empty on SAS 9 so the browser resolves the origin', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        serverUrl: '',
+        executionPath: '/SASStoredProcess/do'
+      })
+    ).toBe(
+      '/SASStoredProcess/do/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('builds the SASjs Server URL from its execution path', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        executionPath: '/SASjsApi/stp/execute'
+      })
+    ).toBe(
+      'https://sas.example.com/SASjsApi/stp/execute/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('appends encoded parameters before the debug flag', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        params: { _contextname: 'SAS Job Execution compute context' }
+      })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_contextname=SAS%20Job%20Execution%20compute%20context&_debug=131'
+    )
+  })
+
+  it('skips parameters with no value', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        params: { _contextname: undefined, admin: null, dcpath: 'sas' }
+      })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&dcpath=sas&_debug=131'
+    )
+  })
+
+  it('omits the debug flag when debug is off', () => {
+    expect(buildServiceUrl({ ...viyaConfig, debug: false })).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice'
+    )
+  })
+
+  it('accepts a service with a leading slash', () => {
+    expect(
+      buildServiceUrl({ ...viyaConfig, service: '/public/startupservice' })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('accepts the services/ prefix the request wrapper reports', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        service: 'services/public/startupservice'
+      })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+})
@@ -0,0 +1,46 @@
+/**
+ * Builds the URL that runs a SAS service directly in a browser window.
+ *
+ * The adapter submits requests from JavaScript, and on Viya that can route the
+ * run through the Compute service (`_executionTasks=true`), where the SAS log
+ * is not returned to the caller and the request can time out before the
+ * service finishes. A browser request to the JES / STP web app runs the same
+ * `_program` without any task involved, and `_debug=131` returns the SAS log
+ * as the page body - which is what makes it useful when a service fails.
+ *
+ * `serverUrl` is left empty on SAS 9 when the app is served from the SAS
+ * server itself; the browser then resolves the path against the current
+ * origin.
+ */
+export interface ServiceUrlOptions {
+  serverUrl: string
+  appLoc: string
+  executionPath: string
+  service: string
+  params?: { [name: string]: string | number | null | undefined }
+  debug?: boolean
+}
+
+export const buildServiceUrl = ({
+  serverUrl,
+  appLoc,
+  executionPath,
+  service,
+  params = {},
+  debug = true
+}: ServiceUrlOptions): string => {
+  // Callers pass the service as the app knows it: `public/getdata` in the
+  // request path, but the request wrapper reports `services/public/getdata`
+  // on a failed request. Both resolve to the same program.
+  const servicePath = service.replace(/^\/+/, '').replace(/^services\//, '')
+  const paramString = Object.entries(params)
+    .filter(([, value]) => value !== null && value !== undefined)
+    .map(
+      ([name, value]) =>
+        `&${encodeURIComponent(name)}=${encodeURIComponent(String(value))}`
+    )
+    .join('')
+  const debugParam = debug ? '&_debug=131' : ''
+
+  return `${serverUrl}${executionPath}/?_program=${appLoc}/services/${servicePath}${paramString}${debugParam}`
+}
@@ -1,6 +1,6 @@
 {
  "name": "dcfrontend",
-  "version": "7.17.0",
+  "version": "7.18.0",
  "description": "Data Controller",
  "devDependencies": {
    "@nogoo9/gitleaks": "8.30.1-post.2",
@@ -14,7 +14,6 @@ function makeRows(n) {
    const rows = []
    for (let i = 0; i < n; i++) {
        rows.push({
-            _____DELETE__THIS__RECORD_____: "No",
            PRIMARY_KEY_FIELD: i,
            SOME_CHAR: "dummy data row " + i,
            SOME_DROPDOWN: dropdowns[i % dropdowns.length],
@@ -241,7 +240,6 @@ const data = {
        { ODS_TABLE: "ENGINEHOST", NAME: "File Size (bytes)", VALUE: "1048576" }
    ],
    maxvarlengths: [
-        { NAME: "_____DELETE__THIS__RECORD_____", MAXLEN: 3 },
        { NAME: "PRIMARY_KEY_FIELD", MAXLEN: 4 },
        { NAME: "some_char", MAXLEN: 591 },
        { NAME: "some_dropdown", MAXLEN: 8 },
@@ -261,7 +259,6 @@ const data = {
    sasdata: makeRows(100),
    $sasdata: {
        vars: {
-            _____DELETE__THIS__RECORD_____: { format: "$3.", label: "_____DELETE__THIS__RECORD_____", length: "3", type: "char" },
            PRIMARY_KEY_FIELD: { format: "best.", label: "PRIMARY_KEY_FIELD", length: "8", type: "num" },
            SOME_CHAR: { format: "$32767.", label: "SOME_CHAR", length: "32767", type: "char" },
            SOME_DROPDOWN: { format: "$128.", label: "SOME_DROPDOWN", length: "128", type: "char" },
@@ -280,14 +277,14 @@ const data = {
    },
    sasparams: [
        {
-            COLHEADERS: "_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,SOME_HARDSELECT,SOME_NUM,SOME_DATE,SOME_DATETIME,SOME_TIME,SOME_SHORTNUM,SOME_BESTNUM,READONLY_COL,HIDDEN_COL,ROUND_COL,NUMFMT_COL",
+            COLHEADERS: "PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,SOME_HARDSELECT,SOME_NUM,SOME_DATE,SOME_DATETIME,SOME_TIME,SOME_SHORTNUM,SOME_BESTNUM,READONLY_COL,HIDDEN_COL,ROUND_COL,NUMFMT_COL",
            FILTER_TEXT: "",
            PKCNT: 1,
            PK: "PRIMARY_KEY_FIELD",
            DTVARS: " SOME_DATE",
            DTTMVARS: " SOME_DATETIME",
            TMVARS: " SOME_TIME",
-            COLTYPE: "{\"data\":\"_____DELETE__THIS__RECORD_____\",\"type\":\"dropdown\",\"source\":[\"No\",\"Yes\"]},{\"data\":\"PRIMARY_KEY_FIELD\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_CHAR\"},{\"data\":\"SOME_DROPDOWN\"},{\"data\":\"SOME_HARDSELECT\"},{\"data\":\"SOME_NUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_DATE\",\"type\":\"date\"},{\"data\":\"SOME_DATETIME\",\"type\":\"datetime\"},{\"data\":\"SOME_TIME\",\"type\":\"time\"},{\"data\":\"SOME_SHORTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_BESTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"READONLY_COL\"},{\"data\":\"HIDDEN_COL\"},{\"data\":\"ROUND_COL\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"NUMFMT_COL\",\"type\":\"numeric\",\"format\":\"0\"}",
+            COLTYPE: "{\"data\":\"PRIMARY_KEY_FIELD\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_CHAR\"},{\"data\":\"SOME_DROPDOWN\"},{\"data\":\"SOME_HARDSELECT\"},{\"data\":\"SOME_NUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_DATE\",\"type\":\"date\"},{\"data\":\"SOME_DATETIME\",\"type\":\"datetime\"},{\"data\":\"SOME_TIME\",\"type\":\"time\"},{\"data\":\"SOME_SHORTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"SOME_BESTNUM\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"READONLY_COL\"},{\"data\":\"HIDDEN_COL\"},{\"data\":\"ROUND_COL\",\"type\":\"numeric\",\"format\":\"0\"},{\"data\":\"NUMFMT_COL\",\"type\":\"numeric\",\"format\":\"0\"}",
            LOADTYPE: "UPDATE",
            RK_FLAG: 0,
            CLS_FLAG: 0
@@ -281,6 +281,53 @@ const tables = {
      DD_SENSITIVITY: 'Low'
    }
  ],
+  // Derived (computed) variables.  Values are display-time only - the logic is
+  // applied by mpe_derived in the mock's getdata/getcolvals, and must use the
+  // expression subset that compileSasExpression supports (the real evaluation
+  // is done by SAS).
+  MPE_DERIVED: [
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'REGION',
+      var_deps: 'STATE',
+      var_type: 'C',
+      var_len: 8,
+      var_fmt: '',
+      var_logic: "ifc(STATE in ('CA','OR','WA'),'West','East')",
+      active: 1
+    },
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'LINE_VALUE',
+      var_deps: 'QTY PRICE',
+      var_type: 'N',
+      var_len: 8,
+      var_fmt: 'dollar12.2',
+      var_logic: 'QTY * PRICE',
+      active: 1
+    },
+    {
+      // built on REGION, so it proves the dependency order is applied
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'REGION_LABEL',
+      var_deps: 'REGION STATE',
+      var_type: 'C',
+      var_len: 14,
+      var_fmt: '',
+      var_label: 'Region and state',
+      var_logic: "cats(REGION,'-',STATE)",
+      active: 1
+    }
+  ],
  MPE_DATALOADS: [],
  MPE_EMAILS: [],
  MPE_EXCEL_CONFIG: [],
@@ -2226,7 +2273,7 @@ tables.MPE_TABLES.push({

 // ─── MPE_X_WIDE: wide table for the EDIT cell limit ─────────────────────────
 // 200 rows x 1001 columns (ROW_ID + 500 single-character + 500 numeric) is
-// 200,400 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
+// 200,200 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
 // DC_MAXOBS_WEBEDIT (250), so it exercises the CELL limit on its own and the
 // row limit spec can use MPE_X_SEARCH (1000 rows) instead.  Values are one
 // character / one digit on purpose: the point is width, and small values keep
@@ -3262,6 +3309,20 @@ const schema = {
      notnull: false
    }
  ],
+  MPE_DERIVED: [
+    { name: 'TX_FROM', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true },
+    { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
+    { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_DEPS', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
+    { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_LABEL', type: 'C', length: 256, format: '', label: '', notnull: false },
+    { name: 'VAR_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'ACTIVE', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'TX_TO', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true }
+  ],
  MPE_DATALOADS: [
    {
      name: 'LIBREF',
@@ -5349,7 +5410,15 @@ const testTables = {
    { ORDER_ID: 1003, CUSTOMER: 'Cobalt GmbH', REGION: 'EU', AMOUNT: 620, STATUS: 'CLOSED' },
    { ORDER_ID: 1004, CUSTOMER: 'Delta Systems', REGION: 'US', AMOUNT: 4500, STATUS: 'NEW' }
  ],
-  DEMO_MIRROR: []
+  DEMO_MIRROR: [],
+  // Derived variable demo - REGION and LINE_VALUE are not columns of this
+  // table, they are computed at read time from MPE_DERIVED rows.
+  DERIVED_DEMO: [
+    { ID: 1, STATE: 'CA', QTY: 2, PRICE: 10.5 },
+    { ID: 2, STATE: 'NY', QTY: 1, PRICE: 99 },
+    { ID: 3, STATE: 'OR', QTY: 5, PRICE: 3 },
+    { ID: 4, STATE: 'TX', QTY: 3, PRICE: 7.25 }
+  ]
 }

 // Schemas for the TESTDATA tables, matching the old inline getdata.js payload
@@ -5691,6 +5760,12 @@ const testSchema = {
      label: 'Last reviewed',
      notnull: false
    }
+  ],
+  DERIVED_DEMO: [
+    { name: 'ID', type: 'N', length: 8, format: '', label: 'ID', notnull: true },
+    { name: 'STATE', type: 'C', length: 2, format: '', label: 'State', notnull: false },
+    { name: 'QTY', type: 'N', length: 8, format: '', label: 'Quantity', notnull: false },
+    { name: 'PRICE', type: 'N', length: 8, format: 'dollar12.2', label: 'Price', notnull: false }
  ]
 }

@@ -5739,6 +5814,16 @@ tables.MPE_TABLES.push(
    buskey: 'ID',
    notes: 'Compact rule demo table (eight columns)'
  },
+  {
+    tx_from: 0,
+    tx_to: 127490111999,
+    libref: testDataLibref,
+    dsn: 'DERIVED_DEMO',
+    num_of_approvals_required: 1,
+    loadtype: 'UPDATE',
+    buskey: 'ID',
+    notes: 'Derived variable demo table'
+  },
  {
    tx_from: 0,
    tx_to: 127490111999,
@@ -460,6 +460,60 @@ function sasVarsEntry(col) {
    return { format: fmt, label: col.label || col.name, length: String(col.length), type: col.type === 'N' ? 'num' : 'char' }
 }

+/**
+ * Assigns the retained key (RK) to staged rows for a table registered with
+ * rk_underlying, mirroring %mp_retainedkey as %mpe_loader invokes it: any RK
+ * column in the input is dropped, and every row's RK is derived from its
+ * business key - the value that key already holds on the base table, or the
+ * next key value for a new one. Business keys are matched the way SAS compares
+ * them: case-sensitive, trailing blanks ignored.
+ *
+ * @param {Object} opts - { rows, tableReg, baseData }
+ *   tableReg - the table's MPE_TABLES row (needs rk_underlying and buskey)
+ *   baseData - { columns, rows } of the base table, for the key lookup
+ * @returns {String} the RK column name when the assignment ran, else ''
+ */
+function mpeAssignRetainedKey(opts) {
+    const { rows, tableReg, baseData } = opts
+    const rkUnderlying = tableReg && tableReg.rk_underlying
+        ? String(tableReg.rk_underlying).trim()
+        : ''
+    const rkCol = tableReg && tableReg.buskey ? String(tableReg.buskey).trim() : ''
+    if (!rkUnderlying || !rkCol || !rows || rows.length === 0) return ''
+
+    const base = baseData || { columns: [], rows: [] }
+    const submitted = Object.keys(rows[0] || {}).find(
+        (k) => k.toUpperCase() === rkCol.toUpperCase()
+    )
+    const rkName = submitted ||
+        (base.columns.find((c) => c.name.toUpperCase() === rkCol.toUpperCase()) || {}).name ||
+        rkCol
+    const keys = rkUnderlying.split(/\s+/)
+    const keyValue = (value) => {
+        if (value === null || value === undefined) return ''
+        return typeof value === 'number' ? String(value) : String(value).replace(/\s+$/, '')
+    }
+    const businessKey = (row) => keys.map((k) => keyValue(colVal(row, k))).join('|')
+
+    const baseByKey = new Map()
+    let nextRk = 0
+    for (const row of base.rows) {
+        const rk = colVal(row, rkCol)
+        baseByKey.set(businessKey(row), rk)
+        const num = Number(rk)
+        if (isFinite(num) && num > nextRk) nextRk = num
+    }
+
+    for (const row of rows) {
+        Object.keys(row).forEach((k) => {
+            if (k.toUpperCase() === rkCol.toUpperCase()) delete row[k]
+        })
+        const existing = baseByKey.get(businessKey(row))
+        row[rkName] = existing === undefined || existing === null ? ++nextRk : existing
+    }
+    return rkName
+}
+
 /**
 * Stages a submission: writes staged.json under staging/<tableId>/, copies the
 * session audit items, and inserts the MPE_SUBMIT row. Shared by the
@@ -580,6 +634,7 @@ function makeTableId() {
 *   ('A','B')   -> ["A","B"]        (IN list)
 *   (1,2,3)     -> [1,2,3]          (IN list, numeric)
 *   1 AND 5     -> [1, 5]           (BETWEEN)
+ *   '20DEC2053'd-> 34322            (date / datetime / time literal)
 */
 function parseFilterRawValue(raw, operator) {
    if (raw === undefined || raw === null || raw === '') return null
@@ -598,8 +653,40 @@ function parseFilterRawValue(raw, operator) {
    return parseFilterScalar(s)
 }

+/**
+ * SAS date / datetime / time literal -> the SAS internal value, or null.
+ *   34322 <- '20DEC2053'd
+ *   42    <- '01JAN1960:00:00:42'dt
+ *   42    <- '00:00:42't
+ * The client builds these in canonical ddMMMyyyy form (HelperService
+ * .sasTemporalLiteral), so only that shape needs parsing here.
+ */
+const TEMPORAL_LITERAL_RE = /^'(\d{2})([A-Z]{3})(\d{4})(?::(\d{2}):(\d{2}):(\d{2})(\.\d+)?)?'(dt|d|t)$/i
+/* A time literal carries no date part, so it is matched separately */
+const TIME_LITERAL_RE = /^'(\d{2}):(\d{2}):(\d{2})(\.\d+)?'t$/i
+
+function parseTemporalLiteral(s) {
+    const t = TIME_LITERAL_RE.exec(s)
+    if (t) {
+        return Number(t[1]) * 3600 + Number(t[2]) * 60 + Number(t[3]) +
+            (t[4] ? Number(t[4]) : 0)
+    }
+    const m = TEMPORAL_LITERAL_RE.exec(s)
+    if (!m) return null
+    const month = MONTHS_ABBR.indexOf(m[2].toUpperCase())
+    if (month < 0) return null
+    const days = (Date.UTC(Number(m[3]), month, Number(m[1])) - SAS_EPOCH_MS) / 86400000
+    const suffix = m[8].toLowerCase()
+    if (suffix === 'd') return days
+    const secs = Number(m[4]) * 3600 + Number(m[5]) * 60 + Number(m[6]) +
+        (m[7] ? Number(m[7]) : 0)
+    return suffix === 'dt' ? days * 86400 + secs : secs
+}
+
 function parseFilterScalar(s) {
    s = s.trim()
+    const temporal = parseTemporalLiteral(s)
+    if (temporal !== null) return temporal
    if (s.startsWith("'") && s.endsWith("'")) {
        return s.slice(1, -1).replace(/''/g, "'")
    }
@@ -1121,6 +1208,289 @@ function mpFilterStore(opts) {
    }
 }

+// ─── mpe_derived.sas ────────────────────────────────────────────────────────
+
+/**
+ * Compiles a SAS expression into a JS function - for the mock only.
+ *
+ * The real evaluation is done by SAS. This covers the subset used by the mock
+ * and demo configs, so the EDIT screen can be exercised (display, filtering,
+ * submission) without a SAS server:
+ *
+ *   - column names, numbers, quoted strings
+ *   - + - * / and comparisons  (= ne < > <= >=)
+ *   - AND OR, and `X IN (a, b, c)`
+ *   - ifc(), cats(), catx(), upcase(), lowcase(), strip(), trim(), length(),
+ *     substr(), index()
+ *
+ * Anything outside that subset throws when a row is derived, and the column is
+ * left missing for that row - the same way a failing SAS expression would.
+ */
+function compileSasExpression(expr, columnNames) {
+    // Tokenise first, so operators are only rewritten outside quoted strings
+    // (a literal such as 'OR' must never be read as an operator).
+    const tokens = []
+    const s = String(expr)
+    let i = 0
+    while (i < s.length) {
+        const c = s[i]
+        if (/\s/.test(c)) { i++; continue }
+        if (c === "'" || c === '"') {
+            const q = c
+            let j = i + 1
+            let val = ''
+            while (j < s.length) {
+                if (s[j] === q) {
+                    if (s[j + 1] === q) { val += q; j += 2; continue }
+                    break
+                }
+                val += s[j]
+                j++
+            }
+            tokens.push({ type: 'str', value: val })
+            i = j + 1
+            continue
+        }
+        if (/[0-9]/.test(c)) {
+            let j = i
+            while (j < s.length && /[0-9.]/.test(s[j])) j++
+            tokens.push({ type: 'num', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        if (/[A-Za-z_]/.test(c)) {
+            let j = i
+            while (j < s.length && /[A-Za-z0-9_]/.test(s[j])) j++
+            tokens.push({ type: 'word', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        const two = s.slice(i, i + 2)
+        if (['<=', '>=', '!=', '<>'].includes(two)) {
+            tokens.push({ type: 'punct', value: two })
+            i += 2
+            continue
+        }
+        // only the operators in the documented subset may reach the generated
+        // function - anything else is rejected rather than executed
+        if (!'+-*/(),<>='.includes(c)) {
+            throw new Error('Unsupported character in derived expression: ' + c)
+        }
+        tokens.push({ type: 'punct', value: c })
+        i++
+    }
+
+    // X IN (a, b, c)  ->  __in([a, b, c], X)
+    const out = []
+    for (let t = 0; t < tokens.length; t++) {
+        const tok = tokens[t]
+        if (
+            tok.type === 'word' &&
+            tok.value.toLowerCase() === 'in' &&
+            out.length > 0 &&
+            tokens[t + 1] &&
+            tokens[t + 1].value === '('
+        ) {
+            let depth = 0
+            let j = t + 1
+            for (; j < tokens.length; j++) {
+                if (tokens[j].value === '(') depth++
+                else if (tokens[j].value === ')') {
+                    depth--
+                    if (depth === 0) break
+                }
+            }
+            if (j < tokens.length) {
+                const left = out.pop()
+                const inner = tokens.slice(t + 2, j)
+                out.push(
+                    { type: 'word', value: '__in' },
+                    { type: 'punct', value: '(' },
+                    { type: 'punct', value: '[' },
+                    ...inner,
+                    { type: 'punct', value: ']' },
+                    { type: 'punct', value: ',' },
+                    left,
+                    { type: 'punct', value: ')' }
+                )
+                t = j
+                continue
+            }
+        }
+        out.push(tok)
+    }
+
+    const OPS = {
+        and: '&&', or: '||',
+        eq: '==', ne: '!=', lt: '<', gt: '>', le: '<=', ge: '>='
+    }
+    const js = out
+        .map(tok => {
+            if (tok.type === 'str') return JSON.stringify(tok.value)
+            if (tok.type === 'num') return tok.value
+            if (tok.type === 'word') return OPS[tok.value.toLowerCase()] || tok.value
+            if (tok.value === '=') return '=='
+            if (tok.value === '<>') return '!='
+            return tok.value
+        })
+        .join(' ')
+
+    const sasTrue = v => {
+        if (typeof v === 'boolean') return v
+        if (typeof v === 'number') return v !== 0 && !isNaN(v)
+        if (v === null || v === undefined) return false
+        const str = String(v).trim()
+        return str !== '' && str !== '0'
+    }
+    const toStr = v => {
+        if (v === null || v === undefined) return ''
+        if (typeof v === 'number') return isNaN(v) ? '' : String(v)
+        return String(v).trim()
+    }
+    const fns = {
+        ifc: (c, a, b) => (sasTrue(c) ? a : b),
+        cats: (...args) => args.map(toStr).join(''),
+        catx: (sep, ...args) => args.map(toStr).filter(v => v !== '').join(toStr(sep)),
+        upcase: v => toStr(v).toUpperCase(),
+        lowcase: v => toStr(v).toLowerCase(),
+        strip: v => toStr(v),
+        trim: v => toStr(v),
+        length: v => toStr(v).length,
+        substr: (v, p, n) => {
+            const str = toStr(v)
+            const start = Number(p) - 1
+            return n === undefined
+                ? str.substring(start)
+                : str.substring(start, start + Number(n))
+        },
+        index: (v, sub) => toStr(v).indexOf(toStr(sub)) + 1,
+        __in: (list, v) => list.some(item => String(item) === String(v) || Number(item) === Number(v))
+    }
+
+    const names = Object.keys(fns)
+    const fn = new Function(...columnNames, ...names, 'return (' + js + ');')
+    return row => fn(...columnNames.map(n => row[n]), ...names.map(n => fns[n]))
+}
+
+/**
+ * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
+ * variables with a compiled row function, in dependency order (ties by name),
+ * the same sequence the real macro emits.
+ *
+ * @param {Object} opts
+ * @param {string} opts.libref - target libref
+ * @param {string} opts.table - target dataset
+ * @param {string} opts.dataDir - data directory of the DC control library
+ * @param {Array<Object>} opts.columns - physical columns of the target table
+ */
+function mpeDerived(opts) {
+    const { libref, table, dataDir, columns } = opts
+    const loadTableData = makeTableLoader(dataDir)
+    const cfg = loadTableData('MPE_DERIVED')
+    if (!cfg || !Array.isArray(cfg.rows) || cfg.rows.length === 0) return []
+
+    const now = Date.now() / 1000
+    const lib = String(libref || '').toUpperCase()
+    const dsn = String(table || '').toUpperCase()
+    const columnNames = (columns || []).map(c => c.name)
+
+    const vars = cfg.rows
+        .filter(r =>
+            String(colVal(r, 'LIBREF') || '').toUpperCase() === lib &&
+            String(colVal(r, 'DSN') || '').toUpperCase() === dsn &&
+            Number(colVal(r, 'ACTIVE')) === 1 &&
+            (!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
+        )
+        .map(r => {
+            const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
+            const nm = String(colVal(r, 'VAR_NM') || '').toUpperCase()
+            return {
+                name: nm,
+                // every input the expression uses - columns and derived vars
+                deps: String(colVal(r, 'VAR_DEPS') || '')
+                    .toUpperCase()
+                    .split(/\s+/)
+                    .filter(Boolean),
+                type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
+                length: Number(colVal(r, 'VAR_LEN')) || 8,
+                format: String(colVal(r, 'VAR_FMT') || ''),
+                logic,
+                label: String(colVal(r, 'VAR_LABEL') || '').trim() || nm + '=' + logic
+            }
+        })
+        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
+
+    // dependency order: repeatedly take the variables whose derived inputs are
+    // already placed, alphabetically within a round
+    const byName = new Map(vars.map(v => [v.name, v]))
+    const placed = []
+    const done = new Set()
+    while (placed.length < vars.length) {
+        const ready = vars
+            .filter(v => !done.has(v.name))
+            .filter(v => v.deps.every(d => !byName.has(d) || done.has(d)))
+            .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+        if (ready.length === 0) {
+            console.log('derived variables for ' + lib + '.' + dsn + ' contain a cycle')
+            return []
+        }
+        for (const v of ready) {
+            placed.push(v)
+            done.add(v.name)
+        }
+    }
+    vars.length = 0
+    vars.push.apply(vars, placed)
+
+    // compile each against the physical columns plus the variables defined
+    // earlier, so a derived variable can build on another one
+    const compileCols = columnNames.slice()
+    for (const d of vars) {
+        try {
+            // slice(): the compiled function captures the column list it was
+            // built with, not the array as it grows
+            d.fn = compileSasExpression(d.logic, compileCols.slice())
+        } catch (err) {
+            console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
+            d.fn = () => (d.type === 'N' ? null : '')
+        }
+        compileCols.push(d.name)
+    }
+    return vars
+}
+
+/**
+ * Adds the derived variables to every row and to the column list, so that the
+ * filter (which runs afterwards) and the grid both see them.
+ */
+function mpeDeriveRows(tableData, derived) {
+    if (!derived || derived.length === 0) return tableData
+    const rows = tableData.rows.map(row => {
+        const copy = Object.assign({}, row)
+        for (const d of derived) {
+            try {
+                copy[d.name] = d.fn(copy)
+            } catch (err) {
+                copy[d.name] = d.type === 'N' ? null : ''
+            }
+        }
+        return copy
+    })
+    const derivedCols = derived.map(d => ({
+        name: d.name,
+        type: d.type,
+        length: d.length,
+        format: d.format,
+        label: d.label,
+        notnull: false
+    }))
+    return {
+        name: tableData.name,
+        columns: (tableData.columns || []).concat(derivedCols),
+        rows
+    }
+}
+
 // ─── DC settings (services/settings.js) ──────────────────────────────────────

 /**
@@ -77,6 +77,42 @@ if (mpeTablesData && mpeTablesData.rows) {
    tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
 }

+// The variables exposed to the editor as the primary key, matching getdata.sas:
+// pk = coalesce(rk_underlying, buskey). A table registered with rk_underlying
+// is keyed on its business key in the editor - the RK itself is assigned by the
+// backend loader, so it is neither shown nor sent by the user.
+const rkUnderlying = tableReg && tableReg.rk_underlying ? tableReg.rk_underlying.trim() : ''
+const buskey = tableReg && tableReg.buskey ? tableReg.buskey.trim() : ''
+const pkFields = rkUnderlying || buskey
+const pkColumns = pkFields ? pkFields.split(/\s+/) : []
+
+// Order columns with the PK variables first, in PK order, then the remaining
+// columns in schema order. getdata.sas does the same ("order delete var and pk
+// fields at start of table") and the client reads the first PKCNT columns
+// positionally when it detects duplicate keys.
+function pkFirst(columns, keys) {
+    const rank = new Map(keys.map((name, i) => [name.toUpperCase(), i]))
+    const keyColumns = columns
+        .filter(col => rank.has(col.name.toUpperCase()))
+        .sort((a, b) => rank.get(a.name.toUpperCase()) - rank.get(b.name.toUpperCase()))
+    return keyColumns.concat(columns.filter(col => !rank.has(col.name.toUpperCase())))
+}
+
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Applied before the filter, so a filter can reference them.  The derived
+// columns are added to the schema too, which is what puts them in COLS and in
+// the grid (read only, since the mock marks every column READ).
+
+const derived = mpeDerived({
+    libref: requestedLibref,
+    table: requestedTable,
+    dataDir: mpeDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
 // Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
 // NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
@@ -85,10 +121,7 @@ const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
 let dqrules = []
 let dqdata = []
 if (validationsData && validationsData.rows) {
-    const pkCols = new Set()
-    if (tableReg && tableReg.buskey) {
-        tableReg.buskey.trim().split(/\s+/).forEach(c => pkCols.add(c.toUpperCase()))
-    }
+    const pkCols = new Set(pkColumns.map(c => c.toUpperCase()))
    dqrules = validationsData.rows
        .filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
        .map(r => ({
@@ -191,9 +224,8 @@ if (tableData && tableData.rows && tableData.columns) {
 // ─── Build response ──────────────────────────────────────────────────────────

 let response
-// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
-// getdata.sas, and is emitted as sasjsAbort below.
-let abortMsg = ''
+// Set when the row or cell limit cut the selection down - mirrors the
+// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.

 if (tableData && tableData.columns && tableData.rows) {
    // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
@@ -206,8 +238,17 @@ if (tableData && tableData.columns && tableData.rows) {
        if (tableReg.var_busfrom) excludeCols.add(tableReg.var_busfrom.toUpperCase())
        if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
    }
+    // getdata.sas drops the RK field from the editor view when the table is
+    // registered with rk_underlying (drop_rk=&buskey), and exposes the business
+    // key instead.
+    if (rkUnderlying && buskey) {
+        buskey.split(/\s+/).forEach(c => excludeCols.add(c.toUpperCase()))
+    }

-    let visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
+    let visibleColumns = pkFirst(
+        tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase())),
+        pkColumns
+    )
    let visibleRows = tableData.rows
        .filter(r => filterPredicate(r))
        .map(r => {
@@ -236,15 +277,15 @@ if (tableData && tableData.columns && tableData.rows) {
    }

    // ─── Row and cell limits ─────────────────────────────────────────────────
-    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
-    // cells (rows x columns), whichever is reached first.  Both are read from
-    // MPE_CONFIG, the way the real service picks them up from the settings
-    // program it generates - so editing the option in DC changes what the mock
-    // allows, and an option that is absent (or switched off, var_active=0)
-    // falls back to the shipped default.  The check runs after the
-    // PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
-    // replace the data with a larger table.  The cell count includes the
-    // housekeeping delete column, which is what getdata.sas counts.
+    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
+    // cells (rows x columns), whichever is reached first - the selection is
+    // cut to min(rows, cells/columns) and flagged in sasparams.
+    // Both options are read from MPE_CONFIG, the way the real service picks
+    // them up from the settings program it generates - so editing the option
+    // in DC changes what the mock serves, and an option that is absent (or
+    // switched off, var_active=0) falls back to the shipped default.  The
+    // check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
+    // does - a hook may replace the data with a larger table.
    // MPE_CONFIG lives in the control library, not in the table's own library,
    // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
    // uses.  Passing the requested table's dataDir made the read fail for every
@@ -252,20 +293,32 @@ if (tableData && tableData.columns && tableData.rows) {
    const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
    const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
    const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
-    const editColCount = visibleColumns.length + 1
-    const cellCount = visibleRows.length * editColCount
-    if (visibleRows.length > maxObsWebEdit) {
-        abortMsg =
-            'Table is too big (' +
-            visibleRows.length +
-            ' rows) - please filter and try again!'
-    } else if (cellCount > maxCellsWebEdit) {
-        abortMsg =
-            'Selection is too wide (' +
-            visibleRows.length +
-            ' rows x ' +
-            editColCount +
-            ' cols) - please filter and try again!'
+    // getdata.sas counts the columns of work.out, i.e. the table's own
+    // columns - the delete column is client-owned and not part of it.
+    const editColCount = visibleColumns.length
+    let maxRows = maxObsWebEdit
+    let limitVar = 'DC_MAXOBS_WEBEDIT'
+    if (editColCount > 0) {
+        const cellRows = Math.floor(maxCellsWebEdit / editColCount)
+        if (cellRows < maxRows) {
+            maxRows = cellRows
+            limitVar = 'DC_MAXCELLS_WEBEDIT'
+        }
+    }
+    if (maxRows < 1) maxRows = 1
+    let truncated = 0
+    let truncMsg = ''
+    if (visibleRows.length > maxRows) {
+        truncated = 1
+        // Same wording as the macro getdata.sas sets, so the spec can assert
+        // on the text the real service would send.
+        truncMsg =
+            'Only the first ' +
+            maxRows +
+            ' rows are loaded - the ' +
+            limitVar +
+            ' limit has been reached.  Filter the table to work with the rest.'
+        visibleRows = visibleRows.slice(0, maxRows)
    }

    // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
@@ -280,16 +333,16 @@ if (tableData && tableData.columns && tableData.rows) {
            const targetKey = info ? info.name : key
            normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
        }
-        return { _____DELETE__THIS__RECORD_____: 'No', ...normalised }
+        return { ...normalised }
    })

    // $sasdata vars: temporal columns are char ($200.), others keep their SAS type
-    const vars = { _____DELETE__THIS__RECORD_____: { format: '$3.', label: '_____DELETE__THIS__RECORD_____', length: '3', type: 'char' } }
+    const vars = {}
    for (const col of visibleColumns) {
        vars[col.name] = sasVarsEntry(col)
    }

-    const colHeaders = ['_____DELETE__THIS__RECORD_____', ...visibleColumns.map(c => c.name)]
+    const colHeaders = visibleColumns.map(c => c.name)

    // Determine DTVARS, DTTMVARS, TMVARS from column formats
    let dtVars = [], dttmVars = [], tmVars = []
@@ -330,7 +383,6 @@ if (tableData && tableData.columns && tableData.rows) {

    // Build maxvarlengths
    const maxvarlengths = [
-        { NAME: '_____DELETE__THIS__RECORD_____', MAXLEN: 3 },
        ...visibleColumns.map(col => {
            let maxLen = 0
            const lcName = col.name.toLowerCase()
@@ -364,15 +416,17 @@ if (tableData && tableData.columns && tableData.rows) {
        sasparams: [{
            COLHEADERS: colHeaders.join(','),
            FILTER_TEXT: filterText === '' ? ' ' : filterText,
-            PKCNT: tableReg && tableReg.buskey ? tableReg.buskey.trim().split(/\s+/).length : 0,
-            PK: tableReg ? tableReg.buskey || '' : '',
+            PKCNT: pkColumns.length,
+            PK: pkFields,
            DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
            DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
            TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',
            LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
            RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
            CLS_FLAG: 0,
-            ISMAP: 0
+            ISMAP: 0,
+            TRUNCATED: truncated,
+            TRUNC_MSG: truncMsg
        }],
        versions: [],
        xl_rules: []
@@ -387,26 +441,21 @@ if (tableData && tableData.columns && tableData.rows) {
        ],
        maxvarlengths: [], query: [], sasdata: [],
        $sasdata: { vars: {} },
-        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0 }],
+        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
        versions: [], xl_rules: []
    }
 }

 webOutOpen()
-if (abortMsg) {
-    webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
-    webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
-} else {
-    webOutObj(response.approvers, 'approvers')
-    webOutObj(response.cols, 'cols')
-    webOutObj(response.dqdata, 'dqdata')
-    webOutObj(response.dqrules, 'dqrules')
-    webOutObj(response.dsmeta, 'dsmeta')
-    webOutObj(response.maxvarlengths, 'maxvarlengths')
-    webOutObj(response.query, 'query')
-    webOutObj(response.sasdata, 'sasdata', response.$sasdata)
-    webOutObj(response.sasparams, 'sasparams')
-    webOutObj(response.versions, 'versions')
-    webOutObj(response.xl_rules, 'xl_rules')
-}
+webOutObj(response.approvers, 'approvers')
+webOutObj(response.cols, 'cols')
+webOutObj(response.dqdata, 'dqdata')
+webOutObj(response.dqrules, 'dqrules')
+webOutObj(response.dsmeta, 'dsmeta')
+webOutObj(response.maxvarlengths, 'maxvarlengths')
+webOutObj(response.query, 'query')
+webOutObj(response.sasdata, 'sasdata', response.$sasdata)
+webOutObj(response.sasparams, 'sasparams')
+webOutObj(response.versions, 'versions')
+webOutObj(response.xl_rules, 'xl_rules')
 webOutClose()
@@ -147,6 +147,12 @@ if (!msg) {
                    if (tableEntry.var_txfrom) computedCols.push(tableEntry.var_txfrom.toUpperCase())
                    if (tableEntry.var_txto) computedCols.push(tableEntry.var_txto.toUpperCase())
                    if (tableEntry.var_processed) computedCols.push(tableEntry.var_processed.toUpperCase())
+                    // loadfile.sas drops the RK from the comparison for a table
+                    // with rk_underlying (the loader derives it), so an upload
+                    // for one legitimately omits it.
+                    if (tableEntry.rk_underlying && tableEntry.buskey) {
+                        computedCols.push(String(tableEntry.buskey).toUpperCase())
+                    }
                }
            }
            const baseCols = schema.map((c) => c.name.toUpperCase())
@@ -191,6 +197,23 @@ if (!msg) {
        }

        tableId = makeTableId()
+        // ─── RK assignment (mpe_loader) ──────────────────────────────────────
+        // The uploaded file carries the business key and no RK (see the
+        // expected-columns check above), so the loader derives it before the
+        // rows are staged.
+        const finalTables = mpeLoadTableData('MPE_TABLES')
+        const finalReg =
+            finalTables && finalTables.rows
+                ? finalTables.rows.find((r) => r.libref === libref && r.dsn === dsn)
+                : null
+        const rkName = mpeAssignRetainedKey({
+            rows,
+            tableReg: finalReg,
+            baseData: makeTableLoader(libDataDir(libref))(dsn)
+        })
+        if (rkName && !headers.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
+            headers.push(rkName)
+        }
        stageSubmission({
            dataDir: libDataDir(libref),
            stagingDir,
@@ -81,6 +81,26 @@ if (hookReg && hookReg.post_edit_hook) {
    }
 }

+// ─── RK assignment (mpe_loader) ──────────────────────────────────────────────
+// A table registered with rk_underlying is edited on its business key and the
+// RK column is withheld from the editor (see getdata.js), so the submitted rows
+// carry no RK. The loader derives it, and MPE_SUBMIT records the staged width
+// - which includes the derived column.
+
+const finalTables = makeTableLoader(mpeDataDir)('MPE_TABLES')
+const finalReg =
+    finalTables && finalTables.rows
+        ? finalTables.rows.find((r) => r.libref === libref && r.dsn === dsn)
+        : null
+const rkName = mpeAssignRetainedKey({
+    rows: jsDataRows,
+    tableReg: finalReg,
+    baseData: loadTableData(dsn)
+})
+if (rkName && !jsDataHeaders.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
+    jsDataHeaders.push(rkName)
+}
+
 // ─── Stage the data ──────────────────────────────────────────────────────────
 // One subfolder per submission (DSID) under the staging directory, so that the
 // staged data can sit alongside related artifacts (submit/approval logs,
@@ -21,9 +21,25 @@ const tableName = libds.split('.')[1] || ''
 // ─── Load table data (searches all libref data folders) ──────────────────────

 const _loaded = loadTableAnyLib(tableName) || {}
-const tableData = _loaded.data
+let tableData = _loaded.data
 const tableLibref = _loaded.libref || libref

+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Present in the schema and in the rows before the filter is applied, so the
+// filter may reference them and the values below are computed from the same
+// row set the grid shows.
+
+const controlDataDir = libDataDir(dcLibref)
+const derived = mpeDerived({
+    libref: tableLibref,
+    table: tableName,
+    dataDir: controlDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── Resolve column metadata ─────────────────────────────────────────────────

 let colMeta = null
@@ -6,8 +6,8 @@
    "": {
      "name": "dc-sas",
      "dependencies": {
-        "@sasjs/cli": "4.20.5",
-        "@sasjs/core": "5.2.8"
+        "@sasjs/cli": "6.0.0",
+        "@sasjs/core": "6.0.0"
      }
    },
    "node_modules/@asamuzakjp/css-color": {
@@ -140,56 +140,49 @@
      }
    },
    "node_modules/@fast-csv/format": {
-      "version": "4.3.5",
-      "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz",
-      "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==",
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-5.0.7.tgz",
+      "integrity": "sha512-VdypoRxv7PF+LsyPouTMKdB0d76hync+gLpgdNqfqVK44MsgW4oiCJSdrki2FisWT7v2QGUYDHjp4L7w5oO6gw==",
      "license": "MIT",
      "dependencies": {
-        "@types/node": "^14.0.1",
-        "lodash.escaperegexp": "^4.1.2",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isequal": "^4.5.0",
-        "lodash.isfunction": "^3.0.9",
-        "lodash.isnil": "^4.0.0"
+        "lodash.escaperegexp": "^4.1.2"
      }
    },
    "node_modules/@sasjs/adapter": {
-      "version": "4.19.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.19.1.tgz",
-      "integrity": "sha512-IpN6UPSdqVb5cAWEBt+jG/i0rhgCubdyp5kxdouuX3ifL313oJW2ZPVUNUvs0+ZGzsXWcGDTNoEzzTrGKiqJlQ==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-5.1.1.tgz",
+      "integrity": "sha512-rso6+xRpZhFuuUbgHmGztznuse4xZjlu7+dOcYMUMJezAd5p/xdCt+wvv4eIKejsMWDcIteSeRPOkgKr0TlxWg==",
      "license": "ISC",
      "dependencies": {
-        "@sasjs/utils": "^3.6.0",
+        "@sasjs/utils": "4.0.1",
        "axios": "1.20.0",
        "axios-cookiejar-support": "5.0.5",
        "form-data": "4.0.6",
-        "https": "1.0.0",
-        "tough-cookie": "4.1.3"
+        "tough-cookie": "5.1.2"
+      },
+      "engines": {
+        "node": ">=22"
      }
    },
    "node_modules/@sasjs/cli": {
-      "version": "4.20.5",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.20.5.tgz",
-      "integrity": "sha512-/HSpueCFS+SvXas9FwfP8qoD9skhdTDE5r7dT9Xlrs7HGCBjxDYvSiw6tW0GIjfw3/49Yf8TCVDFBsISang1hg==",
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-6.0.0.tgz",
+      "integrity": "sha512-nsdL5J+Y5FHZL4pieqQHjsNmY2DZ4gQcGAupR0dv8aGlweEHqdVErtx3B7+EmIiyfRt+oE3YEofzF41WBWL7tg==",
      "license": "ISC",
      "dependencies": {
-        "@sasjs/adapter": "4.19.1",
-        "@sasjs/core": "5.2.10",
-        "@sasjs/lint": "4.1.0",
-        "@sasjs/utils": "3.6.2",
+        "@sasjs/adapter": "5.1.1",
+        "@sasjs/core": "6.0.0",
+        "@sasjs/utils": "4.0.1",
+        "axios": "1.20.0",
        "chalk": "4.1.2",
-        "dotenv": "17.4.2",
-        "find": "0.3.0",
-        "js-base64": "3.9.3",
+        "dotenv": "18.0.8",
+        "js-base64": "3.9.4",
        "jsdom": "26.0.0",
-        "jwt-decode": "3.1.2",
-        "lodash.groupby": "4.6.0",
        "lodash.uniqby": "4.7.0",
        "node-graphviz": "0.1.1",
        "node-powershell": "5.0.1",
        "ora": "5.4.1",
        "prompts": "2.4.2",
-        "shelljs": "0.10.0",
        "ssl-root-cas": "1.3.1",
        "xml": "1.0.1",
        "yargs": "17.6.2",
@@ -198,65 +191,88 @@
      },
      "bin": {
        "sasjs": "build/index.js"
-      }
-    },
-    "node_modules/@sasjs/cli/node_modules/@sasjs/core": {
-      "version": "5.2.10",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.10.tgz",
-      "integrity": "sha512-Lk/ENHVuXaeAl8M0+Mjh1rcBSdFt4tIM6OPxPPmk78ZD0gYFqAOUZreqIoV9EN+VwPDqYsUPDqXzh0crW6j8OA==",
-      "license": "MIT"
-    },
-    "node_modules/@sasjs/cli/node_modules/shelljs": {
-      "version": "0.8.5",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
-      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
-      "license": "BSD-3-Clause",
-      "dependencies": {
-        "glob": "^7.0.0",
-        "interpret": "^1.0.0",
-        "rechoir": "^0.6.2"
-      },
-      "bin": {
-        "shjs": "bin/shjs"
      },
      "engines": {
-        "node": ">=4"
+        "node": ">=22"
+      },
+      "peerDependencies": {
+        "@sasjs/lint": "4.3.0"
      }
    },
    "node_modules/@sasjs/core": {
-      "version": "5.2.8",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.2.8.tgz",
-      "integrity": "sha512-+K8tXTrsw52NR1y5MFHW0+Jf9euL+GukwqA79FIHHu4Oj+BhHBm1uZiyB5tPrvwl3V8mY1CQhMOT83//bV+jWg==",
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.0.0.tgz",
+      "integrity": "sha512-E9iXPvNgCux80xHhfopCsQE/CUNp1NEF1Z7NRj089A1T8WXP9AWJYcfFfOqoljNh8rnMo1fbIgoNrIaaAgW0MA==",
      "license": "MIT"
    },
    "node_modules/@sasjs/lint": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-4.1.0.tgz",
-      "integrity": "sha512-Vpg1s0/zdPMFJ0hRXNXk53HhE3kfRr5gFRS0kvrse+bxgcbqih1qqVW7yZ2krhSCYylp9z13eBXLhXbRrj70VQ==",
-      "hasInstallScript": true,
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-4.3.0.tgz",
+      "integrity": "sha512-65cjy4IjA4PgSlicOruF0DI9N2HO3LJqtgphsZCg8g7N/StE2FWLJLJbD1N4OHxN6htrUfj5SEawJJiHo5T9Yw==",
      "license": "MIT",
+      "peer": true,
      "dependencies": {
-        "@sasjs/utils": "3.6.2",
+        "@sasjs/utils": "4.0.0",
        "ignore": "7.0.11"
+      },
+      "engines": {
+        "node": ">=22"
+      }
+    },
+    "node_modules/@sasjs/lint/node_modules/@sasjs/utils": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-4.0.0.tgz",
+      "integrity": "sha512-lARoCnRM4gSVf8ShoP5elHi7W28K/14MK037dQ0ABgMlP3+r6OuhAZYwK/a70tED47tKxyIoCAKLdP6jrG0ewQ==",
+      "license": "ISC",
+      "peer": true,
+      "dependencies": {
+        "@fast-csv/format": "5.0.7",
+        "@types/fs-extra": "11.0.4",
+        "@types/prompts": "2.4.9",
+        "chalk": "4.1.1",
+        "consola": "3.4.2",
+        "fs-extra": "11.4.1",
+        "jwt-decode": "4.0.0",
+        "prompts": "2.4.2"
+      },
+      "engines": {
+        "node": ">=22"
+      }
+    },
+    "node_modules/@sasjs/lint/node_modules/chalk": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "license": "MIT",
+      "peer": true,
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
      }
    },
    "node_modules/@sasjs/utils": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.2.tgz",
-      "integrity": "sha512-AfC5OJuJx3NAbntKq2J8E8NNpRXthYfuqOyp2lOmOkTctL3mAfrA54U/cdPQvrlftXNxpK9P2U0RU10qNjlEzQ==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-4.0.1.tgz",
+      "integrity": "sha512-aNEbOJFZ2Ru71/XkHKtywexBQYdst8ovq0a9vuVO4subZuKMjRtYcn4kTRr9VVIz65p8xGtVUkFEf+6t9Vbfcg==",
      "license": "ISC",
      "dependencies": {
-        "@fast-csv/format": "4.3.5",
+        "@fast-csv/format": "5.0.7",
        "@types/fs-extra": "11.0.4",
-        "@types/prompts": "2.0.13",
+        "@types/prompts": "2.4.9",
        "chalk": "4.1.1",
-        "cli-table": "0.3.6",
-        "consola": "2.15.0",
-        "find": "0.3.0",
-        "fs-extra": "11.3.0",
-        "jwt-decode": "3.1.2",
-        "prompts": "2.4.1",
-        "valid-url": "1.0.9"
+        "consola": "3.4.2",
+        "fs-extra": "11.4.1",
+        "jwt-decode": "4.0.0",
+        "prompts": "2.4.2"
+      },
+      "engines": {
+        "node": ">=22"
      }
    },
    "node_modules/@sasjs/utils/node_modules/chalk": {
@@ -275,19 +291,6 @@
        "url": "https://github.com/chalk/chalk?sponsor=1"
      }
    },
-    "node_modules/@sasjs/utils/node_modules/prompts": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.1.tgz",
-      "integrity": "sha512-EQyfIuO2hPDsX1L/blblV+H7I0knhgAd82cVneCwcdND9B8AuCDuRcBH6yIcG4dFzlOUqbazQqwGjx5xmsNLuQ==",
-      "license": "MIT",
-      "dependencies": {
-        "kleur": "^3.0.3",
-        "sisteransi": "^1.0.5"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
    "node_modules/@types/fs-extra": {
      "version": "11.0.4",
      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz",
@@ -308,18 +311,22 @@
      }
    },
    "node_modules/@types/node": {
-      "version": "14.18.63",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz",
-      "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==",
-      "license": "MIT"
-    },
-    "node_modules/@types/prompts": {
-      "version": "2.0.13",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "version": "26.6.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz",
+      "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==",
      "license": "MIT",
      "dependencies": {
-        "@types/node": "*"
+        "undici-types": "~8.9.0"
+      }
+    },
+    "node_modules/@types/prompts": {
+      "version": "2.4.9",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.4.9.tgz",
+      "integrity": "sha512-qTxFi6Buiu8+50/+3DGIWLHM6QuWsEKugJnnP6iv2Mc4ncxE4A/OJkjuVOA+5X0X1S/nq5VJRa8Lu+nwcvbrKA==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*",
+        "kleur": "^3.0.3"
      }
    },
    "node_modules/accumulate-stream": {
@@ -402,12 +409,6 @@
        "tough-cookie": ">=4.0.0"
      }
    },
-    "node_modules/balanced-match": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
-      "license": "MIT"
-    },
    "node_modules/base64-js": {
      "version": "1.5.1",
      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -439,16 +440,6 @@
        "readable-stream": "^3.4.0"
      }
    },
-    "node_modules/brace-expansion": {
-      "version": "1.1.21",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz",
-      "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==",
-      "license": "MIT",
-      "dependencies": {
-        "balanced-match": "^1.0.0",
-        "concat-map": "0.0.1"
-      }
-    },
    "node_modules/buffer": {
      "version": "5.7.1",
      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -559,17 +550,6 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
-    "node_modules/cli-table": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.6.tgz",
-      "integrity": "sha512-ZkNZbnZjKERTY5NwC2SeMeLeifSPq/pubeRoTpdr3WchLlnZg6hEgvHkK5zL7KNFdd9PmHN8lxrENUwI3cE8vQ==",
-      "dependencies": {
-        "colors": "1.0.3"
-      },
-      "engines": {
-        "node": ">= 0.2.0"
-      }
-    },
    "node_modules/cliui": {
      "version": "8.0.1",
      "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
@@ -611,15 +591,6 @@
      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
      "license": "MIT"
    },
-    "node_modules/colors": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
-      "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.1.90"
-      }
-    },
    "node_modules/combined-stream": {
      "version": "1.0.8",
      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -632,17 +603,14 @@
        "node": ">= 0.8"
      }
    },
-    "node_modules/concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
-      "license": "MIT"
-    },
    "node_modules/consola": {
-      "version": "2.15.0",
-      "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.0.tgz",
-      "integrity": "sha512-vlcSGgdYS26mPf7qNi+dCisbhiyDnrN1zaRbw3CSuc2wGOMEGGPsp46PdRG5gqXwgtJfjxDkxRNAgRPr1B77vQ==",
-      "license": "MIT"
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz",
+      "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==",
+      "license": "MIT",
+      "engines": {
+        "node": "^14.18.0 || >=16.10.0"
+      }
    },
    "node_modules/cssstyle": {
      "version": "4.6.0",
@@ -715,10 +683,13 @@
      }
    },
    "node_modules/dotenv": {
-      "version": "17.4.2",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
-      "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
+      "version": "18.0.8",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-18.0.8.tgz",
+      "integrity": "sha512-2KDeoFoK/zRtcnbunCDlLG5VWVMcyqeQc6iK65o231nfNAcuGrKoqUQfJFUtBI+QlShk5HKIHxwebBtUl3rEfA==",
      "license": "BSD-2-Clause",
+      "bin": {
+        "dotenv": "dist/index.cjs"
+      },
      "engines": {
        "node": ">=12"
      },
@@ -818,15 +789,6 @@
      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
      "license": "MIT"
    },
-    "node_modules/find": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
-      "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==",
-      "license": "MIT",
-      "dependencies": {
-        "traverse-chain": "~0.1.0"
-      }
-    },
    "node_modules/follow-redirects": {
      "version": "1.16.1",
      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
@@ -864,9 +826,9 @@
      }
    },
    "node_modules/fs-extra": {
-      "version": "11.3.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
-      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
+      "version": "11.4.1",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.1.tgz",
+      "integrity": "sha512-KYAb4c9BJQI6QqGKthV68OHe0badztdXJWKo0WtBA9IuCFPTKvE5ZdUBglP833aMjhaSPNO4A5j/EkzZtGlKjA==",
      "license": "MIT",
      "dependencies": {
        "graceful-fs": "^4.2.0",
@@ -877,12 +839,6 @@
        "node": ">=14.14"
      }
    },
-    "node_modules/fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-      "license": "ISC"
-    },
    "node_modules/function-bind": {
      "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -938,27 +894,6 @@
        "node": ">= 0.4"
      }
    },
-    "node_modules/glob": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
-      "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
-      "license": "ISC",
-      "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^3.1.1",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
-      },
-      "engines": {
-        "node": "*"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
    "node_modules/gopd": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -1074,12 +1009,6 @@
        "node": ">= 14"
      }
    },
-    "node_modules/https": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/https/-/https-1.0.0.tgz",
-      "integrity": "sha512-4EC57ddXrkaF0x83Oj8sM6SLQHAWXw90Skqu2M4AEWENZ3F02dFJE/GARA8igO79tcgYqGrD7ae4f5L3um2lgg==",
-      "license": "ISC"
-    },
    "node_modules/https-proxy-agent": {
      "version": "5.0.1",
      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
@@ -1142,51 +1071,17 @@
      "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.11.tgz",
      "integrity": "sha512-YChdK5txDjwGUvgR7oCJcLGkwi3LDh8Zx8tS7ndYciLEg/oOCL26VUBSUEtgS7EjiybcBqFxh5j0rAdDX7/bbg==",
      "license": "MIT",
+      "peer": true,
      "engines": {
        "node": ">= 4"
      }
    },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
-      "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
-      "license": "ISC",
-      "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
-      }
-    },
    "node_modules/inherits": {
      "version": "2.0.4",
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
      "license": "ISC"
    },
-    "node_modules/interpret": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
-      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 0.10"
-      }
-    },
-    "node_modules/is-core-module": {
-      "version": "2.17.0",
-      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz",
-      "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==",
-      "license": "MIT",
-      "dependencies": {
-        "hasown": "^2.0.4"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/is-docker": {
      "version": "2.2.1",
      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
@@ -1251,9 +1146,9 @@
      }
    },
    "node_modules/js-base64": {
-      "version": "3.9.3",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.3.tgz",
-      "integrity": "sha512-uwYQp+VJ38FVvtim6qNbit6e9uT6dwWQ4Y1+H9TxhW5hcHjpHwoxlR0nMpqUmIFOmu4VqMxwdJA88gIVuZJQ/g==",
+      "version": "3.9.4",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.4.tgz",
+      "integrity": "sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q==",
      "license": "BSD-3-Clause"
    },
    "node_modules/jsdom": {
@@ -1309,18 +1204,6 @@
        "node": ">= 14"
      }
    },
-    "node_modules/jsdom/node_modules/tough-cookie": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
-      "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
-      "license": "BSD-3-Clause",
-      "dependencies": {
-        "tldts": "^6.1.32"
-      },
-      "engines": {
-        "node": ">=16"
-      }
-    },
    "node_modules/jsonfile": {
      "version": "6.2.1",
      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz",
@@ -1334,10 +1217,13 @@
      }
    },
    "node_modules/jwt-decode": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.1.2.tgz",
-      "integrity": "sha512-UfpWE/VZn0iP50d8cz9NrZLM9lSWhcJ+0Gt/nm4by88UL+J1SiKN8/5dkjMmbEzwL2CAe+67GsegCbIKtbp75A==",
-      "license": "MIT"
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-4.0.0.tgz",
+      "integrity": "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
    },
    "node_modules/kind-of": {
      "version": "6.0.3",
@@ -1363,37 +1249,6 @@
      "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
      "license": "MIT"
    },
-    "node_modules/lodash.groupby": {
-      "version": "4.6.0",
-      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
-      "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isboolean": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
-      "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isequal": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
-      "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==",
-      "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isfunction": {
-      "version": "3.0.9",
-      "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz",
-      "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isnil": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz",
-      "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==",
-      "license": "MIT"
-    },
    "node_modules/lodash.uniqby": {
      "version": "4.7.0",
      "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
@@ -1461,18 +1316,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/minimatch": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz",
-      "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==",
-      "license": "ISC",
-      "dependencies": {
-        "brace-expansion": "^1.1.7"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
    "node_modules/ms": {
      "version": "2.1.3",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -1519,15 +1362,6 @@
      "integrity": "sha512-gQPNF78qebCQ6tvVFBYrvJdBNOrYZm90ZlXgpIFm06p6qHDHq/XC4TnJftN6OMbxVE0UTBAoRgcsDeJBBooITw==",
      "license": "MIT"
    },
-    "node_modules/once": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-      "license": "ISC",
-      "dependencies": {
-        "wrappy": "1"
-      }
-    },
    "node_modules/onetime": {
      "version": "5.1.2",
      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
@@ -1624,21 +1458,6 @@
        "url": "https://github.com/inikulin/parse5?sponsor=1"
      }
    },
-    "node_modules/path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
-    "node_modules/path-parse": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
-      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
-      "license": "MIT"
-    },
    "node_modules/pend": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
@@ -1667,18 +1486,6 @@
        "node": ">=10"
      }
    },
-    "node_modules/psl": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz",
-      "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==",
-      "license": "MIT",
-      "dependencies": {
-        "punycode": "^2.3.1"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/lupomontero"
-      }
-    },
    "node_modules/punycode": {
      "version": "2.3.1",
      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
@@ -1688,12 +1495,6 @@
        "node": ">=6"
      }
    },
-    "node_modules/querystringify": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
-      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
-      "license": "MIT"
-    },
    "node_modules/readable-stream": {
      "version": "3.6.2",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -1708,17 +1509,6 @@
        "node": ">= 6"
      }
    },
-    "node_modules/rechoir": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
-      "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
-      "dependencies": {
-        "resolve": "^1.1.6"
-      },
-      "engines": {
-        "node": ">= 0.10"
-      }
-    },
    "node_modules/require-directory": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -1728,33 +1518,6 @@
        "node": ">=0.10.0"
      }
    },
-    "node_modules/requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
-      "license": "MIT"
-    },
-    "node_modules/resolve": {
-      "version": "1.22.12",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz",
-      "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==",
-      "license": "MIT",
-      "dependencies": {
-        "es-errors": "^1.3.0",
-        "is-core-module": "^2.16.1",
-        "path-parse": "^1.0.7",
-        "supports-preserve-symlinks-flag": "^1.0.0"
-      },
-      "bin": {
-        "resolve": "bin/resolve"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/restore-cursor": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz",
@@ -1880,18 +1643,6 @@
        "node": ">=8"
      }
    },
-    "node_modules/supports-preserve-symlinks-flag": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
-      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
    "node_modules/symbol-tree": {
      "version": "3.2.4",
      "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
@@ -1917,27 +1668,15 @@
      "license": "MIT"
    },
    "node_modules/tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
+      "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
      "license": "BSD-3-Clause",
      "dependencies": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
+        "tldts": "^6.1.32"
      },
      "engines": {
-        "node": ">=6"
-      }
-    },
-    "node_modules/tough-cookie/node_modules/universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 4.0.0"
+        "node": ">=16"
      }
    },
    "node_modules/tr46": {
@@ -1952,18 +1691,18 @@
        "node": ">=18"
      }
    },
-    "node_modules/traverse-chain": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
-      "integrity": "sha512-up6Yvai4PYKhpNp5PkYtx50m3KbwQrqDwbuZP/ItyL64YEWHAvH6Md83LFLV/GRSk/BoUVwwgUzX6SOQSbsfAg==",
-      "license": "MIT"
-    },
    "node_modules/trim-buffer": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/trim-buffer/-/trim-buffer-5.0.0.tgz",
      "integrity": "sha512-PqdsRmhir3lUwraK/O6sdZ9QcmtUK0c6F1utX/WNquJsL4Jpdck+DSe9Mqy8Wk7uAQD/SlVn6cyGF5bj6VF33Q==",
      "license": "MIT"
    },
+    "node_modules/undici-types": {
+      "version": "8.9.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
+      "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
+      "license": "MIT"
+    },
    "node_modules/universalify": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
@@ -1973,27 +1712,12 @@
        "node": ">= 10.0.0"
      }
    },
-    "node_modules/url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
-      "license": "MIT",
-      "dependencies": {
-        "querystringify": "^2.1.1",
-        "requires-port": "^1.0.0"
-      }
-    },
    "node_modules/util-deprecate": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
      "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
      "license": "MIT"
    },
-    "node_modules/valid-url": {
-      "version": "1.0.9",
-      "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz",
-      "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA=="
-    },
    "node_modules/w3c-xmlserializer": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz",
@@ -2076,12 +1800,6 @@
        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
      }
    },
-    "node_modules/wrappy": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-      "license": "ISC"
-    },
    "node_modules/ws": {
      "version": "8.21.3",
      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
@@ -30,11 +30,10 @@
  },
  "private": true,
  "dependencies": {
-    "@sasjs/cli": "4.20.5",
-    "@sasjs/core": "5.2.8"
+    "@sasjs/cli": "6.0.0",
+    "@sasjs/core": "6.0.0"
  },
  "overrides": {
-    "nanoid": "3.3.18",
-    "shelljs": "0.8.5"
+    "nanoid": "3.3.18"
  }
 }
@@ -0,0 +1,32 @@
+/**
+  @file mpe_derived.ddl
+  @brief ddl file
+  @details Defines the derived (computed) variable configuration table.  This
+  drives MPE_DERIVED rows in MPE_TABLES, so that the table itself can be
+  maintained through the EDIT screen with the usual approve workflow.
+
+  Derived values are not persisted - the logic is applied at read time, before
+  the filter, so that the variables can be filtered on.
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+
+create table &curlib..MPE_DERIVED
+   (
+    TX_FROM num not null format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VAR_NM varchar(32),
+    VAR_DEPS varchar(4000),
+    VAR_TYPE varchar(1),
+    VAR_LEN int,
+    VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
+    VAR_LOGIC varchar(4000),
+    ACTIVE int,
+    TX_TO num not null format=datetime19.3,
+    constraint pk_mpe_derived
+        primary key(tx_from, libref, dsn, var_nm));
@@ -118,3 +118,16 @@ insert into &mpelib..mpe_tables
      ,var_txfrom='TX_FROM'
      ,var_txto='TX_TO'
    ;
+  insert into &mpelib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&mpelib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;
@@ -97,6 +97,10 @@ quit;
      set tx_to=&now
      where var_name='DC_MAXCELLS_WEBEDIT' and var_active=1 and &now lt tx_to;

+    /* Why the cell limit exists, kept out of var_desc - MPE_CONFIG.VAR_DESC
+      is varchar(300), and a truncation WARNING aborts the build.  A wide
+      table is expensive long before it is tall - 250 rows of a 1000 column
+      table is a quarter of a million cells. */
    insert into &dclib..mpe_config
      set tx_from=&now
        ,tx_to='31DEC9999:23:59:59'dt
@@ -106,10 +110,8 @@ quit;
        ,var_active=1
        ,var_desc='This sets the maximum number of CELLS (rows multiplied by'
          !!' columns) that can be loaded into the browser for editing in the'
-          !!' EDIT screen.  It exists because a wide table is expensive long'
-          !!' before it is tall - 250 rows of a 1000 column table is a quarter'
-          !!' of a million cells.  Whichever of this and DC_MAXOBS_WEBEDIT is'
-          !!' reached first applies.';
+          !!' EDIT screen.  Whichever of this and DC_MAXOBS_WEBEDIT is reached'
+          !!' first applies.';
  quit;
 %end;
 %else %do;
@@ -0,0 +1,169 @@
+/**
+  @file
+  @brief migration script to move to v7.19 of Data Controller
+
+  REQUIRED CHANGE - adds the MPE_DERIVED table, which holds derived (computed)
+  variable definitions for the EDIT screen, and registers it in MPE_TABLES so
+  that it can be maintained through the app with the usual approve workflow.
+
+  Replace YOURDCLIB and the library path below, then run the script.  Any
+  MPE_DERIVED rows added before this script runs will be lost when the table
+  is created, so run it before configuring any derived variables.
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+proc sql;
+create table &dclib..mpe_derived
+  (
+    tx_from num not null format=datetime19.3,
+    libref varchar(8),
+    dsn varchar(32),
+    var_nm varchar(32),
+    var_deps varchar(4000),
+    var_type varchar(1),
+    var_len num,
+    var_fmt varchar(32),
+    var_label varchar(256),
+    var_logic varchar(4000),
+    active num,
+    tx_to num not null format=datetime19.3
+  );
+quit;
+
+proc datasets lib=&dclib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn var_nm)
+    /nomiss unique;
+quit;
+
+proc sql;
+insert into &dclib..mpe_tables
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="&dclib"
+    ,dsn='MPE_DERIVED'
+    ,num_of_approvals_required=1
+    ,loadtype='TXTEMPORAL'
+    ,buskey='LIBREF DSN VAR_NM'
+    ,post_edit_hook='services/hooks/mpe_derived_postedit'
+    ,notes='Derived (computed) variables for the EDIT screen'
+    ,var_txfrom='TX_FROM'
+    ,var_txto='TX_TO'
+  ;
+quit;
+
+/* Dropdown values for VAR_TYPE */
+proc sql noprint;
+  select max(selectbox_rk) into: maxrk from &dclib..mpe_selectbox;
+quit;
+
+%let rk1=%eval(&maxrk+1);
+%let rk2=%eval(&maxrk+2);
+
+proc sql;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk1
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='C'
+    ,selectbox_order=1
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk2
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='N'
+    ,selectbox_order=2
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+quit;
+
+/* Field rules for the EDIT screen.  These are client side - the server side
+  * enforcement is the post edit hook registered above. */
+proc sql;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+quit;
@@ -0,0 +1,103 @@
+/**
+  @file
+  @brief Generates the SAS statements that create Data Controller derived variables
+  @details Derived (computed) variables are configured in MPE_DERIVED.  Each
+  row defines one variable, which is created in a DATA step view ahead of the
+  filter, so that it can be filtered on in the EDIT screen.
+
+  The engine lives in @sasjs/core as mp_derivedvars - this macro only supplies
+  what is specific to Data Controller: the control table name, the bitemporal
+  filter that picks the current version of a row, and the list of names that
+  belong to the metadata and may not be used as a variable.
+
+  The macro writes the generated statements to a fileref, so that callers can
+  compose them into a view:
+
+      %mpe_derived(libds=&libds,dclib=&mpelib,outref=dercode)
+      data work.vw_derived / view=work.vw_derived;
+        set &libds;
+        %include dercode;
+      run;
+
+  Alternatively, pass outview= and the macro will build the view itself.
+
+  The filter must be applied by the caller, in the step that reads the view:
+
+      data out;
+        set work.vw_derived;
+        where %inc filtref;;
+      run;
+
+  A WHERE statement applies to the data set named on the SET statement, so it
+  cannot reference a variable that is created in the same data step.  Applying
+  it to the view puts the derived variables on the input, which is what makes
+  them filterable.
+
+  Each statement block is of the form:
+
+      attrib REGION length=$21 label='REGION=<expression>' format=$21.;
+      REGION = <expression>;
+
+  Type and length are declared by the administrator, so the result is
+  deterministic.  VAR_LABEL is optional: when it is missing or blank the label
+  is the variable name, an equals sign and the expression, otherwise the
+  configured label is used as given.
+
+  VAR_DEPS lists every input the expression uses - physical columns and
+  derived variables alike.  The statements are generated in dependency order,
+  so a derived variable may build on another one without a sequence number
+  being maintained.  A cycle is rejected.
+
+  Semicolons, macro triggers and comments are rejected, so a row can only ever
+  be an expression.  A derived variable may not reuse the name of a physical
+  column, and may not reference itself: an intra-row calculation would imply a
+  retained value, which depends on the order rows happen to arrive in.
+
+  <h4> SAS Macros </h4>
+  @li mp_derivedvars.sas
+  @li mp_abort.sas
+
+  @param [in] libds= The target table (libref.dataset, lib must be assigned)
+  @param [in] dclib= The Data Controller control library
+  @param [out] outref= (0) Fileref to receive the generated statements
+  @param [out] outvars= (work.derived_vars) Dataset of derived variable names
+  @param [in] outview= (0) Optional two-level name for the derived view
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%macro mpe_derived(libds=,
+  dclib=,
+  outref=0,
+  outvars=work.derived_vars,
+  outview=0
+);
+
+%local reserved;
+
+/* the only name that has to be refused outright - the metadata columns of the
+  target table are physical columns, so the engine's collision check covers
+  them whatever MPE_TABLES calls them */
+%let reserved=%str(_____DELETE__THIS__RECORD_____);
+
+%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing: libds dclib)
+)
+
+%mp_derivedvars(
+  libds=&libds
+  ,control=&dclib..mpe_derived
+  ,where=%str(&dc_dttmtfmt lt tx_to and active=1)
+  ,outref=&outref
+  ,outvars=&outvars
+  ,outview=&outview
+  ,reserved=&reserved
+)
+
+%mend mpe_derived;
@@ -0,0 +1,296 @@
+/**
+  @file
+  @brief testing the mpe_derived macro
+
+  @details Covers the generated statements, the view, the metadata the callers
+  consume, and the dependency rules.
+
+  The configuration checks abort the run when they fail, so they are not
+  exercised here - this suite covers the behaviour that callers depend on.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mpe_derived.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+
+/* a source table, and a table with no derived variables */
+data work.class;
+  set sashelp.class;
+run;
+data work.noconfig;
+  set sashelp.class;
+run;
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+/**
+  * Test 1 - statements, view, chaining
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='BAND'
+    ,var_deps='AGE'
+    ,var_type='C'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_label='Age band'
+    ,var_logic="ifc(AGE>13,'TEEN','JUNIOR')"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='BAND_AGE'
+    ,var_deps='BAND AGE'
+    ,var_type='C'
+    ,var_len=16
+    ,var_fmt=''
+    ,var_logic="cats(BAND,'-',AGE)"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='DOUBLE_AGE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE * 2'
+    ,active=1;
+quit;
+
+/* lower case libds on purpose - the config match must be case insensitive */
+/* the setup SQL above creates the SQL automatic macro variables, so the
+  snapshot is taken here - the check is for the macro's own leakage */
+%mp_assertscope(SNAPSHOT)
+
+%mpe_derived(libds=work.class,
+  dclib=&dclib,
+  outref=derref1,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+
+%mp_assertdsobs(work.derived_vars,
+  desc=Test1 - one row per derived variable,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+%mp_assertcols(work.vw_derived,
+  desc=Test1 - the view carries the derived columns,
+  cols=BAND BAND_AGE DOUBLE_AGE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk1;
+  set work.vw_derived;
+  where BAND='TEEN';
+run;
+%mp_assertdsobs(work.chk1,
+  desc=Test1 - the filter on a derived column selects rows,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* the chained value only resolves if the statements run in order */
+data work.chk1b;
+  set work.vw_derived;
+  where BAND='TEEN';
+  if BAND_AGE ne cats(BAND,'-',AGE) then output;
+run;
+%mp_assertdsobs(work.chk1b,
+  desc=Test1 - a derived variable can reference an earlier one,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+data work.chk1c;
+  set work.vw_derived;
+  if DOUBLE_AGE ne AGE*2 then output;
+run;
+%mp_assertdsobs(work.chk1c,
+  desc=Test1 - the numeric derived value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* proc contents on a view reports a format name without its width, so the
+  format is compared on the name only */
+proc contents noprint data=work.vw_derived
+  out=work.vw_cols(keep=name label format);
+run;
+data work.chk1d;
+  set work.vw_cols;
+  where name='BAND' and label='Age band' and format=:'$';
+run;
+%mp_assertdsobs(work.chk1d,
+  desc=Test1 - the label and default format are applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - a configured format is used as-is
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='FLAG_DT'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt='DATE9.'
+    ,var_logic='mdy(1,1,2020)'
+    ,active=1;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref2,
+  outview=work.vw_derived2,
+  outvars=work.derived_vars2
+)
+
+/* asserted on the emitted statement - proc contents on a view does not
+  report the format in full */
+data work.chk2;
+  infile derref2 truncover;
+  input line $4000.;
+  if index(line,'format=DATE9.')>0 then output;
+run;
+%mp_assertdsobs(work.chk2,
+  desc=Test2 - the configured format is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk2b;
+  set work.vw_derived2;
+  if flag_dt ne mdy(1,1,2020) then output;
+  stop;
+run;
+%mp_assertdsobs(work.chk2b,
+  desc=Test2 - the numeric value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - no configuration
+  */
+
+%mpe_derived(libds=WORK.NOCONFIG,
+  dclib=&dclib,
+  outref=derref3,
+  outview=work.vw_derived3,
+  outvars=work.derived_vars3
+)
+
+%mp_assertdsobs(work.derived_vars3,
+  desc=Test3 - no derived variables returned,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+%mp_assert(iftrue=(%mf_existds(libds=work.vw_derived3)=0),
+  desc=Test3 - no view is built when there is no config,
+  outds=work.test_results
+)
+
+data _null_;
+  infile derref3 end=eof;
+  input;
+  if not eof or cats(_infile_) ne '' then do;
+    putlog 'ERR' 'OR: derived statement file is not empty';
+    call symputx('emptycheck','0','l');
+  end;
+run;
+%mp_assert(iftrue=(%symexist(emptycheck)=0),
+  desc=Test3 - no statements are generated,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - the wrapper scopes the config: only the current, active row of a
+  * table is used
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='01JAN2020:00:00:00'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='CLOSED_OFF'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 1'
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='NOT_ACTIVE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 2'
+    ,active=0;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.chk4;
+  set work.derived_vars4;
+  where name in ('CLOSED_OFF','NOT_ACTIVE');
+run;
+
+%mp_assertdsobs(work.chk4,
+  desc=Test4 - a closed version and an inactive row are ignored,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* tidy up so repeated runs start clean */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+%mp_assertscope(COMPARE,ignorelist=_SASJS_WEBOUT_HEADERS,outds=work.test_results)
@@ -79,6 +79,10 @@ insert into &lib..mpe_config set
      !!' will require a decent browser (ie, not IE) and more memory on the'
      !!' client side.  The DC_MAXCELLS_WEBEDIT limit is applied at the same'
      !!' time, whichever is reached first.';
+/* Why the cell limit exists, kept out of var_desc - MPE_CONFIG.VAR_DESC is
+  varchar(300), and a truncation WARNING aborts this whole build.  A wide
+  table is expensive long before it is tall - 250 rows of a 1000 column
+  table is a quarter of a million cells. */
 insert into &lib..mpe_config set
    tx_from=0
    ,tx_to='31DEC9999:23:59:59'dt
@@ -88,9 +92,7 @@ insert into &lib..mpe_config set
    ,var_active=1
    ,var_desc='This sets the maximum number of CELLS (rows multiplied by'
      !!' columns) that can be loaded into the browser for editing in the EDIT'
-      !!' screen.  It exists because a wide table is expensive long before it'
-      !!' is tall - 250 rows of a 1000 column table is a quarter of a million'
-      !!' cells.  Whichever of this and DC_MAXOBS_WEBEDIT is reached first'
+      !!' screen.  Whichever of this and DC_MAXOBS_WEBEDIT is reached first'
      !!' applies.';
 insert into &lib..mpe_config set
    tx_from=0
@@ -677,6 +679,24 @@ insert into &lib..mpe_security set
      ,selectbox_value='UNLOCKED'
      ,selectbox_order=2
      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='C'
+      ,selectbox_order=1
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='N'
+      ,selectbox_order=2
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
  insert into &lib..mpe_selectbox set
      selectbox_rk=%mf_increment(rk)
      ,ver_from_dttm=0
@@ -1493,6 +1513,19 @@ insert into &lib..mpe_selectbox set
      ,var_txfrom='TX_FROM'
      ,var_txto='TX_TO'
    ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;
  insert into &lib..mpe_tables
    set tx_from=0
      ,tx_to='31DEC5999:23:59:59'dt
@@ -1942,6 +1975,87 @@ insert into &lib..MPE_VALIDATIONS set
    ,rule_value="services/validations/columns_in_libds"
    ,rule_active=1
    ,tx_to='31DEC5999:23:59:59'dt;
+
+/* MPE_DERIVED - the field rules are also enforced server-side, in
+  * services/hooks/mpe_derived_postedit.sas.  These rows give the user
+  * immediate feedback in the EDIT screen. */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
    tx_from=0
    ,base_lib="&lib"
@@ -692,6 +692,29 @@ proc datasets lib=&lib noprint;
    pk_mpe_validations=(tx_from base_lib base_ds base_col rule_type)
    /nomiss unique;
 quit;
+/* mpe_derived - computed variables for the EDIT screen */
+proc sql;
+create table &lib..MPE_DERIVED
+  (
+    TX_FROM num &notnull format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VAR_NM varchar(32),
+    VAR_DEPS varchar(4000),
+    VAR_TYPE varchar(1),
+    VAR_LEN num,
+    VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
+    VAR_LOGIC varchar(4000),
+    ACTIVE num,
+    TX_TO num &notnull format=datetime19.3
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn var_nm)
+    /nomiss unique;
+quit;
 proc sql;
 create table &lib..mpe_x_test(
          primary_key_field num &notnull,
@@ -17,6 +17,10 @@
  @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
    else set to 1 (CLS rules exist)
  @li ISMAP - set to 1 if the target DS is an excel map target, else 0
+  @li TRUNCATED - set to 1 if the selection was cut down to the
+    DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
+  @li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
+    else empty

  <h5> approvers </h5>
  <h5> dqrules </h5>
@@ -68,11 +72,14 @@
  @li mp_validatecol.sas
  @li mpe_accesscheck.sas
  @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
  @li mpe_dsmeta.sas
  @li mpe_getlabels.sas
  @li mpe_getversions.sas
  @li mpe_filtermaster.sas
  @li mpe_runhook.sas
+  @li mpeinit.sas
+  @li mpeterm.sas

  @version 9.2
  @author 4GL Apps Ltd
@@ -201,6 +208,21 @@ select upcase(loadtype)
  outds=work.query
 )

+/**
+  * Derived (computed) variables, configured in MPE_DERIVED.  They are added
+  * via a view, so that the filter (applied by the step that reads the view)
+  * can reference them - a WHERE clause cannot see a variable that is created
+  * in the same data step.
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%put &=derived_nobs;
+
 %macro mpestp_getdata();
  %if not %symexist(DC_MAXOBS_WEBEDIT) %then %do;
    %put NOTE:;%put NOTE- DC_MAXOBS_WEBEDIT not found!;
@@ -237,8 +259,12 @@ select upcase(loadtype)

  %if &loadtype=BITEMPORAL %then %do;
    data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
-      _____DELETE__THIS__RECORD_____="No";
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
      where %inc filtref;;
    run;
    proc sort data=out;
@@ -252,8 +278,12 @@ select upcase(loadtype)
  %end;
  %else %do;
    data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
-      _____DELETE__THIS__RECORD_____="No";
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
      where %inc filtref;;
    run;
  %end;
@@ -264,49 +294,63 @@ select upcase(loadtype)
  )

  /* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
-    then run the hook, then check the row and cell limits.  Checking once,
-    after both, means the counts cover everything that could reach the
-    browser: the rows the selection matched, and anything the hook has added
-    to work.out.  A wide table exhausts the cell budget long before it
-    reaches the row limit - 250 rows of a 1000 column table is a quarter of
-    a million cells - so capping rows alone does not bound the work the
-    browser is asked to do.  The payload itself is capped further down, when
-    work.outdata is built. */
+    then run the hook, then apply the row and cell limits (below). */
  %let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
  proc sort data=work.out; by &sortPK; run;

  %mpe_runhook(PRE_EDIT_HOOK)

+  /* Serve the selection truncated, at whichever limit binds first, and carry
+    the reason in sasparams for the EDIT screen to show against the table
+    name.  Bounding the browser's work is the point: a wide table exhausts the
+    cell budget long before it reaches the row limit - 250 rows of a 1000
+    column table is a quarter of a million cells - so the row cap is
+    min(DC_MAXOBS_WEBEDIT, DC_MAXCELLS_WEBEDIT / columns).  Counting after the
+    PRE_EDIT_HOOK means the cap covers what the hook added to work.out. */
  %let obscnt=%mf_getattrn(work.out,NLOBS);
  %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
-  %let cellcnt=%eval(&obscnt * &colcnt);
-  %mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Table is too big (&obscnt rows) - please filter and try again!
-  )
-  %mp_abort(iftrue=(&cellcnt>&DC_MAXCELLS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Selection is too wide (&obscnt rows x &colcnt cols) - please filter and try again!
-  )
+  %let maxrows=&DC_MAXOBS_WEBEDIT;
+  %let limitvar=DC_MAXOBS_WEBEDIT;
+  %if &colcnt>0 %then %do;
+    %let cellrows=%sysfunc(floor(&DC_MAXCELLS_WEBEDIT/&colcnt));
+    %if %sysevalf(&cellrows lt &maxrows) %then %do;
+      %let maxrows=&cellrows;
+      %let limitvar=DC_MAXCELLS_WEBEDIT;
+    %end;
+  %end;
+  %if %sysevalf(&maxrows lt 1) %then %let maxrows=1;
+  %global truncated trunc_msg;
+  %let truncated=0;
+  %if %sysevalf(&obscnt gt &maxrows) %then %let truncated=1;
+  %let trunc_msg=;
+  %if &truncated=1 %then %do;
+    %let trunc_msg=Only the first &maxrows rows are loaded;
+    %let trunc_msg=&trunc_msg - the &limitvar limit has been reached.;
+    %let trunc_msg=&trunc_msg  Filter the table to work with the rest.;
+  %end;


-  /* order delete var and pk fields at start of table */
+  /* order pk fields at start of table.  The delete flag is NOT added here:
+    _____DELETE__THIS__RECORD_____ is a client-owned column, created by the
+    editor itself (see deleteRecordColumnRule.ts), and getdata must never
+    emit it - a real table column of that name is refused by the frontend. */
  %let sourcevars=%mf_wordsInStr1ButNotStr2(
                            Str1=%mf_getvarlist(work.out)
-                            ,Str2= _____DELETE__THIS__RECORD_____ &pk
+                            ,Str2=&pk
                          );
  %put sourcevars=&sourcevars;
-  /* the payload is capped here, at the row limit - the limits above are
-    checked against the whole selection, this is what goes to the browser */
-  options obs=&DC_MAXOBS_WEBEDIT;
+  /* the payload is capped at &maxrows - whichever limit that works out to be
+    - this is what goes to the browser */
+  options obs=&maxrows;
  data outdata;
-    /* delete & pk fields come first */
-    attrib _____DELETE__THIS__RECORD_____ &pk label='';
+    /* pk fields come first */
+    %if %length(&pk)>0 %then %do;
+      attrib &pk label='';
+    %end;
    /* keep remaining variable order */
    %if %length(&sourcevars)>0 %then %do;
      attrib &sourcevars label='';
    %end;
-    _____DELETE__THIS__RECORD_____="No ";
    %if %mf_nobs(work.out)=0 %then %do;
      /* send empty row if empty table to help with hot rendering */
      output;
@@ -326,7 +370,7 @@ select upcase(loadtype)
    by varnum;
  run;

-  data vars3(keep=name type length format label pk varnum  ctrloptions formatd);
+  data vars3(keep=name type length format label pk varnum formatd);
    set vars(rename=(format=format2 type=type2));
    name=upcase(name);
    /* not interested in transaction or processing dates
@@ -347,10 +391,6 @@ select upcase(loadtype)

    if name in ('',%upcase(%mf_getQuotedStr(&pk,dlm=%str(,),quote=S)))
      then PK='YES';
-
-    length ctrlOptions $500;
-    if name="_____DELETE__THIS__RECORD_____" then ctrlOptions='["No","Yes"]';
-    else ctrlOptions='';
  run;
  %mp_abort(iftrue= (&syscc ne 0)
    ,mac=&_program..sas
@@ -365,9 +405,7 @@ select upcase(loadtype)
    length coltype $500.;
    format=upcase(format);
    coltype=cats('{"data":"',name,'"');
-    if ctrlOptions ne '' then
-      colType=cats(coltype,',"type":"dropdown","source":',ctrlOptions,"}");
-    else if type='num' then do;
+    if type='num' then do;
      if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
      then do;
        colType=cats(coltype,',"type":"datetime"}');
@@ -517,13 +555,28 @@ select upcase(loadtype)
 /* extract col info */
 %mp_getcols(&libds, outds=cols1)

+/* derived variables are added to the view, so take their attributes from
+  there - this keeps type, length, format and label identical to the data */
+%if &derived_nobs>0 %then %do;
+  %mp_getcols(work.vw_derived, outds=derived_cols_all)
+  proc sql;
+  create table work.derived_cols as
+    select a.*
+    from work.derived_cols_all a
+    where a.NAME in (select NAME from work.derived_vars);
+  quit;
+  data work.cols1;
+    set work.cols1 work.derived_cols;
+  run;
+%end;
+
 /* join with cls rules.  Trim COLS payload in the SQL below - DDTYPE
  C=CHARACTER N=NUMERIC (dates kept in full), VARNUM dropped (client
  relies on column order, not position) */
 proc sql;
 create table work.cols as
  select a.NAME
-    ,coalesce(c.desc,a.NAME) as LABEL
+    ,coalesce(c.desc,a.LABEL,a.NAME) as LABEL
    ,a.FMTNAME
    ,case a.DDTYPE
      when 'CHARACTER' then 'C'
@@ -576,7 +629,7 @@ select count(*) into: ismap from &mpelib..mpe_xlmap_info
  where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;

 data sasparams;
-  length colHeaders $20000 filter_text $32767;
+  length colHeaders $20000 filter_text $32767 trunc_msg $200;
  colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
  pkCnt=countw("&pk");
  pk="&pk";
@@ -587,6 +640,10 @@ data sasparams;
  if trim(symget('rk_underlying')) ne '' then rk_flag=1;
  else rk_flag=0;
  filter_text=symget('filter_text');
+  /* set when the selection was cut down to the row / cell limit, so the EDIT
+    screen can say so against the table name */
+  truncated=&truncated;
+  trunc_msg="&trunc_msg";
  if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
  else cls_flag=1;
  put (_all_)(=);
@@ -714,11 +771,6 @@ run;

 %mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)

-data maxvarlengths;
-  set maxvarlengths;
-  if name='_____DELETE__THIS__RECORD_____' then mAXLEN=3;
-run;
-
 data xl_rules;
  set &mpelib..mpe_excel_config;
  where &dc_dttmtfmt. lt tx_to;
@@ -2,17 +2,19 @@
  @file
  @brief testing getdata row and cell limits

-  @details The EDIT screen refuses a selection that is too big on either
-  count - rows (DC_MAXOBS_WEBEDIT) or cells (DC_MAXCELLS_WEBEDIT).  The row
-  check runs first, so a selection over the row limit reports "Table is too
-  big" and one under the row limit but over the cell limit reports
-  "Selection is too wide".  Both aborts are proven by the child job state
-  (an aborted service registers no webout).
+  @details The EDIT screen is bounded by two limits - rows
+  (DC_MAXOBS_WEBEDIT) and cells, rows x columns (DC_MAXCELLS_WEBEDIT) -
+  whichever is reached first.  A selection over either is SERVED TRUNCATED:
+  the grid gets the first rows that fit, and sasparams carries TRUNCATED=1
+  plus a TRUNC_MSG naming the limit, which the EDIT screen shows as a warning
+  against the table name.

-  Fixture: dctest.wideboy (1000 rows x 1004 columns), created and registered
-  by testsetup.  Filtered to 300 rows it is over the 250 row limit; filtered
-  to 200 rows it is under the row limit but 200 x 1004 = 200,800 cells, over
-  the 200,000 cell limit.
+  Test 1 uses MPE_X_TEST (several hundred rows, few columns) so the row
+  limit alone decides the cap.  Test 2 uses the fixture dctest.wideboy (1000
+  rows x 1004 columns, created and registered by testsetup) filtered to 200
+  rows: that is under the row limit but 200 x 1004 = 200,800 cells, over the
+  200,000 cell limit, so the cell limit decides the cap -
+  floor(200000 / 1004) = 199 rows.

  <h4> SAS Macros </h4>
  @li mp_assert.sas
@@ -25,7 +27,7 @@
 %let _program=&appLoc/services/editors/getdata;

 /**
-  * build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
+  * Build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
  * in &filter_rk
  */
 %macro wideboy_filter(maxrow);
@@ -57,32 +59,56 @@
 %mend wideboy_filter;

 /**
-  * Test 1 - row limit (300 rows > 250)
+  * Call getdata as a child job and expose the webout as the &outlib JSON
+  * libname, so the test can read what the browser would have received.
  */
-%wideboy_filter(300)
-%let f1=%mf_getuniquefileref();
-data _null_;
-  file &f1 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%macro call_getdata(libds,filter_rk,outlib);
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref termstr=crlf;
+    put 'LIBDS:$43. FILTER_RK:8.';
+    put "&libds,&filter_rk";
+  run;
+  %mx_execute(&_program,
+    viyacontext=&defaultcontext,
+    inputfiles=&fref:sascontroltable,
+    outlib=&outlib
+  )
+  %let syscc=0;
+%mend call_getdata;

-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f1:sascontroltable,
-  outref=web1
-)
+/**
+  * Test 1 - row limit.  MPE_X_TEST is well under the cell limit, so the
+  * 250 row default is what caps the payload.
+  */
+%call_getdata(&dclib..MPE_X_TEST,-1,web1)

-%let abort1=0;
+%let t1_nobs=-1;
+%let t1_trunc=-1;
+%let t1_msg=;
+proc sql noprint;
+select count(*) into: t1_nobs from web1.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
+  set web1.sasparams;
+  call symputx('t1_trunc',truncated);
+  call symputx('t1_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
-%let syscc=0;

 %mp_assert(
-  iftrue=(&abort1=1),
-  desc=Test 1 - a selection over the row limit is refused,
+  iftrue=(&t1_trunc=1),
+  desc=Test 1 - a selection over the row limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t1_nobs=250),
+  desc=Test 1 - the payload is capped at DC_MAXOBS_WEBEDIT (250) rows,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t1_msg,DC_MAXOBS_WEBEDIT)>0),
+  desc=Test 1 - the truncation message names the DC_MAXOBS_WEBEDIT limit,
  outds=work.test_results
 )

@@ -90,28 +116,34 @@ run;
  * Test 2 - cell limit (200 rows x 1004 cols = 200,800 > 200,000)
  */
 %wideboy_filter(200)
-%let f2=%mf_getuniquefileref();
-data _null_;
-  file &f2 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%call_getdata(dctest.wideboy,&filter_rk,web2)

-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f2:sascontroltable,
-  outref=web2
-)
-
-%let abort2=0;
+%let t2_nobs=-1;
+%let t2_trunc=-1;
+%let t2_msg=;
+proc sql noprint;
+select count(*) into: t2_nobs from web2.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
+  set web2.sasparams;
+  call symputx('t2_trunc',truncated);
+  call symputx('t2_msg',trunc_msg);
+  putlog (_all_)(=);
 run;

 %mp_assert(
-  iftrue=(&abort2=1),
-  desc=Test 2 - a selection over the cell limit is refused,
+  iftrue=(&t2_trunc=1),
+  desc=Test 2 - a selection over the cell limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t2_nobs=199),
+  desc=Test 2 - the payload is capped at floor(DC_MAXCELLS_WEBEDIT / cols),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t2_msg,DC_MAXCELLS_WEBEDIT)>0),
+  desc=Test 2 - the truncation message names the DC_MAXCELLS_WEBEDIT limit,
  outds=work.test_results
 )

@@ -21,10 +21,10 @@
  editing MPE_GROUPS alone - Row Level Security is skipped for them - so the
  test reports a skip row instead of running the assertions.

-  The call passes a filter (PRIMARY_KEY_FIELD < 100) because getdata.sas also
-  enforces the row / cell limits, and the unfiltered MPE_X_TEST is above the
-  row limit (642 rows against DC_MAXOBS_WEBEDIT:250) - the limits test
-  (getdata.test.1.sas) covers that abort separately.
+  The call passes a filter (PRIMARY_KEY_FIELD < 100) so the whole selection is
+  served - the unfiltered MPE_X_TEST is above the row limit (642 rows against
+  DC_MAXOBS_WEBEDIT:250) and would come back truncated.  The limits test
+  (getdata.test.1.sas) covers that truncation separately.

  <h4> SAS Macros </h4>
  @li mf_getuniquefileref.sas
@@ -3,12 +3,14 @@
  @brief testing gethistory service

  <h4> SAS Macros </h4>
+  @li mp_assert.sas
  @li mp_assertcols.sas
  @li mp_assertcolvals.sas
  @li mp_assertdsobs.sas
  @li mf_getuniquefileref.sas
  @li mf_getuniquename.sas
  @li mp_filterstore.sas
+  @li mpe_derived.sas
  @li mx_execute.sas

 **/
@@ -240,3 +242,179 @@ run;
  test=EQUALS 10,
  outds=work.test_results
 )
+
+
+/**
+  * Test 4 - derived variables
+  */
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_deps='LIBREF DSN'
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
+    ,active=1;
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE2'
+    ,var_deps='DERIVED_NOTE'
+    ,var_type='C'
+    ,var_len=45
+    ,var_fmt=''
+    ,var_logic="cats(DERIVED_NOTE,' (2)')"
+    ,active=1;
+quit;
+
+/* a filter may reference the derived variable, so validate against the view */
+%mpe_derived(libds=&dc_libref..MPE_TABLES,
+  dclib=&dc_libref,
+  outref=dercode4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.inquery4;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
+;;;;
+run;
+%mp_filterstore(
+  libds=&dc_libref..MPE_TABLES,
+  targetds=work.vw_derived4,
+  filter_summary=&dc_libref..mpe_filteranytable,
+  filter_detail=&dc_libref..mpe_filtersource,
+  lock_table=&dc_libref..mpe_lockanytable,
+  maxkeytable=&dc_libref..mpe_maxkeyvalues,
+  queryds=work.inquery4,
+  outresult=work.result4,
+  outquery=work.query4
+)
+data _null_;
+  set work.result4;
+  call symputx('filter_rk4',filter_rk);
+run;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "&dclib..MPE_TABLES,&filter_rk4";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:sascontroltable,
+  outlib=web4,
+  mdebug=&sasjs_mdebug
+)
+
+/* a cancelled child returns no webout, so the outlib keeps whatever an
+  earlier call left in it - check the job state before reading its members,
+  otherwise the assertion failures below describe the wrong problem */
+%let abort4=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort4',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort4=0),
+  desc=Test4 - the derived variable call completes,
+  outds=work.test_results
+)
+
+%if &abort4=0 %then %do;
+  data sasdata4;
+    set web4.sasdata;
+  run;
+  data cols4;
+    set web4.cols;
+  run;
+%end;
+%else %do;
+  data cols4;
+    length name $32;
+    stop;
+  run;
+%end;
+
+%mp_assertcols(work.cols4,
+  desc=Test4 - derived column is in the COLS payload,
+  cols=DERIVED_NOTE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk4a;
+  set work.cols4;
+  where name='DERIVED_NOTE' and cls_rule='READ';
+run;
+%mp_assertdsobs(work.chk4a,
+  desc=Test4 - derived column is read only,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4b;
+  set work.cols4;
+  where name='DERIVED_NOTE' and label=:'DERIVED_NOTE=';
+run;
+%mp_assertdsobs(work.chk4b,
+  desc=Test4 - derived column is labelled,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+%mp_assertdsobs(work.sasdata4,
+  desc=Test4 - filter on the derived column is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4c;
+  set work.sasdata4;
+  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCK_ANYTABLE');
+run;
+%mp_assertdsobs(work.chk4c,
+  desc=Test4 - derived value is correct,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* the second variable is built on the first, so this proves statement order */
+data work.chk4d;
+  set work.sasdata4;
+  if derived_note2=cats(derived_note,' (2)');
+run;
+%mp_assertdsobs(work.chk4d,
+  desc=Test4 - a derived variable can reference an earlier one,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* tidy up the config so repeated runs start clean.  Delete every row for the
+  table, not just the first variable: leaving DERIVED_NOTE2 behind means the
+  next run starts with a dependency on a variable that is no longer declared,
+  and every call aborts */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;
@@ -22,6 +22,7 @@
  @li mf_getuniquefileref.sas
  @li mpe_loader.sas
  @li mpe_filtermaster.sas
+  @li mpe_derived.sas
  @li mp_abort.sas
  @li mp_binarycopy.sas
  @li mp_cntlout.sas
@@ -106,6 +107,37 @@ run;
 )

 %put Verify that the upload does not violate Row Level Security checks:;
+
+/* derived variables are display-only and must not be submitted, so drop them
+  from the data before the security check runs - otherwise a crafted
+  submission could satisfy an RLS clause that references a derived column */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outvars=work.derived_vars
+)
+%let derived_droplist=;
+%if %mf_nobs(work.derived_vars)>0 %then %do;
+  proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
+  run;
+  proc sql noprint;
+  select a.name into: derived_droplist separated by ' '
+    from work.derived_vars a
+    inner join work.jsdata_vars b
+    on a.name=b.name;
+  quit;
+%end;
+
+/* drop them from the submitted data itself - a DROP on the output of a later
+  step would leave the columns in that step's input, and the security clause
+  is evaluated against the input */
+%if %length(&derived_droplist)>0 %then %do;
+  data work.jsdata;
+    set work.jsdata;
+    drop &derived_droplist;
+  run;
+%end;
+
 %mpe_filtermaster(ULOAD,&libds,
  dclib=&mpelib,
  outref=filtref,
@@ -220,6 +252,9 @@ data _null_;
  put '%let ' name '=' value ';';
 run;

+/* derived variables are display-only - they never reach the approval package
+  or the target table.  work.jsdata was rewritten without them before the
+  security check, so there is nothing left to drop here */
 data approve.jsdset;
  length _____DELETE__THIS__RECORD_____ $3;
  set jsdata;
@@ -132,6 +132,88 @@ quit;
  outds=work.test_results
 )

+/**
+  * Test 3 - an RLS rule naming a DERIVED variable must be rejected at save
+  * time: the rule is evaluated against the submitted data, where a derived
+  * column does not exist
+  */
+%let t3=%sysfunc(datetime());
+
+/* define a derived variable on the target table */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_REPLACE'
+    ,var_nm='DERIVED_X'
+    ,var_deps='X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='RLS rule on a derived variable should abort';
+  libds="&dc_libref..MPE_ROW_LEVEL_SECURITY";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length RLS_SCOPE $8 RLS_GROUP $128 RLS_LIBREF $8 RLS_TABLE $32
+    RLS_GROUP_LOGIC $3 RLS_SUBGROUP_LOGIC $3 RLS_VARIABLE_NM $32
+    RLS_OPERATOR_NM $12 RLS_RAW_VALUE $4000
+    _____DELETE__THIS__RECORD_____ $3;
+  RLS_SCOPE='EDIT';
+  RLS_GROUP='SASAdministrators';
+  RLS_LIBREF='DCTEST';
+  RLS_TABLE='DC_REPLACE';
+  RLS_GROUP_LOGIC='AND';
+  RLS_SUBGROUP_LOGIC='OR';
+  RLS_SUBGROUP_ID=0;
+  RLS_VARIABLE_NM='DERIVED_X';
+  RLS_OPERATOR_NM='NE';
+  RLS_RAW_VALUE='1';
+  RLS_ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort3=0;
+proc sql noprint;
+select count(*) into: abort3
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t3
+    and reason_txt="RLS rule on a derived variable should abort"
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort3>0),
+  desc=Checking an RLS rule on a derived variable is rejected,
+  outds=work.test_results
+)
+
+/* tidy up the derived configuration */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+quit;
+
 /**
  * Test 2 - applying REPLACE loadtype to a table with RLS EDIT security
  * should abort.  The active EDIT rule was inserted in the prep section,
@@ -0,0 +1,413 @@
+/**
+  @file
+  @brief testing the derived-variable submit guards (MPE_DERIVED post edit hook)
+  @details Backend validations should abort when:
+    1. an active row's VAR_DEPS names an inactive (active=0) row of the same
+      table - the engine reads only active rows, so the dependency can never
+      resolve at run time
+    2. a row is staged with active=0 while a surviving active row depends on it
+
+  They should NOT abort when:
+    3. a new active row is added to a table that also holds an inactive row -
+      the inactive row is not part of the dependency graph, so it cannot
+      create a cycle or a stale reference
+    4. a delete-marked RLS rule naming a derived variable is submitted - the
+      rule is being removed, so its variables are not validated
+
+  The staged loads are aborted by the post edit hook inside the child service.
+  An aborted load leaves its MPE_LOADS row IN PROGRESS, so each assertion
+  matches on the unique reason_txt (the submitted message) rather than any load
+  in the time window - the suite runs in parallel sessions.
+
+  <h4> SAS Macros </h4>
+  @li mx_execute.sas
+  @li mp_assert.sas
+
+
+**/
+
+%let _program=&appLoc/services/editors/stagedata;
+
+/**
+  * Prep - a target table registration for the derived configuration, and the
+  * hook registration (some estates were initialised before the hook existed)
+  */
+proc sql noprint;
+update &dc_libref..mpe_tables
+  set post_edit_hook='services/hooks/mpe_derived_postedit'
+  where upcase(dsn)='MPE_DERIVED' and post_edit_hook='';
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+quit;
+
+/**
+  * Test 1 - a new active row depending on an inactive row must be rejected
+  */
+%let t1=%sysfunc(datetime());
+
+proc sql noprint;
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='INACTIVE_DEP'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=0;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Derived var depending on an inactive row should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
+    VAR_LOGIC $4000 VAR_FMT $32
+    _____DELETE__THIS__RECORD_____ $3;
+  LIBREF='DCTEST';
+  DSN='DC_DERIV';
+  VAR_NM='USES_INACTIVE';
+  VAR_DEPS='INACTIVE_DEP';
+  VAR_TYPE='N';
+  VAR_LEN=8;
+  VAR_FMT='';
+  VAR_LOGIC='INACTIVE_DEP + 1';
+  ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web1,
+  outref=wb1,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort1=0;
+proc sql noprint;
+select count(*) into: abort1
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t1
+    and reason_txt='Derived var depending on an inactive row should abort'
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort1>0),
+  desc=Checking a dependency on an inactive derived row is rejected,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - deactivating a row that a surviving active row depends on must be
+  * rejected: the control table still shows the row as active, so the check has
+  * to read the staged row too
+  */
+%let t2=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='BASE_X'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=1;
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='USES_BASE'
+    ,var_deps='BASE_X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='BASE_X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Deactivating a relied-on derived row should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dc_libref..mpe_derived(where=(upcase(libref)='DCTEST'
+    and upcase(var_nm)='BASE_X'));
+  length _____DELETE__THIS__RECORD_____ $3;
+  ACTIVE=0;
+  _____DELETE__THIS__RECORD_____='No';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web2,
+  outref=wb2,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort2=0;
+proc sql noprint;
+select count(*) into: abort2
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t2
+    and reason_txt='Deactivating a relied-on derived row should abort'
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort2>0),
+  desc=Checking deactivating a relied-on derived row is rejected,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - an inactive row must not block a submission that does not depend
+  * on it: with INACTIVE_DEP present (active=0) a new independent row loads
+  */
+%let t3=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='INACTIVE_DEP'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=0;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Active row next to an inactive row should load';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
+    VAR_LOGIC $4000 VAR_FMT $32
+    _____DELETE__THIS__RECORD_____ $3;
+  LIBREF='DCTEST';
+  DSN='DC_DERIV';
+  VAR_NM='FRESH';
+  VAR_DEPS='X';
+  VAR_TYPE='N';
+  VAR_LEN=8;
+  VAR_FMT='';
+  VAR_LOGIC='X + 1';
+  ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let status3=0;
+data _null_;
+  set web3.sasparams;
+  putlog (_all_)(=);
+  if status='SUCCESS' then call symputx('status3',1);
+run;
+
+%mp_assert(
+  iftrue=(&status3=1),
+  desc=Checking an inactive row does not block an unrelated derived variable,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - a delete-marked RLS rule naming a derived variable is a removal,
+  * so it must not abort: the rule is gone by the time the guards run
+  */
+%let t4=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='DERIVED_X'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=1;
+delete from &dc_libref..mpe_row_level_security
+  where upcase(rls_libref)='DCTEST' and upcase(rls_table)='DC_DERIV';
+select coalesce(max(rls_rk),0)+1 into: rlsrk
+  from &dc_libref..mpe_row_level_security;
+insert into &dc_libref..mpe_row_level_security
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,rls_rk=&rlsrk
+    ,rls_scope='EDIT'
+    ,rls_group='SASAdministrators'
+    ,rls_libref='DCTEST'
+    ,rls_table='DC_DERIV'
+    ,rls_group_logic='AND'
+    ,rls_subgroup_logic='OR'
+    ,rls_subgroup_id=0
+    ,rls_variable_nm='DERIVED_X'
+    ,rls_operator_nm='NE'
+    ,rls_raw_value='1'
+    ,rls_active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Delete-marked RLS rule on a derived variable should load';
+  libds="&dc_libref..MPE_ROW_LEVEL_SECURITY";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dc_libref..mpe_row_level_security(where=(upcase(rls_libref)='DCTEST'
+    and upcase(rls_table)='DC_DERIV'));
+  length _____DELETE__THIS__RECORD_____ $3;
+  _____DELETE__THIS__RECORD_____='Yes';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web4,
+  outref=wb4,
+  mdebug=&sasjs_mdebug
+)
+
+%let status4=0;
+data _null_;
+  set web4.sasparams;
+  putlog (_all_)(=);
+  if status='SUCCESS' then call symputx('status4',1);
+run;
+
+%mp_assert(
+  iftrue=(&status4=1),
+  desc=Checking a delete-marked RLS rule on a derived variable is not rejected,
+  outds=work.test_results
+)
+
+/**
+  * Test 5 - deleting a derived row that a surviving active row depends on must
+  * be rejected: the delete is not in the control table yet and the staged copy
+  * is delete-marked, so the inactive set has to admit it
+  */
+%let t5=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='BASE_X'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=1;
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='USES_BASE'
+    ,var_deps='BASE_X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='BASE_X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Deleting a relied-on derived row should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dc_libref..mpe_derived(where=(upcase(libref)='DCTEST'
+    and upcase(var_nm)='BASE_X'));
+  length _____DELETE__THIS__RECORD_____ $3;
+  _____DELETE__THIS__RECORD_____='Yes';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web5,
+  outref=wb5,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort5=0;
+proc sql noprint;
+select count(*) into: abort5
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t5
+    and reason_txt='Deleting a relied-on derived row should abort'
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort5>0),
+  desc=Checking deleting a relied-on derived row is rejected,
+  outds=work.test_results
+)
+
+/* tidy up so repeated runs start clean */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+delete from &dc_libref..mpe_row_level_security
+  where upcase(rls_libref)='DCTEST' and upcase(rls_table)='DC_DERIV';
+quit;
@@ -0,0 +1,433 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_DERIVED table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+  This hook normalises the incoming rows and applies the configuration rules
+  at submit time, so a bad derived variable is rejected while the submitter is
+  still watching - rather than breaking the EDIT screen later, when the view
+  is built.
+
+  VAR_DEPS must list every input the expression uses, physical columns
+  and derived variables alike.  The hook checks the part that does not need
+  the target library: a derived variable referenced by an expression must be
+  declared, a variable may not depend on itself, and the resulting graph must
+  be acyclic - across the incoming rows and the current rows they do not
+  replace.
+
+  The target table is not assigned in this context, so the collision between
+  VAR_NM and a physical column, and the resolution of a physical dependency,
+  are left to %mpe_derived, which runs with the target library assigned.
+  Uniqueness on the business key (LIBREF DSN VAR_NM) is checked by mpe_loader
+  before this hook runs, but on the raw submitted values, so it is case
+  sensitive: 'Region' and 'REGION' for the same table both pass the loader and
+  collide only at approve time.  The post-normalisation check below rejects
+  the second one at submit time, naming the variable.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mpe_derived.sas
+  @li mpe_loader.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let der_errflag=0;
+%let der_errmsg=;
+
+/* a hook is included at execution time, so this is open code - the macro
+  language has to be inside a macro definition for %local to be valid. The
+  control variables are set here, in the including scope (which %mp_include
+  makes local): a %global inside the macro would conflict with them and abort
+  the step. */
+%macro _mpe_derived_postedit();
+  %local left nready cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
+    cyc_n cyc_found;
+
+/* normalise the incoming values */
+data work.staging_ds;
+  set work.staging_ds;
+  if upcase(_____DELETE__THIS__RECORD_____) ne 'YES' then do;
+    libref=upcase(strip(libref));
+    dsn=upcase(strip(dsn));
+    var_nm=upcase(strip(var_nm));
+    var_type=upcase(strip(var_type));
+    var_fmt=upcase(strip(var_fmt));
+    var_label=strip(var_label);
+    var_deps=upcase(strip(var_deps));
+    var_logic=strip(var_logic);
+  end;
+run;
+
+/**
+  * Duplicate names after normalisation - mpe_loader compares the raw values,
+  * so a case variant reaches us here
+  */
+proc sql;
+create table work.derived_dup as
+  select libref, dsn, var_nm, count(*) as n
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  group by libref, dsn, var_nm
+  having count(*)>1;
+quit;
+
+data _null_;
+  set work.derived_dup(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',var_nm,' is declared more than once for '
+      ,libref,'.',dsn));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_dup))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/**
+  * Field level checks - the same rules %mpe_derived applies at read time
+  */
+data work.derived_check;
+  set work.staging_ds;
+  length reason $200;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  call missing(reason);
+
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if var_nm='' then reason='VAR_NM is required';
+  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VAR_NM must be a valid SAS name';
+  else if verify(substr(var_nm,1,1),'0123456789')=0
+    then reason='VAR_NM must not start with a digit';
+  else if var_nm='_____DELETE__THIS__RECORD_____'
+    then reason='VAR_NM is reserved';
+  else if var_type not in ('C','N')
+    then reason='VAR_TYPE must be C or N';
+  else if missing(var_len) or var_len ne int(var_len) or var_len<1
+    then reason='VAR_LEN must be a positive integer';
+  else if var_type='C' and var_len>32767
+    then reason='VAR_LEN exceeds 32767 for a character variable';
+  else if var_type='N' and var_len>8
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
+  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
+    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
+    then reason='VAR_FMT must be a format name, without ; & % or //*';
+  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VAR_FMT must be a format name';
+  else if missing(var_logic)
+    then reason='VAR_LOGIC is required';
+  else if index(var_logic,';')>0
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
+  else if indexc(var_logic,'&%')>0
+    then reason='VAR_LOGIC must not contain macro triggers';
+  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
+    then reason='VAR_LOGIC must not contain comments';
+
+  if not missing(reason) then output;
+run;
+
+%if %mf_nobs(work.derived_check)>0 %then %do;
+  data _null_;
+    set work.derived_check(obs=1);
+    call symputx('der_errmsg',cats('MPE_DERIVED: ',var_nm,': ',reason));
+    call symputx('der_errflag',1);
+  run;
+%end;
+
+%mp_abort(iftrue= (&der_errflag=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/**
+  * The graph has to hold for the resulting configuration: the incoming rows
+  * plus the current rows of the same tables that this submission does not
+  * replace. Only active rows are part of it - the engine reads nothing else
+  * (mpe_derived.sas), so an inactive row can neither resolve a dependency nor
+  * be resolved by one.
+  */
+proc sql;
+create table work.derived_rows as
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_deps)) as deps
+    ,strip(var_logic) as logic
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+    and active=1
+  union all
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_deps)) as deps
+    ,strip(var_logic) as logic
+  from &DC_LIBREF..mpe_derived
+  where &dc_dttmtfmt. lt tx_to
+    and active=1
+    and catx('.',libref,dsn,var_nm) not in
+      (select catx('.',libref,dsn,var_nm) from work.staging_ds)
+    and catx('.',libref,dsn) in
+      (select catx('.',libref,dsn) from work.staging_ds);
+quit;
+
+data work.derived_dep;
+  set work.derived_rows;
+  length dep_nm $32;
+  do i=1 to countw(deps,' ');
+    dep_nm=scan(deps,i,' ');
+    if not missing(dep_nm) then output;
+  end;
+  keep libref dsn name dep_nm;
+run;
+
+/* a dependency on a row the engine will not read cannot resolve: it aborts
+  every read of the table with "VAR_DEPS lists X, which is not a column of the
+  table or a derived variable". Catch it here, covering a row this submission
+  deactivates or deletes - neither shows in the control table yet. */
+proc sql;
+create table work.derived_baddep as
+  select d.libref, d.dsn, d.name, d.dep_nm
+  from work.derived_dep d
+  inner join (
+    select upcase(strip(libref)) as libref
+      ,upcase(strip(dsn)) as dsn
+      ,upcase(strip(var_nm)) as name
+    from work.staging_ds
+    where upcase(_____DELETE__THIS__RECORD_____) = 'YES'
+      or active ne 1
+    union all
+    select upcase(strip(libref)) as libref
+      ,upcase(strip(dsn)) as dsn
+      ,upcase(strip(var_nm)) as name
+    from &DC_LIBREF..mpe_derived
+    where &dc_dttmtfmt. lt tx_to
+      and active ne 1
+  ) inactive
+    on inactive.libref=d.libref
+    and inactive.dsn=d.dsn
+    and inactive.name=d.dep_nm;
+quit;
+
+data _null_;
+  set work.derived_baddep(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS lists ',dep_nm
+      ,', which is not active after this load - the engine cannot resolve it'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_baddep))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+data work.derived_ref;
+  set work.derived_rows;
+  length l $4000 tok $32;
+  l=prxchange("s/'([^']|'')*'//",-1,logic);
+  l=prxchange('s/"([^"]|"")*"//',-1,l);
+  l=prxchange('s/[^A-Za-z0-9_]/ /',-1,upcase(l));
+  do i=1 to countw(l,' ');
+    tok=scan(l,i,' ');
+    if not missing(tok) then output;
+  end;
+  keep libref dsn name tok;
+run;
+
+proc sql;
+create table work.derived_self as
+  select libref, dsn, name, dep_nm
+  from work.derived_dep
+  where name=dep_nm;
+
+create table work.derived_undeclared as
+  select r.libref, r.dsn, r.name, r.tok as dep_nm
+  from work.derived_ref r
+  where exists (
+      select 1 from work.derived_rows n
+      where n.libref=r.libref and n.dsn=r.dsn and n.name=r.tok
+    )
+    and r.tok ne r.name
+    and not exists (
+      select 1 from work.derived_dep d
+      where d.libref=r.libref and d.dsn=r.dsn and d.name=r.name
+        and d.dep_nm=r.tok
+    );
+quit;
+
+/* the message is always prepared, the abort is driven by the count - an
+  iftrue=1 inside a wrapping %if leaves the step alive on some engines */
+data _null_;
+  set work.derived_self(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_self))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+data _null_;
+  set work.derived_undeclared(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
+      ,', which is not in VAR_DEPS'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_undeclared))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/* the inverse of the check above: a declared name the expression never uses
+  is stale, and the read-time engine rejects it too */
+proc sql;
+create table work.derived_stale as
+  select d.libref, d.dsn, d.name, d.dep_nm
+  from work.derived_dep d
+  where not exists (
+    select 1 from work.derived_ref r
+    where r.libref=d.libref and r.dsn=d.dsn and r.name=d.name
+      and r.tok=d.dep_nm
+  );
+quit;
+
+data _null_;
+  set work.derived_stale(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS lists ',dep_nm
+      ,', which VAR_LOGIC does not reference'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_stale))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/* the same invariant the RLS hook enforces from its side: a derived variable
+  cannot be named by an active row level security rule of any scope. Only the
+  VIEW scope could evaluate it (on the view the EDIT screen reads), but the RLS
+  hook validates every rule against the physical table, where the column does
+  not exist, so the pairing is refused up front rather than failing there. */
+proc sql;
+create table work.derived_rls as
+  select d.libref, d.dsn, d.name
+  from work.derived_rows d
+  inner join &DC_LIBREF..mpe_row_level_security r
+    on upcase(r.rls_libref)=upcase(d.libref)
+    and upcase(r.rls_table)=upcase(d.dsn)
+    and upcase(r.rls_variable_nm)=upcase(d.name)
+  where r.rls_active=1
+    and &dc_dttmtfmt lt r.tx_to;
+quit;
+
+data _null_;
+  set work.derived_rls(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,' is named by a row level security rule on '
+      ,libref,'.',dsn,' - derived variables cannot be used there'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_rls))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/**
+  * Cycle check - repeatedly take the variables whose derived inputs have all
+  * been placed; a round that places nothing leaves a cycle
+  */
+proc sql;
+create table work.derived_edge as
+  select d.libref, d.dsn, d.name as child, d.dep_nm as parent
+  from work.derived_dep d
+  where d.dep_nm in (select name from work.derived_rows);
+quit;
+
+data work.derived_todo;
+  set work.derived_rows(keep=libref dsn name);
+run;
+
+%let left=%mf_nobs(work.derived_todo);
+%let cyc_found=0;
+
+%do %while (&left>0);
+  proc sql;
+  create table work.derived_ready as
+    select t.libref, t.dsn, t.name
+    from work.derived_todo t
+    where not exists (
+      select 1 from work.derived_edge e
+      inner join work.derived_todo t2
+        on e.libref=t2.libref and e.dsn=t2.dsn and e.parent=t2.name
+      where e.libref=t.libref and e.dsn=t.dsn and e.child=t.name
+    );
+  quit;
+
+  %let nready=%mf_nobs(work.derived_ready);
+  %if &nready=0 %then %do;
+    /* walk the edges, so the message names the cycle rather than just the
+      names that could not be placed */
+    %let cyc_n=%mf_nobs(work.derived_todo);
+    data _null_;
+      set work.derived_todo(obs=1);
+      call symputx('cyc_start',name);
+    run;
+    %let cyc_path=&cyc_start;
+    %let cyc_cur=&cyc_start;
+    %let cyc_seen=%str(.&cyc_start..);
+    %do cyc_i=1 %to &cyc_n;
+      %let cyc_next=;
+      proc sql noprint;
+        select e.parent into: cyc_next
+        from work.derived_edge e
+        where e.child="&cyc_cur"
+          and catx('.',e.libref,e.dsn,e.parent) in
+            (select catx('.',libref,dsn,name) from work.derived_todo);
+      quit;
+      %if %length(&cyc_next)=0 %then %let cyc_i=&cyc_n;
+      %else %do;
+        %let cyc_path=&cyc_path. -> &cyc_next;
+        %if %index(&cyc_seen,%str(.&cyc_next..))>0 %then %let cyc_i=&cyc_n;
+        %else %do;
+          %let cyc_seen=&cyc_seen.&cyc_next..;
+          %let cyc_cur=&cyc_next;
+        %end;
+      %end;
+    %end;
+    /* leave the loop before aborting - an abort that only cancels the step
+      would re-enter it with the ready count still zero */
+    %let cyc_found=1;
+    %let left=0;
+  %end;
+
+  %if &cyc_found=0 %then %do;
+    proc sql;
+    delete from work.derived_todo
+      where catx('.',libref,dsn,name) in
+        (select catx('.',libref,dsn,name) from work.derived_ready);
+    quit;
+
+    %let left=%mf_nobs(work.derived_todo);
+  %end;
+%end;
+
+%mp_abort(iftrue= (&cyc_found=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
+)
+
+%mend _mpe_derived_postedit;
+
+%_mpe_derived_postedit()
@@ -18,6 +18,7 @@

  <h4> SAS Macros </h4>
  @li dc_assignlib.sas
+  @li mf_nobs.sas
  @li mp_abort.sas
  @li mp_filtercheck.sas

@@ -54,7 +55,7 @@ run;
  ) change the loadtype.)
 )

-/* ignore scope and group for validation */
+/* group and subgroup are not used for validation */
 proc sql;
 create table work.batches as
  select distinct upcase(rls_libref) as rls_libref,
@@ -67,6 +68,7 @@ create table work.batches as
    rls_raw_value as raw_value
  from work.staging_ds
  where rls_active=1
+    and upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
  order by rls_libref, rls_table;

 %let cnt=0;
@@ -81,6 +83,37 @@ data _null_;
  end;
 run;

+/* derived variables exist only in the view the EDIT screen reads - a rule on
+  one cannot be validated against the physical table, so reject it here rather
+  than let mp_filtercheck fail with a variable-not-found error. Applies to
+  every scope: the VIEW scope could evaluate the column, but this hook checks
+  rules against the table they are submitted for. */
+proc sql;
+create table work.rls_derived as
+  select b.rls_libref, b.rls_table, b.variable_nm
+  from work.batches b
+  inner join &DC_LIBREF..mpe_derived d
+    on upcase(d.libref)=b.rls_libref
+    and upcase(d.dsn)=b.rls_table
+    and upcase(d.var_nm)=upcase(b.variable_nm)
+  where &dc_dttmtfmt lt d.tx_to
+    and d.active=1;
+quit;
+
+%let rls_msg=;
+data _null_;
+  set work.rls_derived(obs=1);
+  call symputx('rls_msg'
+    ,cats('MPE_ROW_LEVEL_SECURITY: ',variable_nm
+      ,' is a derived variable in ',rls_libref,'.',rls_table
+      ,' - derived variables cannot be used in a row level security rule'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.rls_derived))
+  ,mac=mpe_row_level_security_postedit
+  ,msg=%superq(rls_msg)
+)
+
 %macro quickloop();
 %do i=1 %to &cnt;
  data work.inds&i;
@@ -35,6 +35,7 @@

  <h4> SAS Macros </h4>
  @li mf_existds.sas
+  @li mf_nobs.sas
  @li mf_verifymacvars.sas
  @li dc_assignlib.sas
  @li mf_getvarformat.sas
@@ -42,6 +43,7 @@
  @li mp_cntlout.sas
  @li mp_filtercheck.sas
  @li mp_filtergenerate.sas
+  @li mpe_derived.sas
  @li mp_validatecol.sas
  @li mpe_validatecol.sas

@@ -150,10 +152,23 @@ run;
 )


+/**
+  * Derived (computed) variables - these exist only in a view, so the filter
+  * must be validated and applied against that view
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&libds));
+
 /**
  * Validate the filter query
  */
-%mp_filtercheck(work.filterquery,targetds=&libds,abort=YES)
+%mp_filtercheck(work.filterquery,targetds=&filterds,abort=YES)

 /**
  * Prepare the query
@@ -162,14 +177,14 @@ run;

 /* cannot %inc in a sql where clause, only data step, so - use a view */
 data work.vw_vals/view=work.vw_vals;
-  set &libds;
+  set &filterds;
  where %inc myfilter;;
 run;

 proc sql;
 create view work.vw_vals_sorted as
  select distinct
-      put(&col2,%mf_getVarFormat(&libds,&col2,force=1)) as formatted,
+      put(&col2,%mf_getVarFormat(work.vw_vals,&col2,force=1)) as formatted,
      &col2 as unformatted
  from work.vw_vals;

@@ -194,7 +209,7 @@ proc sql noprint;
 select count(*) into: nobs from work.vw_vals_sorted;
 data meta;
  column="&col2";
-  sasformat="%mf_getVarFormat(&libds,&col2)";
+  sasformat="%mf_getVarFormat(work.vw_vals,&col2)";
  startrow=&startrow;
  rows=&rows;
  nobs=&nobs;
@@ -4,6 +4,7 @@

  <h4> SAS Macros </h4>
  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
  @li mx_execute.sas
  @li mf_getuniquefileref.sas

@@ -36,3 +37,54 @@ run;
  desc=DCLIB found in unfiltered getgolvals response,
  test=ANYVAL
 )
+
+/**
+  * derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
+    ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
+    ,var_nm='DERIVED_KEY'
+    ,var_deps='PRIMARY_KEY_FIELD'
+    ,var_type='C'
+    ,var_len=20
+    ,var_fmt=''
+    ,var_logic='cats(primary_key_field)'
+    ,active=1;
+quit;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. COL:$32.';
+  put "&dclib..MPE_X_TEST,DERIVED_KEY";
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:iwant,
+  outlib=web4
+)
+
+data work.vals4;
+  set web4.vals;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.vals4,
+  desc=Values are returned for a derived column,
+  test=HASOBS
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where var_nm='DERIVED_KEY'
+    and upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+quit;
@@ -21,6 +21,8 @@
  @li mp_ds2csv.sas
  @li mp_streamfile.sas
  @li mp_validatecol.sas
+  @li mf_nobs.sas
+  @li mpe_derived.sas
  @li mpe_validatecol.sas
  @li mpe_filtermaster.sas

@@ -143,10 +145,32 @@ data _null_;
  putlog _infile_;
 run;

+/* derived variables exist only in a view, so a filter that references one
+  must be applied against it.  They are dropped from the download - they are
+  an EDIT screen feature. */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let derived_droplist=;
+%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&libds));
+%if &derived_nobs>0 %then %do;
+  proc sql noprint;
+  select name into: derived_droplist separated by ' '
+    from work.derived_vars;
+  quit;
+%end;
+
 options obs=200000;/* stop limit */
 data staged(drop=&txfrom &txto);
-  set &libds;
+  set &filterds;
  where %inc filtref;;
+  %if %length(&derived_droplist)>0 %then %do;
+    drop &derived_droplist;
+  %end;
 run;
 options obs=max;

@@ -26,8 +26,10 @@

  <h4> SAS Macros </h4>
  @li dc_assignlib.sas
+  @li mf_nobs.sas
  @li mp_abort.sas
  @li mp_filterstore.sas
+  @li mpe_derived.sas
  @li mpe_validatecol.sas
  @li removecolsfromwork.sas

@@ -69,8 +71,19 @@ run;

 %dc_assignlib(WRITE,%scan(&ds,1,.))

+/* derived variables exist only in a view, so validate against that */
+%mpe_derived(libds=&ds,
+  dclib=&dc_libref,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&ds));
+
 %mp_filterstore(
  libds=&ds,
+  targetds=&filterds,
  queryds=work.filterquery,
  filter_summary=&dc_libref..mpe_filteranytable,
  filter_detail=&dc_libref..mpe_filtersource,
@@ -48,6 +48,8 @@ run;

 data work.iwant;
  filter_table='DCTEST.DCFMTS-FC';
+  /* mx_execute reads the program to run from the input dataset */
+  _program="&_program";
 run;
 data work.filterquery;
 GROUP_LOGIC='AND';
@@ -77,3 +79,66 @@ run;
  desc=Test2 result filter,
  test=EQUALS 1
 )
+
+/**
+  * Test 3 - filter on a derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_deps='LIBREF DSN'
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
+    ,active=1;
+quit;
+
+data work.iwant;
+  filter_table="&dclib..MPE_TABLES";
+run;
+data work.filterquery;
+  GROUP_LOGIC='AND';
+  SUBGROUP_LOGIC='AND';
+  SUBGROUP_ID=1;
+  VARIABLE_NM='DERIVED_NOTE';
+  OPERATOR_NM='CONTAINS';
+  RAW_VALUE="'MPE_LOCK'";
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.iwant work.filterquery,
+  outlib=web3,
+  outref=web3ref
+)
+data _null_;
+  infile web3ref;
+  input;
+  putlog _infile_;
+run;
+data result;
+  set web3.result;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.result,
+  desc=Test3 derived filter,
+  test=EQUALS 1
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where var_nm='DERIVED_NOTE'
+    and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;
@@ -59,6 +59,7 @@
  @li mp_searchdata.sas
  @li mp_validatecol.sas
  @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
  @li mpe_dsmeta.sas
  @li mpe_getversions.sas
  @li mpe_filtermaster.sas
@@ -218,6 +219,25 @@ run;
    putlog _infile_;
  run;

+  /* derived variables exist only in a view, so a filter that references one
+    must be applied against it.  The columns themselves are dropped from the
+    output - they are an EDIT screen feature. */
+  %mpe_derived(libds=&libds,
+    dclib=&mpelib,
+    outref=dercode,
+    outview=work.vw_derived,
+    outvars=work.derived_vars
+  )
+  %let derived_nobs=%mf_nobs(work.derived_vars);
+  %let derived_droplist=;
+  %let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&libds));
+  %if &derived_nobs>0 %then %do;
+    proc sql noprint;
+    select name into: derived_droplist separated by ' '
+      from work.derived_vars;
+    quit;
+  %end;
+
  %if &searchtype=NONE or "%trim(&searchtype) " = " " %then %do;
    /* get row count */
    filename rows temp;
@@ -227,7 +247,7 @@ run;
      input;
      if _n_=1 then do;
        put 'proc sql;';
-        put "select count(*) into: dsobs from &libds where";
+        put "select count(*) into: dsobs from &filterds where";
      end;
      put _infile_;
      if eof then put ';';
@@ -242,8 +262,11 @@ run;

    /* send actual data, filtered and row-capped */
    data work.viewdata;
-      set &libds;
+      set &filterds;
      where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
      if _n_>&DC_MAXOBS_WEBVIEW then stop;
    run;
    %if %mf_nobs(work.viewdata)=0 %then %do;
@@ -256,8 +279,11 @@ run;
  %end;
  %else %do;
    data work.vwsearch/view=work.vwsearch;
-      set &libds;
+      set &filterds;
      where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
    run;
    %if %upcase(&searchtype)=CHAR %then %do;
      %mp_searchdata(lib=work
@@ -275,6 +301,8 @@ run;
    %end;
    %if %mf_existds(libds=MPSEARCH.vwsearch) %then %do;
      %let dsobs=%mf_nobs(MPSEARCH.vwsearch);
+      /* the derived columns were dropped from work.vwsearch above, so
+        mp_searchdata's output cannot carry them */
      data viewdata;
        set MPSEARCH.vwsearch;
        if _n_<=&DC_MAXOBS_WEBVIEW;
@@ -283,7 +311,7 @@ run;
    %else %do;
      %let dsobs=0;
      data viewdata;
-        set &libds;
+        set &filterds;
        stop;
      run;
    %end;