Merge branch 'feat/derived-vars' into fix/mock-rk-underlying-pk

Two conflicts, both resolved by keeping the intent of each side:

- sas/mocks/sasjs/services/editors/getdata.js: the PK derivation and the
  pkFirst() ordering from this branch, then the derived-variable block from the
  release branch. The derived columns are added to the schema before the
  filter, so the PK-first ordering still applies to the full column set.
- client/package-lock.json: taken from this branch. The merged package.json
  differs from the release branch only in the fast-uri override (4.2.3 here,
  3.1.8 there); the release branch already carries the same @angular 20.3.34
  and @sasjs/adapter 5.1.2 pins, so the lock is unchanged from 74060c0.

Gates on the merged tree: npm audit --omit=dev 0 vulnerabilities, prettier
clean, 603/603 unit tests, ng build, license-checker. Mock services re-probed
against a local SASjs Server: MPE_ROW_LEVEL_SECURITY publishes PKCNT 10 with
the business keys first and no RLS_RK, an edit diffs as UPDATED 1 with the
base RK, and the derived table TESTDATA.DERIVED_DEMO still serves its computed
columns.

.gitea/workflows/build.yaml

@@ -198,7 +198,7 @@ jobs:
           # Start frontend and run cypress
           # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

client/cypress/e2e/derived-variables.cy.ts

@@ -0,0 +1,162 @@
+export {}
+//
+// Derived (computed) variables are configured per table in MPE_DERIVED and
+// created at read time, ahead of the filter, so that they can be filtered on.
+//
+// TESTDATA.DERIVED_DEMO carries two of them, neither of which is a physical
+// column of the table:
+//   REGION      ifc(STATE in ('CA','OR','WA'),'West','East')
+//   LINE_VALUE  QTY * PRICE
+//
+// The mock implements the derivation in getdata.js / getcolvals.js (see
+// mpeDerived in dcMockUtils.js), so the EDIT screen can be exercised without a
+// SAS server. The real evaluation is done by SAS.
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('derived variables: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.viewport(1440, 900)
+    bootApp()
+  })
+
+  it('1 | derived columns are shown in the EDIT grid with computed values', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    getCellByHeaderAndRow(0, 'REGION').should('have.text', 'West')
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'East')
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '21')
+    getCellByHeaderAndRow(1, 'LINE_VALUE').should('have.text', '99')
+    // REGION_LABEL is built on REGION, so it only resolves if the derived
+    // statements run in VARIABLE_ORDER sequence
+    getCellByHeaderAndRow(0, 'REGION_LABEL').should('have.text', 'West-CA')
+    getCellByHeaderAndRow(1, 'REGION_LABEL').should('have.text', 'East-NY')
+
+    cy.screenshot('derived-1-grid', { capture: 'viewport' })
+  })
+
+  it('2 | filters on a character derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('REGION')
+    setFilterValue('West')
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', "REGION = 'West'")
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 2)
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'West')
+
+    cy.screenshot('derived-2-filter-region', { capture: 'viewport' })
+  })
+
+  it('3 | filters on a numeric derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('LINE_VALUE')
+    setFilterOperator('>')
+    setFilterValue('50')
+    submitFilter()
+
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 1)
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '99')
+
+    cy.screenshot('derived-3-filter-value', { capture: 'viewport' })
+  })
+})
+
+/**
+ * Cypress clears cookies between tests, which drops the SASjs Server session
+ * and lands the app back on the evaluation agreement card. Accept it when it is
+ * shown - a no-op when the session survived.
+ */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: longerCommandTimeout })
+  cy.get('body').then(($body: any) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Opens a table straight in the EDIT screen and waits for the grid. */
+const openEditor = (libMem: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/editor/${libMem}`, {
+    timeout: longerCommandTimeout
+  })
+  cy.get('.ht_clone_top .htCore thead tr th', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+  cy.wait(2500)
+}
+
+const openFilterPopup = () => {
+  cy.contains('button.btnView', 'Filter', {
+    timeout: longerCommandTimeout
+  }).click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/**
+ * The soft-select inputs drop a transparent .overlay click-catcher over the
+ * modal while their suggestion list is open, so they need force.
+ */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterOperator = (operator: string) => {
+  cy.get('.filter-modal .operator-col select').first().select(operator)
+  cy.wait(500)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+const getCellByHeaderAndRow = (rowIndex: number, headerText: string) => {
+  return cy
+    .get('.ht_clone_top .htCore thead tr th')
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => (th as HTMLElement).innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => (th as HTMLElement).innerText.trim() === headerText
+      )
+
+      return cy
+        .get('.ht_master tbody tr')
+        .then((rows: any) => rows[rowIndex].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}

client/cypress/e2e/excel.cy.ts

@@ -399,6 +399,26 @@ context('excel tests: ', function () {
     })
   })
 
+  it('25 | Keeps decimals when the upload preview becomes a manual edit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    // Entering manual edit mode discards the source file but keeps the values
+    // parsed from it. SOME_NUM holds 0.901045951946194 in the first data row
+    // of the fixture, so a truncated or zeroed cell fails the assertion.
+    attachExcelFile('regular_excel.xlsx', () => {
+      getFirstRowCells().then((previewCells) => {
+        expect(previewCells).to.include(decimalSomeNum)
+
+        cy.get('.btn-upload-preview.btn-primary').click()
+        cy.get('.modal-footer .btn-success-outline').click()
+
+        getFirstRowCells().then((editCells) => {
+          expect(editCells).to.include(decimalSomeNum)
+        })
+      })
+    })
+  })
+
   // Large files break Cypress
 
   // it ('? | Uploads Excel with size of 5MB', (done) => {
@@ -476,6 +496,19 @@ const attachExcelFile = (excelFilename: string, callback?: any) => {
     })
 }
 
+const decimalSomeNum = '0.901045951946194'
+
+// The row's cell texts, without depending on column positions: the preview
+// grid renders row-status and delete cells in an order that varies.
+const getFirstRowCells = () =>
+  cy
+    .get('#hotTable', { timeout: longerCommandTimeout })
+    .find('div.ht_master.handsontable tbody tr')
+    .eq(0)
+    .then(($row) =>
+      Array.from($row[0].children).map((cell: any) => cell.innerText.trim())
+    )
+
 const submitExcel = (callback?: any) => {
   cy.get('.buttonBar button.preview-submit', { timeout: longerCommandTimeout })
     .click()

client/cypress/e2e/row-cell-limits.cy.ts

@@ -1,13 +1,15 @@
-// The EDIT screen refuses a selection that is too large - DC_MAXOBS_WEBEDIT
-// rows, or DC_MAXCELLS_WEBEDIT cells (rows x columns), whichever is reached
-// first - and the VIEW screen caps at DC_MAXOBS_WEBVIEW rows.
+// The EDIT screen is bounded by DC_MAXOBS_WEBEDIT rows and DC_MAXCELLS_WEBEDIT
+// cells (rows x columns), whichever is reached first, and the VIEW screen caps
+// at DC_MAXOBS_WEBVIEW rows.
 //
+// A selection over an EDIT limit is served TRUNCATED - the grid carries the
+// first rows that fit, and a warning triangle against the table name says why.
 // This spec pins those limits against the mock, which mirrors the shipped
 // defaults (250 rows / 200000 cells for EDIT, 2000 rows for VIEW).  It uses
 // two existing fixtures rather than new ones:
 //   MPE_X_SEARCH - 1000 rows x 9 cols, so it trips the ROW limit
-//   MPE_X_WIDE   - 200 rows x 1001 cols (200,400 cells), under the row limit
-//                  but over the CELL limit
+//   MPE_X_WIDE   - 200 rows x 1001 cols (200,200 cells), under the row limit
+//                  but over the CELL limit, so the row cap is 199
 export {}
 
 const hostUrl = Cypress.env('hosturl')
@@ -60,27 +62,51 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
     })
   })
 
-  it('2 | EDIT refuses a table over the row limit', () => {
+  it('2 | EDIT serves a table over the row limit, truncated, with a warning', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
     openEdit()
 
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 1000 rows x 9 columns is over the 250 row limit, so the grid carries the
+    // first 250 rows and the header says how many it is showing.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
+      '250 rows'
     )
+    cy.get('#hotTable .ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length.at.least', 1)
+
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXOBS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
   })
 
-  it('3 | EDIT refuses a table over the cell limit', () => {
+  it('3 | EDIT serves a table over the cell limit, truncated, with a warning', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_wide')
     openEdit()
 
-    // 200 rows x 1002 columns (the 1001 stored columns plus the housekeeping
-    // delete column) = 200,400 cells, over the 200,000 cell limit, while the
-    // row count is under the 250 row limit.
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 200 rows x 1001 stored columns = 200,200 cells, over the 200,000 cell
+    // limit, while the row count is under the 250 row limit - so the cell
+    // limit sets the cap: floor(200000 / 1001) = 199 rows.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      'Selection is too wide (200 rows x 1002 cols) - please filter and try again!'
+      '199 rows'
     )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
   })
 
   it('4 | the EDIT limits follow MPE_CONFIG, not the shipped defaults', () => {
@@ -90,20 +116,29 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
     // all.  And MPE_CONFIG lives in the control library, so reading it from the
     // table's own library would fail for this table in particular.
     //
-    // The cell limit is the one to move: 7 rows x 18 columns is 126 cells, so
-    // at 110 the TESTDATA.MPE_X_NEW edit must be refused, where a hard-coded
-    // 200000 - or a config read that fails and falls back to it - lets it
-    // through.  110 is still above MPE_CONFIG's own 17 x 6 = 102, so the option
-    // can be put back afterwards.
+    // The cell limit is the one to move: 7 rows x 17 columns is 119 cells, so
+    // at 110 the TESTDATA.MPE_X_NEW edit must be truncated to
+    // floor(110 / 17) = 6 rows, where a hard-coded 200000 - or a config read
+    // that fails and falls back to it - lets it through.  110 is still above
+    // MPE_CONFIG's own 6 x 17 = 102, so the option can be put back afterwards.
     setConfigValue('DC_MAXCELLS_WEBEDIT', '110', () => {
       visitPage('view/data/TESTDATA.MPE_X_NEW')
       closeAbortModalIfPresent()
       openEdit()
 
-      cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
         'contain.text',
-        'Selection is too wide (7 rows x 18 cols) - please filter and try again!'
+        '6 rows'
       )
+      cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+        .should('be.visible')
+        .realHover()
+      cy.get('.editor-title clr-tooltip-content', {
+        timeout: longerCommandTimeout
+      })
+        .should('be.visible')
+        .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+        .should(($el) => expect($el.css('opacity')).to.eq('1'))
     })
 
     // Put the option back, so the rest of the estate sees the shipped default.
@@ -112,28 +147,31 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
       closeAbortModalIfPresent()
       openEdit()
 
-      // 126 cells under a 200000 cell limit: the editor loads, no abort.
-      cy.get('#hotTable .ht_master .htCore tbody tr', {
-        timeout: longerCommandTimeout
-      }).should('exist')
+      // 119 cells under a 200000 cell limit: all 7 rows load, no warning.
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+        'contain.text',
+        '7 rows'
+      )
+      cy.get('.truncation-warning').should('not.exist')
     })
   })
 
-  it('5 | the filter picker still lists the variables after the abort', () => {
+  it('5 | the warning says to filter, and filtering narrows the grid', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
     openEdit()
 
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
-      'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
-    )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'Filter the table to work with the rest')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
 
-    closeAbortModal()
-
-    // The abort returns no table, and the picker's variable list normally comes
-    // from the table's own columns - so without the public/getcols fallback the
-    // "please filter and try again" advice has nothing to pick from.  Every
-    // column of MPE_X_SEARCH must be on offer.
+    // The advice has to be actionable: the picker must offer the table's
+    // columns and the filter must reload the grid.
     openFilterPopup()
 
     filterVariables().should(($options) => {
@@ -155,20 +193,20 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
       ])
     })
 
-    // and the suggestion list is usable, not just populated
     setFilterVariable('SITE_NAME')
-
-    // Submitting is what exercises TYPE.  The picker keys its operator set and
-    // its value quoting on it, and the fallback list never passes through
-    // mergeColsRules, so without TYPE on this payload a character value would be
-    // submitted unquoted and the backend's %mp_filtercheck would reject it.
-    setFilterValue('Bristol')
+    // 100 of the 1000 rows: under the limit, so nothing is truncated.
+    setFilterValue('Nereid Trench')
     submitFilter()
 
     cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      "SITE_NAME = 'Bristol'"
+      "SITE_NAME = 'Nereid Trench'"
     )
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      '100 rows'
+    )
+    cy.get('.truncation-warning').should('not.exist')
   })
 })
 
@@ -185,13 +223,6 @@ const openEdit = () => {
   }).click()
 }
 
-/** The abort modal offers Close, and the filter button sits behind it. */
-const closeAbortModal = () => {
-  cy.get('app-info-modal', { timeout: longerCommandTimeout })
-    .contains('button', 'Close')
-    .click()
-}
-
 /**
  * The EDIT screen's Filter button opens the same .filter-modal the viewer uses.
  * It only renders once the editor is in its read-only (non-editing) state,

client/cypress/e2e/startupservice-launcher.cy.ts

@@ -0,0 +1,81 @@
+// Marks this file as an ES module so its top-level consts don't collide with
+// other spec files under the TS type-checker.
+export {}
+//
+// When the startup service fails without a usable response, the error modal
+// offers a button that runs the same service in a new browser window, where the
+// SAS log is returned instead. This spec forces a startup failure against the
+// mock backend and checks the button and the URL it opens.
+const hostUrl = Cypress.env('hosturl')
+
+const STARTUP_PROGRAM = '/Public/app/dc/services/public/startupservice'
+
+/** The body a Viya Compute task returns when the run outlives the
+ * execution-task timeout: a bare string, so the adapter has no webout and no
+ * log to report. */
+const rawJobError = [
+  'Job error',
+  'The Compute service could not execute the task c74a707a-8b88-46a5-8f28-9a7694ad5e13 because the context is not reusable.',
+  'path: /compute/tasks',
+  'correlator: dc1a9fda-8f65-4f32-b4d4-2a70e02179b1;1bcb6a4c-8a16-4053-a790-292544c25a03'
+].join('\n')
+
+context('startupservice launcher: ', function () {
+  it('1 | runs the startupservice in a new window when its response is malformed', () => {
+    // Report an empty admin folder so the app loads the startup service: on a
+    // freshly seeded mock estate makedata is still present, which sends the app
+    // to the deploy screen instead.
+    cy.intercept('GET', '**/drive/folder**', (req) => {
+      // Guarded parse: a lone '%' in the URL would make decodeURIComponent
+      // throw, and an exception here aborts the request.
+      let url = req.url
+      try {
+        url = decodeURIComponent(req.url)
+      } catch (e) {
+        url = req.url
+      }
+
+      if (url.includes('services/admin')) {
+        req.reply({ files: [], folders: [] })
+        return
+      }
+      req.continue()
+    })
+
+    cy.intercept('**/stp/execute/**', (req) => {
+      if (req.url.includes('public/startupservice')) {
+        // The adapter cannot use a bare string as a response payload, so the
+        // startup request fails with nothing to show.
+        req.reply({
+          statusCode: 200,
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify(rawJobError)
+        })
+        return
+      }
+      req.continue()
+    })
+
+    // The launcher must not actually open a tab under test.
+    cy.visit(hostUrl, {
+      timeout: 60000,
+      onBeforeLoad(win) {
+        cy.stub(win, 'open').as('windowOpen')
+      }
+    })
+
+    cy.get('.clr-abort-modal', { timeout: 60000 }).should('be.visible')
+    cy.get('.clr-abort-modal').last().should('contain.text', 'startupservice')
+    cy.get('.clr-abort-modal').last().contains('button', 'Show log').click()
+
+    cy.get('@windowOpen').should('have.been.calledOnce')
+    cy.get('@windowOpen').then((stub: any) => {
+      const url = stub.firstCall.args[0]
+
+      expect(url).to.contain(STARTUP_PROGRAM)
+      expect(url).to.contain('_debug=131')
+      // The browser route is what makes the log available: no execution task.
+      expect(url).not.to.contain('_executionTasks')
+    })
+  })
+})

client/src/app/editor/editor.component.html

@@ -245,6 +245,25 @@
               }
             </clr-tooltip>
 
+            @if (truncationNote) {
+              <clr-tooltip>
+                <clr-icon
+                  clrTooltipTrigger
+                  shape="warning"
+                  aria-label="The selection was truncated"
+                  class="color-orange truncation-warning"
+                  size="20"
+                ></clr-icon>
+                <clr-tooltip-content
+                  clrPosition="right"
+                  clrSize="lg"
+                  *clrIfOpen
+                >
+                  {{ truncationNote }}
+                </clr-tooltip-content>
+              </clr-tooltip>
+            }
+
             @if (dataSource) {
               @if (!zeroFilterRows) {
                 ({{ dataSource.length | thousandSeparator: ',' }}

client/src/app/editor/editor.component.ts

@@ -160,6 +160,9 @@ export class EditorComponent
   dsmeta: DSMeta[] = []
   versions: Version[] = []
   dsNote = ''
+  /** Set when getdata served a truncated selection - shown as a warning
+   * against the table name, with the reason in a tooltip. */
+  truncationNote = ''
 
   viewboxes = false
 
@@ -1360,8 +1363,14 @@ export class EditorComponent
             colSpecs.TYPE === 'num' &&
             !colSpecs.DDTYPE.includes('TIME') &&
             !colSpecs.DDTYPE.includes('DATE')
-          )
-            row[colKey] = parseInt(row[colKey])
+          ) {
+            const numericValue = parseFloat(row[colKey])
+
+            // A cell the file parser could not read as a number stays as it
+            // was: the editor's numeric validation then flags it as text,
+            // where a NaN would render as an empty cell.
+            if (!Number.isNaN(numericValue)) row[colKey] = numericValue
+          }
         }
       }
     }
@@ -3930,6 +3939,11 @@ export class EditorComponent
       this.dsNote = ''
     }
 
+    // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
+    // truncated to the first rows within the limit, and carries the reason
+    // here.  Empty means the whole selection is in the grid.
+    this.truncationNote = response.data.sasparams[0].TRUNC_MSG || ''
+
     const hot = this.hotInstance
 
     const approvers: Approver[] = response.data.approvers

client/src/app/models/TableData.ts

@@ -47,6 +47,10 @@ export interface SASParam {
   PKCNT: number
   RK_FLAG: number
   TMVARS: string
+  /** 1 when getdata cut the selection down to the row / cell limit */
+  TRUNCATED?: number
+  /** Reason shown in the tooltip when TRUNCATED is 1, else empty */
+  TRUNC_MSG?: string
 }
 
 export interface ExcelRule {

client/src/app/services/app.service.spec.ts

@@ -170,9 +170,11 @@ describe('AppService - startup retry', () => {
 
     expect(deps.sasService.request).toHaveBeenCalledTimes(1)
     expect(deps.eventService.showInfoModal).toHaveBeenCalledTimes(1)
-    const [, message] = deps.eventService.showInfoModal.calls.mostRecent().args
+    const [, message, sasService] =
+      deps.eventService.showInfoModal.calls.mostRecent().args
     expect(message).toContain(rawJobError)
     expect(message).not.toContain('Globvars, Sasdatasets, Saslibs, XLMaps')
+    expect(sasService).toBe('public/startupservice')
     expect(deps.licenceService.isAppActivated.value).toBeFalse()
   })
 })

client/src/app/services/app.service.ts

@@ -107,7 +107,11 @@ export class AppService {
             malformedMessage
           )
           this.startupCheckService.updateStep(2, 'error', 'Malformed response')
-          this.eventService.showInfoModal('Error', malformedMessage)
+          this.eventService.showInfoModal(
+            'Error',
+            malformedMessage,
+            'public/startupservice'
+          )
           this.licenceService.isAppActivated.next(false)
 
           return
@@ -139,7 +143,8 @@ export class AppService {
           )
           this.eventService.showInfoModal(
             'Error',
-            `${missingProps.join(', ')} are not present in the startupservice`
+            `${missingProps.join(', ')} are not present in the startupservice`,
+            'public/startupservice'
           )
           this.licenceService.isAppActivated.next(false)
 
@@ -221,7 +226,8 @@ export class AppService {
         this.startupCheckService.updateStep(2, 'error', 'Request failed')
         this.eventService.showInfoModal(
           'Error',
-          'There is an issue with startupservice response'
+          'There is an issue with startupservice response',
+          'public/startupservice'
         )
         this.licenceService.isAppActivated.next(false)
       })

client/src/app/services/event.service.ts

@@ -42,11 +42,15 @@ export class EventService {
     this.onDemoLimitModalShow.emit(featureName)
   }
 
-  public showInfoModal(modalTitle: string = 'Info', message: string) {
+  public showInfoModal(
+    modalTitle: string = 'Info',
+    message: string,
+    sasService: string | null = null
+  ) {
     let infoModal: InfoModal = {
       modalTitle,
       message,
-      sasService: '',
+      sasService,
       details: null
     }
 

client/src/app/services/licence.service.ts

@@ -142,7 +142,8 @@ export class LicenceService {
       this.isAppActivated.next(false)
       this.eventService.showInfoModal(
         'Error',
-        'Some of the globvars are not present in the startupservice'
+        'Some of the globvars are not present in the startupservice',
+        'public/startupservice'
       )
       // this.router.navigateByUrl('/licensing/key?error=missing')
       return

client/src/app/services/sas.service.spec.ts

@@ -14,7 +14,12 @@ const buildService = () => {
   const sasjsAdapter: any = {
     request: jasmine.createSpy('request').and.resolveTo({}),
     getSasRequests: () => [],
-    setDebugState: jasmine.createSpy('setDebugState')
+    setDebugState: jasmine.createSpy('setDebugState'),
+    getSasjsConfig: () => ({
+      serverUrl: 'https://sas.example.com',
+      appLoc: '/Public/app/dc',
+      serverType: 'SASVIYA'
+    })
   }
   const loggerService: any = {
     logRequestData: jasmine.createSpy('logRequestData')
@@ -163,3 +168,38 @@ describe('SasService debug state', () => {
     expect(initService(true).getSasjsConfig().debug).toBe(false)
   })
 })
+
+/**
+ * The URL shape the deploy configurator already uses for makedata: the service
+ * is run by the JES / STP web app in a browser tab, so the SAS log comes back
+ * as the page body instead of through the adapter.
+ */
+describe('SasService openServiceInNewWindow', () => {
+  it('opens the service URL built from the live adapter config', () => {
+    const { service } = buildService()
+    spyOn(service, 'getExecutionPath').and.returnValue('/SASJobExecution')
+    const openSpy = spyOn(window, 'open')
+
+    service.openServiceInNewWindow('public/startupservice', {
+      _contextname: 'ctx'
+    })
+
+    expect(openSpy).toHaveBeenCalledWith(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_contextname=ctx&_debug=131',
+      '_blank'
+    )
+  })
+
+  it('opens without parameters when none are given', () => {
+    const { service } = buildService()
+    spyOn(service, 'getExecutionPath').and.returnValue('/SASJobExecution')
+    const openSpy = spyOn(window, 'open')
+
+    service.openServiceInNewWindow('public/startupservice')
+
+    expect(openSpy).toHaveBeenCalledWith(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131',
+      '_blank'
+    )
+  })
+})

client/src/app/services/sas.service.ts

@@ -21,6 +21,7 @@ import { SasViyaService } from './sas-viya.service'
 import { ViyaApiFolder } from '../viya-api-explorer/models/viya-api-folder.model'
 import { ViyaApiFolderMembers } from '../viya-api-explorer/models/viya-api-folder-content.model'
 import { parseUserAgent } from '../shared/utils/parse-user-agent'
+import { buildServiceUrl } from '../shared/utils/service-url'
 import { VERSION } from '../../environments/version'
 
 /**
@@ -842,6 +843,33 @@ export class SasService {
     return this.sasjsAdapter
   }
 
+  /**
+   * Runs a SAS service in a new browser window, bypassing the adapter.
+   *
+   * The adapter's request path can hide the backend log when a service fails:
+   * on Viya a long startup service may be submitted as an execution task and
+   * time out before it finishes, returning neither webout nor log. A browser
+   * request to the JES / STP web app runs the same `_program` with no task and
+   * returns the SAS log (`_debug=131`), so this is the diagnostic path offered
+   * when a failed request leaves nothing usable to display.
+   */
+  public openServiceInNewWindow(
+    service: string,
+    params: { [name: string]: string | number | null | undefined } = {}
+  ) {
+    const config = this.getSasjsConfig()
+
+    const url = buildServiceUrl({
+      serverUrl: config.serverUrl || '',
+      appLoc: config.appLoc || '',
+      executionPath: this.getExecutionPath() || '',
+      service,
+      params
+    })
+
+    window.open(url, '_blank')
+  }
+
   public getServerType(): string {
     const sasjsConfig = this.sasjsAdapter.getSasjsConfig()
 

client/src/app/shared/abort-modal/info-modal.component.html

@@ -28,6 +28,16 @@
     }
   </div>
   <div class="modal-footer">
+    @if (showServiceLauncher(data.sasService)) {
+      <button
+        type="button"
+        class="btn btn-primary"
+        (click)="openServiceInNewWindow()"
+      >
+        Show log
+      </button>
+    }
+
     @if (showConfiguratorButton(data.sasService)) {
       <button
         type="button"

client/src/app/shared/abort-modal/info-modal.component.spec.ts

@@ -0,0 +1,71 @@
+import { InfoModalComponent } from './info-modal.component'
+
+describe('InfoModalComponent', () => {
+  let component: InfoModalComponent
+  let sasService: any
+
+  const buildComponent = (sasServiceConfig: any = {}) => {
+    sasService = {
+      getSasjsConfig: () => sasServiceConfig,
+      openServiceInNewWindow: jasmine.createSpy('openServiceInNewWindow')
+    }
+
+    component = new InfoModalComponent(
+      {} as any,
+      {} as any,
+      sasService,
+      {} as any
+    )
+    component.data = {
+      modalTitle: 'Error',
+      message: 'startupservice failed',
+      sasService: 'public/startupservice',
+      details: null
+    }
+    component.ngOnInit()
+  }
+
+  describe('showServiceLauncher', () => {
+    beforeEach(() => buildComponent())
+
+    it('is shown for the startupservice, on every server type', () => {
+      expect(component.showServiceLauncher('public/startupservice')).toBeTrue()
+      expect(
+        component.showServiceLauncher('services/public/startupservice')
+      ).toBeTrue()
+    })
+
+    it('is not shown for other services or a missing service', () => {
+      expect(component.showServiceLauncher('public/getdata')).toBeFalse()
+      expect(component.showServiceLauncher('')).toBeFalse()
+      expect(component.showServiceLauncher(null)).toBeFalse()
+    })
+  })
+
+  describe('openServiceInNewWindow', () => {
+    it('runs the startupservice in a new window', () => {
+      buildComponent({ serverType: 'SASVIYA' })
+
+      component.openServiceInNewWindow()
+
+      expect(sasService.openServiceInNewWindow).toHaveBeenCalledWith(
+        'public/startupservice',
+        {}
+      )
+    })
+
+    it('passes the configured Viya context so the log comes from it', () => {
+      buildComponent({
+        serverType: 'SASVIYA',
+        contextName: 'SAS Job Execution compute context'
+      })
+
+      component.openServiceInNewWindow()
+
+      expect(sasService.openServiceInNewWindow).toHaveBeenCalledWith(
+        'public/startupservice',
+        { _contextname: 'SAS Job Execution compute context' }
+      )
+    })
+  })
+})

client/src/app/shared/abort-modal/info-modal.component.ts

@@ -71,6 +71,34 @@ export class InfoModalComponent implements OnInit {
     this.onConfirmModalClick.emit()
   }
 
+  /**
+   * Whether to show the `Show log` button.
+   *
+   * The startupservice failure modal is a dead end otherwise: the adapter
+   * returned no log, so the only way to see the backend reason is to run the
+   * service through the JES / STP web app in a browser tab. Offered on every
+   * server type - SAS 9, Viya and SASjs Server all accept the `_program` URL.
+   */
+  showServiceLauncher(sasService: string | null) {
+    return !!sasService?.includes('startupservice')
+  }
+
+  /**
+   * Runs the startupservice in a new browser window so the SAS log is
+   * available even when the adapter response carried none (e.g. a Viya
+   * execution task that timed out).
+   */
+  openServiceInNewWindow() {
+    const sasjsConfig = this.sasService.getSasjsConfig()
+    const params: { [name: string]: string } = {}
+
+    if (sasjsConfig.contextName) {
+      params['_contextname'] = sasjsConfig.contextName
+    }
+
+    this.sasService.openServiceInNewWindow(this.data.sasService || '', params)
+  }
+
   /**
    * Only on SAS9, opening a backend configurator/deploy page
    */

client/src/app/shared/utils/service-url.spec.ts

@@ -0,0 +1,86 @@
+import { buildServiceUrl } from './service-url'
+
+describe('buildServiceUrl', () => {
+  const viyaConfig = {
+    serverUrl: 'https://sas.example.com',
+    appLoc: '/Public/app/dc',
+    executionPath: '/SASJobExecution',
+    service: 'public/startupservice'
+  }
+
+  it('builds the Viya JES URL for a service, with the log enabled', () => {
+    expect(buildServiceUrl(viyaConfig)).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('leaves serverUrl empty on SAS 9 so the browser resolves the origin', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        serverUrl: '',
+        executionPath: '/SASStoredProcess/do'
+      })
+    ).toBe(
+      '/SASStoredProcess/do/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('builds the SASjs Server URL from its execution path', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        executionPath: '/SASjsApi/stp/execute'
+      })
+    ).toBe(
+      'https://sas.example.com/SASjsApi/stp/execute/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('appends encoded parameters before the debug flag', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        params: { _contextname: 'SAS Job Execution compute context' }
+      })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_contextname=SAS%20Job%20Execution%20compute%20context&_debug=131'
+    )
+  })
+
+  it('skips parameters with no value', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        params: { _contextname: undefined, admin: null, dcpath: 'sas' }
+      })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&dcpath=sas&_debug=131'
+    )
+  })
+
+  it('omits the debug flag when debug is off', () => {
+    expect(buildServiceUrl({ ...viyaConfig, debug: false })).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice'
+    )
+  })
+
+  it('accepts a service with a leading slash', () => {
+    expect(
+      buildServiceUrl({ ...viyaConfig, service: '/public/startupservice' })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+
+  it('accepts the services/ prefix the request wrapper reports', () => {
+    expect(
+      buildServiceUrl({
+        ...viyaConfig,
+        service: 'services/public/startupservice'
+      })
+    ).toBe(
+      'https://sas.example.com/SASJobExecution/?_program=/Public/app/dc/services/public/startupservice&_debug=131'
+    )
+  })
+})

client/src/app/shared/utils/service-url.ts

@@ -0,0 +1,46 @@
+/**
+ * Builds the URL that runs a SAS service directly in a browser window.
+ *
+ * The adapter submits requests from JavaScript, and on Viya that can route the
+ * run through the Compute service (`_executionTasks=true`), where the SAS log
+ * is not returned to the caller and the request can time out before the
+ * service finishes. A browser request to the JES / STP web app runs the same
+ * `_program` without any task involved, and `_debug=131` returns the SAS log
+ * as the page body - which is what makes it useful when a service fails.
+ *
+ * `serverUrl` is left empty on SAS 9 when the app is served from the SAS
+ * server itself; the browser then resolves the path against the current
+ * origin.
+ */
+export interface ServiceUrlOptions {
+  serverUrl: string
+  appLoc: string
+  executionPath: string
+  service: string
+  params?: { [name: string]: string | number | null | undefined }
+  debug?: boolean
+}
+
+export const buildServiceUrl = ({
+  serverUrl,
+  appLoc,
+  executionPath,
+  service,
+  params = {},
+  debug = true
+}: ServiceUrlOptions): string => {
+  // Callers pass the service as the app knows it: `public/getdata` in the
+  // request path, but the request wrapper reports `services/public/getdata`
+  // on a failed request. Both resolve to the same program.
+  const servicePath = service.replace(/^\/+/, '').replace(/^services\//, '')
+  const paramString = Object.entries(params)
+    .filter(([, value]) => value !== null && value !== undefined)
+    .map(
+      ([name, value]) =>
+        `&${encodeURIComponent(name)}=${encodeURIComponent(String(value))}`
+    )
+    .join('')
+  const debugParam = debug ? '&_debug=131' : ''
+
+  return `${serverUrl}${executionPath}/?_program=${appLoc}/services/${servicePath}${paramString}${debugParam}`
+}

sas/mocks/sasjs/services/admin/makedata.js

@@ -281,6 +281,53 @@ const tables = {
       DD_SENSITIVITY: 'Low'
     }
   ],
+  // Derived (computed) variables.  Values are display-time only - the logic is
+  // applied by mpe_derived in the mock's getdata/getcolvals, and must use the
+  // expression subset that compileSasExpression supports (the real evaluation
+  // is done by SAS).
+  MPE_DERIVED: [
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'REGION',
+      var_deps: 'STATE',
+      var_type: 'C',
+      var_len: 8,
+      var_fmt: '',
+      var_logic: "ifc(STATE in ('CA','OR','WA'),'West','East')",
+      active: 1
+    },
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'LINE_VALUE',
+      var_deps: 'QTY PRICE',
+      var_type: 'N',
+      var_len: 8,
+      var_fmt: 'dollar12.2',
+      var_logic: 'QTY * PRICE',
+      active: 1
+    },
+    {
+      // built on REGION, so it proves the dependency order is applied
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'REGION_LABEL',
+      var_deps: 'REGION STATE',
+      var_type: 'C',
+      var_len: 14,
+      var_fmt: '',
+      var_label: 'Region and state',
+      var_logic: "cats(REGION,'-',STATE)",
+      active: 1
+    }
+  ],
   MPE_DATALOADS: [],
   MPE_EMAILS: [],
   MPE_EXCEL_CONFIG: [],
@@ -2226,7 +2273,7 @@ tables.MPE_TABLES.push({
 
 // ─── MPE_X_WIDE: wide table for the EDIT cell limit ─────────────────────────
 // 200 rows x 1001 columns (ROW_ID + 500 single-character + 500 numeric) is
-// 200,400 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
+// 200,200 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
 // DC_MAXOBS_WEBEDIT (250), so it exercises the CELL limit on its own and the
 // row limit spec can use MPE_X_SEARCH (1000 rows) instead.  Values are one
 // character / one digit on purpose: the point is width, and small values keep
@@ -3262,6 +3309,20 @@ const schema = {
       notnull: false
     }
   ],
+  MPE_DERIVED: [
+    { name: 'TX_FROM', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true },
+    { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
+    { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_DEPS', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
+    { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_LABEL', type: 'C', length: 256, format: '', label: '', notnull: false },
+    { name: 'VAR_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'ACTIVE', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'TX_TO', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true }
+  ],
   MPE_DATALOADS: [
     {
       name: 'LIBREF',
@@ -5349,7 +5410,15 @@ const testTables = {
     { ORDER_ID: 1003, CUSTOMER: 'Cobalt GmbH', REGION: 'EU', AMOUNT: 620, STATUS: 'CLOSED' },
     { ORDER_ID: 1004, CUSTOMER: 'Delta Systems', REGION: 'US', AMOUNT: 4500, STATUS: 'NEW' }
   ],
-  DEMO_MIRROR: []
+  DEMO_MIRROR: [],
+  // Derived variable demo - REGION and LINE_VALUE are not columns of this
+  // table, they are computed at read time from MPE_DERIVED rows.
+  DERIVED_DEMO: [
+    { ID: 1, STATE: 'CA', QTY: 2, PRICE: 10.5 },
+    { ID: 2, STATE: 'NY', QTY: 1, PRICE: 99 },
+    { ID: 3, STATE: 'OR', QTY: 5, PRICE: 3 },
+    { ID: 4, STATE: 'TX', QTY: 3, PRICE: 7.25 }
+  ]
 }
 
 // Schemas for the TESTDATA tables, matching the old inline getdata.js payload
@@ -5691,6 +5760,12 @@ const testSchema = {
       label: 'Last reviewed',
       notnull: false
     }
+  ],
+  DERIVED_DEMO: [
+    { name: 'ID', type: 'N', length: 8, format: '', label: 'ID', notnull: true },
+    { name: 'STATE', type: 'C', length: 2, format: '', label: 'State', notnull: false },
+    { name: 'QTY', type: 'N', length: 8, format: '', label: 'Quantity', notnull: false },
+    { name: 'PRICE', type: 'N', length: 8, format: 'dollar12.2', label: 'Price', notnull: false }
   ]
 }
 
@@ -5739,6 +5814,16 @@ tables.MPE_TABLES.push(
     buskey: 'ID',
     notes: 'Compact rule demo table (eight columns)'
   },
+  {
+    tx_from: 0,
+    tx_to: 127490111999,
+    libref: testDataLibref,
+    dsn: 'DERIVED_DEMO',
+    num_of_approvals_required: 1,
+    loadtype: 'UPDATE',
+    buskey: 'ID',
+    notes: 'Derived variable demo table'
+  },
   {
     tx_from: 0,
     tx_to: 127490111999,

sas/mocks/sasjs/services/dcMockUtils.js

@@ -1208,6 +1208,289 @@ function mpFilterStore(opts) {
     }
 }
 
+// ─── mpe_derived.sas ────────────────────────────────────────────────────────
+
+/**
+ * Compiles a SAS expression into a JS function - for the mock only.
+ *
+ * The real evaluation is done by SAS. This covers the subset used by the mock
+ * and demo configs, so the EDIT screen can be exercised (display, filtering,
+ * submission) without a SAS server:
+ *
+ *   - column names, numbers, quoted strings
+ *   - + - * / and comparisons  (= ne < > <= >=)
+ *   - AND OR, and `X IN (a, b, c)`
+ *   - ifc(), cats(), catx(), upcase(), lowcase(), strip(), trim(), length(),
+ *     substr(), index()
+ *
+ * Anything outside that subset throws when a row is derived, and the column is
+ * left missing for that row - the same way a failing SAS expression would.
+ */
+function compileSasExpression(expr, columnNames) {
+    // Tokenise first, so operators are only rewritten outside quoted strings
+    // (a literal such as 'OR' must never be read as an operator).
+    const tokens = []
+    const s = String(expr)
+    let i = 0
+    while (i < s.length) {
+        const c = s[i]
+        if (/\s/.test(c)) { i++; continue }
+        if (c === "'" || c === '"') {
+            const q = c
+            let j = i + 1
+            let val = ''
+            while (j < s.length) {
+                if (s[j] === q) {
+                    if (s[j + 1] === q) { val += q; j += 2; continue }
+                    break
+                }
+                val += s[j]
+                j++
+            }
+            tokens.push({ type: 'str', value: val })
+            i = j + 1
+            continue
+        }
+        if (/[0-9]/.test(c)) {
+            let j = i
+            while (j < s.length && /[0-9.]/.test(s[j])) j++
+            tokens.push({ type: 'num', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        if (/[A-Za-z_]/.test(c)) {
+            let j = i
+            while (j < s.length && /[A-Za-z0-9_]/.test(s[j])) j++
+            tokens.push({ type: 'word', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        const two = s.slice(i, i + 2)
+        if (['<=', '>=', '!=', '<>'].includes(two)) {
+            tokens.push({ type: 'punct', value: two })
+            i += 2
+            continue
+        }
+        // only the operators in the documented subset may reach the generated
+        // function - anything else is rejected rather than executed
+        if (!'+-*/(),<>='.includes(c)) {
+            throw new Error('Unsupported character in derived expression: ' + c)
+        }
+        tokens.push({ type: 'punct', value: c })
+        i++
+    }
+
+    // X IN (a, b, c)  ->  __in([a, b, c], X)
+    const out = []
+    for (let t = 0; t < tokens.length; t++) {
+        const tok = tokens[t]
+        if (
+            tok.type === 'word' &&
+            tok.value.toLowerCase() === 'in' &&
+            out.length > 0 &&
+            tokens[t + 1] &&
+            tokens[t + 1].value === '('
+        ) {
+            let depth = 0
+            let j = t + 1
+            for (; j < tokens.length; j++) {
+                if (tokens[j].value === '(') depth++
+                else if (tokens[j].value === ')') {
+                    depth--
+                    if (depth === 0) break
+                }
+            }
+            if (j < tokens.length) {
+                const left = out.pop()
+                const inner = tokens.slice(t + 2, j)
+                out.push(
+                    { type: 'word', value: '__in' },
+                    { type: 'punct', value: '(' },
+                    { type: 'punct', value: '[' },
+                    ...inner,
+                    { type: 'punct', value: ']' },
+                    { type: 'punct', value: ',' },
+                    left,
+                    { type: 'punct', value: ')' }
+                )
+                t = j
+                continue
+            }
+        }
+        out.push(tok)
+    }
+
+    const OPS = {
+        and: '&&', or: '||',
+        eq: '==', ne: '!=', lt: '<', gt: '>', le: '<=', ge: '>='
+    }
+    const js = out
+        .map(tok => {
+            if (tok.type === 'str') return JSON.stringify(tok.value)
+            if (tok.type === 'num') return tok.value
+            if (tok.type === 'word') return OPS[tok.value.toLowerCase()] || tok.value
+            if (tok.value === '=') return '=='
+            if (tok.value === '<>') return '!='
+            return tok.value
+        })
+        .join(' ')
+
+    const sasTrue = v => {
+        if (typeof v === 'boolean') return v
+        if (typeof v === 'number') return v !== 0 && !isNaN(v)
+        if (v === null || v === undefined) return false
+        const str = String(v).trim()
+        return str !== '' && str !== '0'
+    }
+    const toStr = v => {
+        if (v === null || v === undefined) return ''
+        if (typeof v === 'number') return isNaN(v) ? '' : String(v)
+        return String(v).trim()
+    }
+    const fns = {
+        ifc: (c, a, b) => (sasTrue(c) ? a : b),
+        cats: (...args) => args.map(toStr).join(''),
+        catx: (sep, ...args) => args.map(toStr).filter(v => v !== '').join(toStr(sep)),
+        upcase: v => toStr(v).toUpperCase(),
+        lowcase: v => toStr(v).toLowerCase(),
+        strip: v => toStr(v),
+        trim: v => toStr(v),
+        length: v => toStr(v).length,
+        substr: (v, p, n) => {
+            const str = toStr(v)
+            const start = Number(p) - 1
+            return n === undefined
+                ? str.substring(start)
+                : str.substring(start, start + Number(n))
+        },
+        index: (v, sub) => toStr(v).indexOf(toStr(sub)) + 1,
+        __in: (list, v) => list.some(item => String(item) === String(v) || Number(item) === Number(v))
+    }
+
+    const names = Object.keys(fns)
+    const fn = new Function(...columnNames, ...names, 'return (' + js + ');')
+    return row => fn(...columnNames.map(n => row[n]), ...names.map(n => fns[n]))
+}
+
+/**
+ * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
+ * variables with a compiled row function, in dependency order (ties by name),
+ * the same sequence the real macro emits.
+ *
+ * @param {Object} opts
+ * @param {string} opts.libref - target libref
+ * @param {string} opts.table - target dataset
+ * @param {string} opts.dataDir - data directory of the DC control library
+ * @param {Array<Object>} opts.columns - physical columns of the target table
+ */
+function mpeDerived(opts) {
+    const { libref, table, dataDir, columns } = opts
+    const loadTableData = makeTableLoader(dataDir)
+    const cfg = loadTableData('MPE_DERIVED')
+    if (!cfg || !Array.isArray(cfg.rows) || cfg.rows.length === 0) return []
+
+    const now = Date.now() / 1000
+    const lib = String(libref || '').toUpperCase()
+    const dsn = String(table || '').toUpperCase()
+    const columnNames = (columns || []).map(c => c.name)
+
+    const vars = cfg.rows
+        .filter(r =>
+            String(colVal(r, 'LIBREF') || '').toUpperCase() === lib &&
+            String(colVal(r, 'DSN') || '').toUpperCase() === dsn &&
+            Number(colVal(r, 'ACTIVE')) === 1 &&
+            (!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
+        )
+        .map(r => {
+            const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
+            const nm = String(colVal(r, 'VAR_NM') || '').toUpperCase()
+            return {
+                name: nm,
+                // every input the expression uses - columns and derived vars
+                deps: String(colVal(r, 'VAR_DEPS') || '')
+                    .toUpperCase()
+                    .split(/\s+/)
+                    .filter(Boolean),
+                type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
+                length: Number(colVal(r, 'VAR_LEN')) || 8,
+                format: String(colVal(r, 'VAR_FMT') || ''),
+                logic,
+                label: String(colVal(r, 'VAR_LABEL') || '').trim() || nm + '=' + logic
+            }
+        })
+        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
+
+    // dependency order: repeatedly take the variables whose derived inputs are
+    // already placed, alphabetically within a round
+    const byName = new Map(vars.map(v => [v.name, v]))
+    const placed = []
+    const done = new Set()
+    while (placed.length < vars.length) {
+        const ready = vars
+            .filter(v => !done.has(v.name))
+            .filter(v => v.deps.every(d => !byName.has(d) || done.has(d)))
+            .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+        if (ready.length === 0) {
+            console.log('derived variables for ' + lib + '.' + dsn + ' contain a cycle')
+            return []
+        }
+        for (const v of ready) {
+            placed.push(v)
+            done.add(v.name)
+        }
+    }
+    vars.length = 0
+    vars.push.apply(vars, placed)
+
+    // compile each against the physical columns plus the variables defined
+    // earlier, so a derived variable can build on another one
+    const compileCols = columnNames.slice()
+    for (const d of vars) {
+        try {
+            // slice(): the compiled function captures the column list it was
+            // built with, not the array as it grows
+            d.fn = compileSasExpression(d.logic, compileCols.slice())
+        } catch (err) {
+            console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
+            d.fn = () => (d.type === 'N' ? null : '')
+        }
+        compileCols.push(d.name)
+    }
+    return vars
+}
+
+/**
+ * Adds the derived variables to every row and to the column list, so that the
+ * filter (which runs afterwards) and the grid both see them.
+ */
+function mpeDeriveRows(tableData, derived) {
+    if (!derived || derived.length === 0) return tableData
+    const rows = tableData.rows.map(row => {
+        const copy = Object.assign({}, row)
+        for (const d of derived) {
+            try {
+                copy[d.name] = d.fn(copy)
+            } catch (err) {
+                copy[d.name] = d.type === 'N' ? null : ''
+            }
+        }
+        return copy
+    })
+    const derivedCols = derived.map(d => ({
+        name: d.name,
+        type: d.type,
+        length: d.length,
+        format: d.format,
+        label: d.label,
+        notnull: false
+    }))
+    return {
+        name: tableData.name,
+        columns: (tableData.columns || []).concat(derivedCols),
+        rows
+    }
+}
+
 // ─── DC settings (services/settings.js) ──────────────────────────────────────
 
 /**

sas/mocks/sasjs/services/editors/getdata.js

@@ -98,6 +98,21 @@ function pkFirst(columns, keys) {
     return keyColumns.concat(columns.filter(col => !rank.has(col.name.toUpperCase())))
 }
 
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Applied before the filter, so a filter can reference them.  The derived
+// columns are added to the schema too, which is what puts them in COLS and in
+// the grid (read only, since the mock marks every column READ).
+
+const derived = mpeDerived({
+    libref: requestedLibref,
+    table: requestedTable,
+    dataDir: mpeDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
 // Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
 // NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
@@ -209,9 +224,8 @@ if (tableData && tableData.rows && tableData.columns) {
 // ─── Build response ──────────────────────────────────────────────────────────
 
 let response
-// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
-// getdata.sas, and is emitted as sasjsAbort below.
-let abortMsg = ''
+// Set when the row or cell limit cut the selection down - mirrors the
+// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.
 
 if (tableData && tableData.columns && tableData.rows) {
     // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
@@ -263,15 +277,15 @@ if (tableData && tableData.columns && tableData.rows) {
     }
 
     // ─── Row and cell limits ─────────────────────────────────────────────────
-    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
-    // cells (rows x columns), whichever is reached first.  Both are read from
-    // MPE_CONFIG, the way the real service picks them up from the settings
-    // program it generates - so editing the option in DC changes what the mock
-    // allows, and an option that is absent (or switched off, var_active=0)
-    // falls back to the shipped default.  The check runs after the
-    // PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
-    // replace the data with a larger table.  The cell count includes the
-    // housekeeping delete column, which is what getdata.sas counts.
+    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
+    // cells (rows x columns), whichever is reached first - the selection is
+    // cut to min(rows, cells/columns) and flagged in sasparams.
+    // Both options are read from MPE_CONFIG, the way the real service picks
+    // them up from the settings program it generates - so editing the option
+    // in DC changes what the mock serves, and an option that is absent (or
+    // switched off, var_active=0) falls back to the shipped default.  The
+    // check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
+    // does - a hook may replace the data with a larger table.
     // MPE_CONFIG lives in the control library, not in the table's own library,
     // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
     // uses.  Passing the requested table's dataDir made the read fail for every
@@ -279,20 +293,32 @@ if (tableData && tableData.columns && tableData.rows) {
     const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
     const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
     const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
-    const editColCount = visibleColumns.length + 1
-    const cellCount = visibleRows.length * editColCount
-    if (visibleRows.length > maxObsWebEdit) {
-        abortMsg =
-            'Table is too big (' +
-            visibleRows.length +
-            ' rows) - please filter and try again!'
-    } else if (cellCount > maxCellsWebEdit) {
-        abortMsg =
-            'Selection is too wide (' +
-            visibleRows.length +
-            ' rows x ' +
-            editColCount +
-            ' cols) - please filter and try again!'
+    // getdata.sas counts the columns of work.out, i.e. the table's own
+    // columns - the delete column is client-owned and not part of it.
+    const editColCount = visibleColumns.length
+    let maxRows = maxObsWebEdit
+    let limitVar = 'DC_MAXOBS_WEBEDIT'
+    if (editColCount > 0) {
+        const cellRows = Math.floor(maxCellsWebEdit / editColCount)
+        if (cellRows < maxRows) {
+            maxRows = cellRows
+            limitVar = 'DC_MAXCELLS_WEBEDIT'
+        }
+    }
+    if (maxRows < 1) maxRows = 1
+    let truncated = 0
+    let truncMsg = ''
+    if (visibleRows.length > maxRows) {
+        truncated = 1
+        // Same wording as the macro getdata.sas sets, so the spec can assert
+        // on the text the real service would send.
+        truncMsg =
+            'Only the first ' +
+            maxRows +
+            ' rows are loaded - the ' +
+            limitVar +
+            ' limit has been reached.  Filter the table to work with the rest.'
+        visibleRows = visibleRows.slice(0, maxRows)
     }
 
     // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
@@ -398,7 +424,9 @@ if (tableData && tableData.columns && tableData.rows) {
             LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
             RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
             CLS_FLAG: 0,
-            ISMAP: 0
+            ISMAP: 0,
+            TRUNCATED: truncated,
+            TRUNC_MSG: truncMsg
         }],
         versions: [],
         xl_rules: []
@@ -413,26 +441,21 @@ if (tableData && tableData.columns && tableData.rows) {
         ],
         maxvarlengths: [], query: [], sasdata: [],
         $sasdata: { vars: {} },
-        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0 }],
+        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
         versions: [], xl_rules: []
     }
 }
 
 webOutOpen()
-if (abortMsg) {
-    webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
-    webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
-} else {
-    webOutObj(response.approvers, 'approvers')
-    webOutObj(response.cols, 'cols')
-    webOutObj(response.dqdata, 'dqdata')
-    webOutObj(response.dqrules, 'dqrules')
-    webOutObj(response.dsmeta, 'dsmeta')
-    webOutObj(response.maxvarlengths, 'maxvarlengths')
-    webOutObj(response.query, 'query')
-    webOutObj(response.sasdata, 'sasdata', response.$sasdata)
-    webOutObj(response.sasparams, 'sasparams')
-    webOutObj(response.versions, 'versions')
-    webOutObj(response.xl_rules, 'xl_rules')
-}
+webOutObj(response.approvers, 'approvers')
+webOutObj(response.cols, 'cols')
+webOutObj(response.dqdata, 'dqdata')
+webOutObj(response.dqrules, 'dqrules')
+webOutObj(response.dsmeta, 'dsmeta')
+webOutObj(response.maxvarlengths, 'maxvarlengths')
+webOutObj(response.query, 'query')
+webOutObj(response.sasdata, 'sasdata', response.$sasdata)
+webOutObj(response.sasparams, 'sasparams')
+webOutObj(response.versions, 'versions')
+webOutObj(response.xl_rules, 'xl_rules')
 webOutClose()

sas/mocks/sasjs/services/public/getcolvals.js

@@ -21,9 +21,25 @@ const tableName = libds.split('.')[1] || ''
 // ─── Load table data (searches all libref data folders) ──────────────────────
 
 const _loaded = loadTableAnyLib(tableName) || {}
-const tableData = _loaded.data
+let tableData = _loaded.data
 const tableLibref = _loaded.libref || libref
 
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Present in the schema and in the rows before the filter is applied, so the
+// filter may reference them and the values below are computed from the same
+// row set the grid shows.
+
+const controlDataDir = libDataDir(dcLibref)
+const derived = mpeDerived({
+    libref: tableLibref,
+    table: tableName,
+    dataDir: controlDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── Resolve column metadata ─────────────────────────────────────────────────
 
 let colMeta = null

sas/package-lock.json

@@ -6,8 +6,8 @@
     "": {
       "name": "dc-sas",
       "dependencies": {
-        "@sasjs/cli": "4.20.9",
-        "@sasjs/core": "5.5.2"
+        "@sasjs/cli": "6.0.0",
+        "@sasjs/core": "6.0.0"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -140,81 +140,44 @@
       }
     },
     "node_modules/@fast-csv/format": {
-      "version": "4.3.5",
-      "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz",
-      "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==",
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-5.0.7.tgz",
+      "integrity": "sha512-VdypoRxv7PF+LsyPouTMKdB0d76hync+gLpgdNqfqVK44MsgW4oiCJSdrki2FisWT7v2QGUYDHjp4L7w5oO6gw==",
       "license": "MIT",
       "dependencies": {
-        "@types/node": "^14.0.1",
-        "lodash.escaperegexp": "^4.1.2",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isequal": "^4.5.0",
-        "lodash.isfunction": "^3.0.9",
-        "lodash.isnil": "^4.0.0"
+        "lodash.escaperegexp": "^4.1.2"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "4.19.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.19.2.tgz",
-      "integrity": "sha512-7uxkpk0kjK4PZ552YK0CsTmEKEurK/1uzloR4rxplbViRkVdaewP5l0hIrGDBVM4z0gmV5pf1jh8uil3/NSAzw==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-5.1.1.tgz",
+      "integrity": "sha512-rso6+xRpZhFuuUbgHmGztznuse4xZjlu7+dOcYMUMJezAd5p/xdCt+wvv4eIKejsMWDcIteSeRPOkgKr0TlxWg==",
       "license": "ISC",
       "dependencies": {
-        "@sasjs/utils": "3.6.4",
+        "@sasjs/utils": "4.0.1",
         "axios": "1.20.0",
         "axios-cookiejar-support": "5.0.5",
         "form-data": "4.0.6",
-        "tough-cookie": "4.1.3"
-      }
-    },
-    "node_modules/@sasjs/adapter/node_modules/@sasjs/utils": {
-      "version": "3.6.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.4.tgz",
-      "integrity": "sha512-tLbvIFMRb3KrHW5EapvMS/0tSa6W/SYCuR+fcdavayQ4eaUFayeRN3HKu2gXLtWJjHf/HijPVTsNwkIRwDvJMA==",
-      "license": "ISC",
-      "dependencies": {
-        "@fast-csv/format": "4.3.5",
-        "@types/fs-extra": "11.0.4",
-        "@types/prompts": "2.4.9",
-        "chalk": "4.1.1",
-        "cli-table": "0.3.11",
-        "consola": "3.4.2",
-        "fs-extra": "11.4.1",
-        "jwt-decode": "3.1.2",
-        "prompts": "2.4.2"
-      }
-    },
-    "node_modules/@sasjs/adapter/node_modules/chalk": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
-      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
-      "license": "MIT",
-      "dependencies": {
-        "ansi-styles": "^4.1.0",
-        "supports-color": "^7.1.0"
+        "tough-cookie": "5.1.2"
       },
       "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/chalk?sponsor=1"
+        "node": ">=22"
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "4.20.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.20.9.tgz",
-      "integrity": "sha512-wJCdd6fRI0RVwEDLDsauUbsKQsH69aaYJqDkV9WfSKkRH1VxgU9zFGHdSxrKLOzrjwwybn/DGzh2xcgeeyl6jA==",
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-6.0.0.tgz",
+      "integrity": "sha512-nsdL5J+Y5FHZL4pieqQHjsNmY2DZ4gQcGAupR0dv8aGlweEHqdVErtx3B7+EmIiyfRt+oE3YEofzF41WBWL7tg==",
       "license": "ISC",
       "dependencies": {
-        "@sasjs/adapter": "4.19.2",
-        "@sasjs/core": "5.3.0",
-        "@sasjs/lint": "4.1.2",
-        "@sasjs/utils": "3.6.5",
+        "@sasjs/adapter": "5.1.1",
+        "@sasjs/core": "6.0.0",
+        "@sasjs/utils": "4.0.1",
+        "axios": "1.20.0",
         "chalk": "4.1.2",
-        "dotenv": "17.4.2",
-        "js-base64": "3.9.3",
+        "dotenv": "18.0.8",
+        "js-base64": "3.9.4",
         "jsdom": "26.0.0",
-        "jwt-decode": "3.1.2",
-        "lodash.groupby": "4.6.0",
         "lodash.uniqby": "4.7.0",
         "node-graphviz": "0.1.1",
         "node-powershell": "5.0.1",
@@ -230,49 +193,50 @@
         "sasjs": "build/index.js"
       },
       "engines": {
-        "node": "^14.18.0 || >=16.10.0"
+        "node": ">=22"
+      },
+      "peerDependencies": {
+        "@sasjs/lint": "4.3.0"
       }
     },
-    "node_modules/@sasjs/cli/node_modules/@sasjs/core": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.3.0.tgz",
-      "integrity": "sha512-6x1hEPa1pCENyeG0Nro3YcZZ+A8Ns60pO6gDuvG1X6oWdwOKKPpZztQNKE6khFmXXIGl0uUgT9CbappUET+odA==",
-      "license": "MIT"
-    },
     "node_modules/@sasjs/core": {
-      "version": "5.5.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.5.2.tgz",
-      "integrity": "sha512-puojJhAKAKg5o0uHdGSkpQ3cXgzajMY9AXAm3R22l6bZ4GKvC03qRq1ueGjBBF3npNxviwIKCABTsNEtjs1LAw==",
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.0.0.tgz",
+      "integrity": "sha512-E9iXPvNgCux80xHhfopCsQE/CUNp1NEF1Z7NRj089A1T8WXP9AWJYcfFfOqoljNh8rnMo1fbIgoNrIaaAgW0MA==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-4.1.2.tgz",
-      "integrity": "sha512-P7QqeMPnhg/AvqH40dmyY2OArlvAYAp81/ijK/PhmS+x/HslpypdLoNy5iN7SwixUWPde5fPwV/ZBmcjVK0uLA==",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-4.3.0.tgz",
+      "integrity": "sha512-65cjy4IjA4PgSlicOruF0DI9N2HO3LJqtgphsZCg8g7N/StE2FWLJLJbD1N4OHxN6htrUfj5SEawJJiHo5T9Yw==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
-        "@sasjs/utils": "3.6.4",
+        "@sasjs/utils": "4.0.0",
         "ignore": "7.0.11"
       },
       "engines": {
-        "node": "^14.18.0 || >=16.10.0"
+        "node": ">=22"
       }
     },
     "node_modules/@sasjs/lint/node_modules/@sasjs/utils": {
-      "version": "3.6.4",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.4.tgz",
-      "integrity": "sha512-tLbvIFMRb3KrHW5EapvMS/0tSa6W/SYCuR+fcdavayQ4eaUFayeRN3HKu2gXLtWJjHf/HijPVTsNwkIRwDvJMA==",
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-4.0.0.tgz",
+      "integrity": "sha512-lARoCnRM4gSVf8ShoP5elHi7W28K/14MK037dQ0ABgMlP3+r6OuhAZYwK/a70tED47tKxyIoCAKLdP6jrG0ewQ==",
       "license": "ISC",
+      "peer": true,
       "dependencies": {
-        "@fast-csv/format": "4.3.5",
+        "@fast-csv/format": "5.0.7",
         "@types/fs-extra": "11.0.4",
         "@types/prompts": "2.4.9",
         "chalk": "4.1.1",
-        "cli-table": "0.3.11",
         "consola": "3.4.2",
         "fs-extra": "11.4.1",
-        "jwt-decode": "3.1.2",
+        "jwt-decode": "4.0.0",
         "prompts": "2.4.2"
+      },
+      "engines": {
+        "node": ">=22"
       }
     },
     "node_modules/@sasjs/lint/node_modules/chalk": {
@@ -280,6 +244,7 @@
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
       "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "ansi-styles": "^4.1.0",
         "supports-color": "^7.1.0"
@@ -292,20 +257,22 @@
       }
     },
     "node_modules/@sasjs/utils": {
-      "version": "3.6.5",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.6.5.tgz",
-      "integrity": "sha512-2iCy0EmZJCQSG1JsVa1t/ulUb7lfdBnp6wiRPRl9/AaxRu8D30ypqUKn6heos1lxhNxg42JFsry8rlupiajdHg==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-4.0.1.tgz",
+      "integrity": "sha512-aNEbOJFZ2Ru71/XkHKtywexBQYdst8ovq0a9vuVO4subZuKMjRtYcn4kTRr9VVIz65p8xGtVUkFEf+6t9Vbfcg==",
       "license": "ISC",
       "dependencies": {
-        "@fast-csv/format": "4.3.5",
+        "@fast-csv/format": "5.0.7",
         "@types/fs-extra": "11.0.4",
         "@types/prompts": "2.4.9",
         "chalk": "4.1.1",
-        "cli-table": "0.3.11",
         "consola": "3.4.2",
         "fs-extra": "11.4.1",
-        "jwt-decode": "3.1.2",
+        "jwt-decode": "4.0.0",
         "prompts": "2.4.2"
+      },
+      "engines": {
+        "node": ">=22"
       }
     },
     "node_modules/@sasjs/utils/node_modules/chalk": {
@@ -344,10 +311,13 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "14.18.63",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz",
-      "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==",
-      "license": "MIT"
+      "version": "26.6.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz",
+      "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==",
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~8.9.0"
+      }
     },
     "node_modules/@types/prompts": {
       "version": "2.4.9",
@@ -580,17 +550,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/cli-table": {
-      "version": "0.3.11",
-      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.11.tgz",
-      "integrity": "sha512-IqLQi4lO0nIB4tcdTpN4LCB9FI3uqrJZK7RC515EnhZ6qBaglkIgICb1wjeAqpdoOabm1+SuQtkXIPdYC93jhQ==",
-      "dependencies": {
-        "colors": "1.0.3"
-      },
-      "engines": {
-        "node": ">= 0.2.0"
-      }
-    },
     "node_modules/cliui": {
       "version": "8.0.1",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
@@ -632,15 +591,6 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "license": "MIT"
     },
-    "node_modules/colors": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
-      "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.1.90"
-      }
-    },
     "node_modules/combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -733,10 +683,13 @@
       }
     },
     "node_modules/dotenv": {
-      "version": "17.4.2",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz",
-      "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==",
+      "version": "18.0.8",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-18.0.8.tgz",
+      "integrity": "sha512-2KDeoFoK/zRtcnbunCDlLG5VWVMcyqeQc6iK65o231nfNAcuGrKoqUQfJFUtBI+QlShk5HKIHxwebBtUl3rEfA==",
       "license": "BSD-2-Clause",
+      "bin": {
+        "dotenv": "dist/index.cjs"
+      },
       "engines": {
         "node": ">=12"
       },
@@ -1118,6 +1071,7 @@
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.11.tgz",
       "integrity": "sha512-YChdK5txDjwGUvgR7oCJcLGkwi3LDh8Zx8tS7ndYciLEg/oOCL26VUBSUEtgS7EjiybcBqFxh5j0rAdDX7/bbg==",
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">= 4"
       }
@@ -1192,9 +1146,9 @@
       }
     },
     "node_modules/js-base64": {
-      "version": "3.9.3",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.3.tgz",
-      "integrity": "sha512-uwYQp+VJ38FVvtim6qNbit6e9uT6dwWQ4Y1+H9TxhW5hcHjpHwoxlR0nMpqUmIFOmu4VqMxwdJA88gIVuZJQ/g==",
+      "version": "3.9.4",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.9.4.tgz",
+      "integrity": "sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q==",
       "license": "BSD-3-Clause"
     },
     "node_modules/jsdom": {
@@ -1250,18 +1204,6 @@
         "node": ">= 14"
       }
     },
-    "node_modules/jsdom/node_modules/tough-cookie": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
-      "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
-      "license": "BSD-3-Clause",
-      "dependencies": {
-        "tldts": "^6.1.32"
-      },
-      "engines": {
-        "node": ">=16"
-      }
-    },
     "node_modules/jsonfile": {
       "version": "6.2.1",
       "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz",
@@ -1275,10 +1217,13 @@
       }
     },
     "node_modules/jwt-decode": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.1.2.tgz",
-      "integrity": "sha512-UfpWE/VZn0iP50d8cz9NrZLM9lSWhcJ+0Gt/nm4by88UL+J1SiKN8/5dkjMmbEzwL2CAe+67GsegCbIKtbp75A==",
-      "license": "MIT"
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-4.0.0.tgz",
+      "integrity": "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
     },
     "node_modules/kind-of": {
       "version": "6.0.3",
@@ -1304,37 +1249,6 @@
       "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==",
       "license": "MIT"
     },
-    "node_modules/lodash.groupby": {
-      "version": "4.6.0",
-      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
-      "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isboolean": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
-      "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isequal": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
-      "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==",
-      "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isfunction": {
-      "version": "3.0.9",
-      "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz",
-      "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isnil": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz",
-      "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==",
-      "license": "MIT"
-    },
     "node_modules/lodash.uniqby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
@@ -1572,18 +1486,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/psl": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz",
-      "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==",
-      "license": "MIT",
-      "dependencies": {
-        "punycode": "^2.3.1"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/lupomontero"
-      }
-    },
     "node_modules/punycode": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
@@ -1593,12 +1495,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/querystringify": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
-      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
-      "license": "MIT"
-    },
     "node_modules/readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -1622,12 +1518,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
-      "license": "MIT"
-    },
     "node_modules/restore-cursor": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz",
@@ -1778,27 +1668,15 @@
       "license": "MIT"
     },
     "node_modules/tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz",
+      "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
       "license": "BSD-3-Clause",
       "dependencies": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
+        "tldts": "^6.1.32"
       },
       "engines": {
-        "node": ">=6"
-      }
-    },
-    "node_modules/tough-cookie/node_modules/universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">= 4.0.0"
+        "node": ">=16"
       }
     },
     "node_modules/tr46": {
@@ -1819,6 +1697,12 @@
       "integrity": "sha512-PqdsRmhir3lUwraK/O6sdZ9QcmtUK0c6F1utX/WNquJsL4Jpdck+DSe9Mqy8Wk7uAQD/SlVn6cyGF5bj6VF33Q==",
       "license": "MIT"
     },
+    "node_modules/undici-types": {
+      "version": "8.9.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
+      "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
+      "license": "MIT"
+    },
     "node_modules/universalify": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
@@ -1828,16 +1712,6 @@
         "node": ">= 10.0.0"
       }
     },
-    "node_modules/url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
-      "license": "MIT",
-      "dependencies": {
-        "querystringify": "^2.1.1",
-        "requires-port": "^1.0.0"
-      }
-    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

sas/package.json

@@ -30,8 +30,8 @@
   },
   "private": true,
   "dependencies": {
-    "@sasjs/cli": "4.20.9",
-    "@sasjs/core": "5.5.2"
+    "@sasjs/cli": "6.0.0",
+    "@sasjs/core": "6.0.0"
   },
   "overrides": {
     "nanoid": "3.3.18"

sas/sasjs/db/datactrl/mpe_derived.ddl

@@ -0,0 +1,32 @@
+/**
+  @file mpe_derived.ddl
+  @brief ddl file
+  @details Defines the derived (computed) variable configuration table.  This
+  drives MPE_DERIVED rows in MPE_TABLES, so that the table itself can be
+  maintained through the EDIT screen with the usual approve workflow.
+
+  Derived values are not persisted - the logic is applied at read time, before
+  the filter, so that the variables can be filtered on.
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+
+create table &curlib..MPE_DERIVED
+   (
+    TX_FROM num not null format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VAR_NM varchar(32),
+    VAR_DEPS varchar(4000),
+    VAR_TYPE varchar(1),
+    VAR_LEN int,
+    VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
+    VAR_LOGIC varchar(4000),
+    ACTIVE int,
+    TX_TO num not null format=datetime19.3,
+    constraint pk_mpe_derived
+        primary key(tx_from, libref, dsn, var_nm));

sas/sasjs/db/datactrl/mpe_tables.ddl

@@ -118,3 +118,16 @@ insert into &mpelib..mpe_tables
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
     ;
+  insert into &mpelib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&mpelib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;

sas/sasjs/db/migrations/20261019_v7.19_release.sas

@@ -0,0 +1,169 @@
+/**
+  @file
+  @brief migration script to move to v7.19 of Data Controller
+
+  REQUIRED CHANGE - adds the MPE_DERIVED table, which holds derived (computed)
+  variable definitions for the EDIT screen, and registers it in MPE_TABLES so
+  that it can be maintained through the app with the usual approve workflow.
+
+  Replace YOURDCLIB and the library path below, then run the script.  Any
+  MPE_DERIVED rows added before this script runs will be lost when the table
+  is created, so run it before configuring any derived variables.
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+proc sql;
+create table &dclib..mpe_derived
+  (
+    tx_from num not null format=datetime19.3,
+    libref varchar(8),
+    dsn varchar(32),
+    var_nm varchar(32),
+    var_deps varchar(4000),
+    var_type varchar(1),
+    var_len num,
+    var_fmt varchar(32),
+    var_label varchar(256),
+    var_logic varchar(4000),
+    active num,
+    tx_to num not null format=datetime19.3
+  );
+quit;
+
+proc datasets lib=&dclib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn var_nm)
+    /nomiss unique;
+quit;
+
+proc sql;
+insert into &dclib..mpe_tables
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="&dclib"
+    ,dsn='MPE_DERIVED'
+    ,num_of_approvals_required=1
+    ,loadtype='TXTEMPORAL'
+    ,buskey='LIBREF DSN VAR_NM'
+    ,post_edit_hook='services/hooks/mpe_derived_postedit'
+    ,notes='Derived (computed) variables for the EDIT screen'
+    ,var_txfrom='TX_FROM'
+    ,var_txto='TX_TO'
+  ;
+quit;
+
+/* Dropdown values for VAR_TYPE */
+proc sql noprint;
+  select max(selectbox_rk) into: maxrk from &dclib..mpe_selectbox;
+quit;
+
+%let rk1=%eval(&maxrk+1);
+%let rk2=%eval(&maxrk+2);
+
+proc sql;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk1
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='C'
+    ,selectbox_order=1
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk2
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='N'
+    ,selectbox_order=2
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+quit;
+
+/* Field rules for the EDIT screen.  These are client side - the server side
+  * enforcement is the post edit hook registered above. */
+proc sql;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+quit;

sas/sasjs/macros/mpe_derived.sas

@@ -0,0 +1,103 @@
+/**
+  @file
+  @brief Generates the SAS statements that create Data Controller derived variables
+  @details Derived (computed) variables are configured in MPE_DERIVED.  Each
+  row defines one variable, which is created in a DATA step view ahead of the
+  filter, so that it can be filtered on in the EDIT screen.
+
+  The engine lives in @sasjs/core as mp_derivedvars - this macro only supplies
+  what is specific to Data Controller: the control table name, the bitemporal
+  filter that picks the current version of a row, and the list of names that
+  belong to the metadata and may not be used as a variable.
+
+  The macro writes the generated statements to a fileref, so that callers can
+  compose them into a view:
+
+      %mpe_derived(libds=&libds,dclib=&mpelib,outref=dercode)
+      data work.vw_derived / view=work.vw_derived;
+        set &libds;
+        %include dercode;
+      run;
+
+  Alternatively, pass outview= and the macro will build the view itself.
+
+  The filter must be applied by the caller, in the step that reads the view:
+
+      data out;
+        set work.vw_derived;
+        where %inc filtref;;
+      run;
+
+  A WHERE statement applies to the data set named on the SET statement, so it
+  cannot reference a variable that is created in the same data step.  Applying
+  it to the view puts the derived variables on the input, which is what makes
+  them filterable.
+
+  Each statement block is of the form:
+
+      attrib REGION length=$21 label='REGION=<expression>' format=$21.;
+      REGION = <expression>;
+
+  Type and length are declared by the administrator, so the result is
+  deterministic.  VAR_LABEL is optional: when it is missing or blank the label
+  is the variable name, an equals sign and the expression, otherwise the
+  configured label is used as given.
+
+  VAR_DEPS lists every input the expression uses - physical columns and
+  derived variables alike.  The statements are generated in dependency order,
+  so a derived variable may build on another one without a sequence number
+  being maintained.  A cycle is rejected.
+
+  Semicolons, macro triggers and comments are rejected, so a row can only ever
+  be an expression.  A derived variable may not reuse the name of a physical
+  column, and may not reference itself: an intra-row calculation would imply a
+  retained value, which depends on the order rows happen to arrive in.
+
+  <h4> SAS Macros </h4>
+  @li mp_derivedvars.sas
+  @li mp_abort.sas
+
+  @param [in] libds= The target table (libref.dataset, lib must be assigned)
+  @param [in] dclib= The Data Controller control library
+  @param [out] outref= (0) Fileref to receive the generated statements
+  @param [out] outvars= (work.derived_vars) Dataset of derived variable names
+  @param [in] outview= (0) Optional two-level name for the derived view
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%macro mpe_derived(libds=,
+  dclib=,
+  outref=0,
+  outvars=work.derived_vars,
+  outview=0
+);
+
+%local reserved;
+
+/* the only name that has to be refused outright - the metadata columns of the
+  target table are physical columns, so the engine's collision check covers
+  them whatever MPE_TABLES calls them */
+%let reserved=%str(_____DELETE__THIS__RECORD_____);
+
+%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing: libds dclib)
+)
+
+%mp_derivedvars(
+  libds=&libds
+  ,control=&dclib..mpe_derived
+  ,where=%str(&dc_dttmtfmt lt tx_to and active=1)
+  ,outref=&outref
+  ,outvars=&outvars
+  ,outview=&outview
+  ,reserved=&reserved
+)
+
+%mend mpe_derived;

sas/sasjs/macros/mpe_derived.test.sas

@@ -0,0 +1,296 @@
+/**
+  @file
+  @brief testing the mpe_derived macro
+
+  @details Covers the generated statements, the view, the metadata the callers
+  consume, and the dependency rules.
+
+  The configuration checks abort the run when they fail, so they are not
+  exercised here - this suite covers the behaviour that callers depend on.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mpe_derived.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+
+/* a source table, and a table with no derived variables */
+data work.class;
+  set sashelp.class;
+run;
+data work.noconfig;
+  set sashelp.class;
+run;
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+/**
+  * Test 1 - statements, view, chaining
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='BAND'
+    ,var_deps='AGE'
+    ,var_type='C'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_label='Age band'
+    ,var_logic="ifc(AGE>13,'TEEN','JUNIOR')"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='BAND_AGE'
+    ,var_deps='BAND AGE'
+    ,var_type='C'
+    ,var_len=16
+    ,var_fmt=''
+    ,var_logic="cats(BAND,'-',AGE)"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='DOUBLE_AGE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE * 2'
+    ,active=1;
+quit;
+
+/* lower case libds on purpose - the config match must be case insensitive */
+/* the setup SQL above creates the SQL automatic macro variables, so the
+  snapshot is taken here - the check is for the macro's own leakage */
+%mp_assertscope(SNAPSHOT)
+
+%mpe_derived(libds=work.class,
+  dclib=&dclib,
+  outref=derref1,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+
+%mp_assertdsobs(work.derived_vars,
+  desc=Test1 - one row per derived variable,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+%mp_assertcols(work.vw_derived,
+  desc=Test1 - the view carries the derived columns,
+  cols=BAND BAND_AGE DOUBLE_AGE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk1;
+  set work.vw_derived;
+  where BAND='TEEN';
+run;
+%mp_assertdsobs(work.chk1,
+  desc=Test1 - the filter on a derived column selects rows,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* the chained value only resolves if the statements run in order */
+data work.chk1b;
+  set work.vw_derived;
+  where BAND='TEEN';
+  if BAND_AGE ne cats(BAND,'-',AGE) then output;
+run;
+%mp_assertdsobs(work.chk1b,
+  desc=Test1 - a derived variable can reference an earlier one,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+data work.chk1c;
+  set work.vw_derived;
+  if DOUBLE_AGE ne AGE*2 then output;
+run;
+%mp_assertdsobs(work.chk1c,
+  desc=Test1 - the numeric derived value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* proc contents on a view reports a format name without its width, so the
+  format is compared on the name only */
+proc contents noprint data=work.vw_derived
+  out=work.vw_cols(keep=name label format);
+run;
+data work.chk1d;
+  set work.vw_cols;
+  where name='BAND' and label='Age band' and format=:'$';
+run;
+%mp_assertdsobs(work.chk1d,
+  desc=Test1 - the label and default format are applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - a configured format is used as-is
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='FLAG_DT'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt='DATE9.'
+    ,var_logic='mdy(1,1,2020)'
+    ,active=1;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref2,
+  outview=work.vw_derived2,
+  outvars=work.derived_vars2
+)
+
+/* asserted on the emitted statement - proc contents on a view does not
+  report the format in full */
+data work.chk2;
+  infile derref2 truncover;
+  input line $4000.;
+  if index(line,'format=DATE9.')>0 then output;
+run;
+%mp_assertdsobs(work.chk2,
+  desc=Test2 - the configured format is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk2b;
+  set work.vw_derived2;
+  if flag_dt ne mdy(1,1,2020) then output;
+  stop;
+run;
+%mp_assertdsobs(work.chk2b,
+  desc=Test2 - the numeric value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - no configuration
+  */
+
+%mpe_derived(libds=WORK.NOCONFIG,
+  dclib=&dclib,
+  outref=derref3,
+  outview=work.vw_derived3,
+  outvars=work.derived_vars3
+)
+
+%mp_assertdsobs(work.derived_vars3,
+  desc=Test3 - no derived variables returned,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+%mp_assert(iftrue=(%mf_existds(libds=work.vw_derived3)=0),
+  desc=Test3 - no view is built when there is no config,
+  outds=work.test_results
+)
+
+data _null_;
+  infile derref3 end=eof;
+  input;
+  if not eof or cats(_infile_) ne '' then do;
+    putlog 'ERR' 'OR: derived statement file is not empty';
+    call symputx('emptycheck','0','l');
+  end;
+run;
+%mp_assert(iftrue=(%symexist(emptycheck)=0),
+  desc=Test3 - no statements are generated,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - the wrapper scopes the config: only the current, active row of a
+  * table is used
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='01JAN2020:00:00:00'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='CLOSED_OFF'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 1'
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='NOT_ACTIVE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 2'
+    ,active=0;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.chk4;
+  set work.derived_vars4;
+  where name in ('CLOSED_OFF','NOT_ACTIVE');
+run;
+
+%mp_assertdsobs(work.chk4,
+  desc=Test4 - a closed version and an inactive row are ignored,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* tidy up so repeated runs start clean */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+%mp_assertscope(COMPARE,ignorelist=_SASJS_WEBOUT_HEADERS,outds=work.test_results)

sas/sasjs/macros/mpe_makedata.sas

@@ -679,6 +679,24 @@ insert into &lib..mpe_security set
       ,selectbox_value='UNLOCKED'
       ,selectbox_order=2
       ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='C'
+      ,selectbox_order=1
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='N'
+      ,selectbox_order=2
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
   insert into &lib..mpe_selectbox set
       selectbox_rk=%mf_increment(rk)
       ,ver_from_dttm=0
@@ -1495,6 +1513,19 @@ insert into &lib..mpe_selectbox set
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
     ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;
   insert into &lib..mpe_tables
     set tx_from=0
       ,tx_to='31DEC5999:23:59:59'dt
@@ -1944,6 +1975,87 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/columns_in_libds"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+
+/* MPE_DERIVED - the field rules are also enforced server-side, in
+  * services/hooks/mpe_derived_postedit.sas.  These rows give the user
+  * immediate feedback in the EDIT screen. */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"

sas/sasjs/macros/mpe_makedatamodel.sas

@@ -692,6 +692,29 @@ proc datasets lib=&lib noprint;
     pk_mpe_validations=(tx_from base_lib base_ds base_col rule_type)
     /nomiss unique;
 quit;
+/* mpe_derived - computed variables for the EDIT screen */
+proc sql;
+create table &lib..MPE_DERIVED
+  (
+    TX_FROM num &notnull format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VAR_NM varchar(32),
+    VAR_DEPS varchar(4000),
+    VAR_TYPE varchar(1),
+    VAR_LEN num,
+    VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
+    VAR_LOGIC varchar(4000),
+    ACTIVE num,
+    TX_TO num &notnull format=datetime19.3
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn var_nm)
+    /nomiss unique;
+quit;
 proc sql;
 create table &lib..mpe_x_test(
           primary_key_field num &notnull,

sas/sasjs/services/editors/getdata.sas

@@ -17,6 +17,10 @@
   @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
     else set to 1 (CLS rules exist)
   @li ISMAP - set to 1 if the target DS is an excel map target, else 0
+  @li TRUNCATED - set to 1 if the selection was cut down to the
+    DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
+  @li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
+    else empty
 
   <h5> approvers </h5>
   <h5> dqrules </h5>
@@ -68,11 +72,14 @@
   @li mp_validatecol.sas
   @li mpe_accesscheck.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getlabels.sas
   @li mpe_getversions.sas
   @li mpe_filtermaster.sas
   @li mpe_runhook.sas
+  @li mpeinit.sas
+  @li mpeterm.sas
 
   @version 9.2
   @author 4GL Apps Ltd
@@ -201,6 +208,21 @@ select upcase(loadtype)
   outds=work.query
 )
 
+/**
+  * Derived (computed) variables, configured in MPE_DERIVED.  They are added
+  * via a view, so that the filter (applied by the step that reads the view)
+  * can reference them - a WHERE clause cannot see a variable that is created
+  * in the same data step.
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%put &=derived_nobs;
+
 %macro mpestp_getdata();
   %if not %symexist(DC_MAXOBS_WEBEDIT) %then %do;
     %put NOTE:;%put NOTE- DC_MAXOBS_WEBEDIT not found!;
@@ -237,7 +259,12 @@ select upcase(loadtype)
 
   %if &loadtype=BITEMPORAL %then %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
       where %inc filtref;;
     run;
     proc sort data=out;
@@ -251,7 +278,12 @@ select upcase(loadtype)
   %end;
   %else %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
       where %inc filtref;;
     run;
   %end;
@@ -262,30 +294,40 @@ select upcase(loadtype)
   )
 
   /* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
-    then run the hook, then check the row and cell limits.  Checking once,
-    after both, means the counts cover everything that could reach the
-    browser: the rows the selection matched, and anything the hook has added
-    to work.out.  A wide table exhausts the cell budget long before it
-    reaches the row limit - 250 rows of a 1000 column table is a quarter of
-    a million cells - so capping rows alone does not bound the work the
-    browser is asked to do.  The payload itself is capped further down, when
-    work.outdata is built. */
+    then run the hook, then apply the row and cell limits (below). */
   %let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
   proc sort data=work.out; by &sortPK; run;
 
   %mpe_runhook(PRE_EDIT_HOOK)
 
+  /* Serve the selection truncated, at whichever limit binds first, and carry
+    the reason in sasparams for the EDIT screen to show against the table
+    name.  Bounding the browser's work is the point: a wide table exhausts the
+    cell budget long before it reaches the row limit - 250 rows of a 1000
+    column table is a quarter of a million cells - so the row cap is
+    min(DC_MAXOBS_WEBEDIT, DC_MAXCELLS_WEBEDIT / columns).  Counting after the
+    PRE_EDIT_HOOK means the cap covers what the hook added to work.out. */
   %let obscnt=%mf_getattrn(work.out,NLOBS);
   %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
-  %let cellcnt=%eval(&obscnt * &colcnt);
-  %mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Table is too big (&obscnt rows) - please filter and try again!
-  )
-  %mp_abort(iftrue=(&cellcnt>&DC_MAXCELLS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Selection is too wide (&obscnt rows x &colcnt cols) - please filter and try again!
-  )
+  %let maxrows=&DC_MAXOBS_WEBEDIT;
+  %let limitvar=DC_MAXOBS_WEBEDIT;
+  %if &colcnt>0 %then %do;
+    %let cellrows=%sysfunc(floor(&DC_MAXCELLS_WEBEDIT/&colcnt));
+    %if %sysevalf(&cellrows lt &maxrows) %then %do;
+      %let maxrows=&cellrows;
+      %let limitvar=DC_MAXCELLS_WEBEDIT;
+    %end;
+  %end;
+  %if %sysevalf(&maxrows lt 1) %then %let maxrows=1;
+  %global truncated trunc_msg;
+  %let truncated=0;
+  %if %sysevalf(&obscnt gt &maxrows) %then %let truncated=1;
+  %let trunc_msg=;
+  %if &truncated=1 %then %do;
+    %let trunc_msg=Only the first &maxrows rows are loaded;
+    %let trunc_msg=&trunc_msg - the &limitvar limit has been reached.;
+    %let trunc_msg=&trunc_msg  Filter the table to work with the rest.;
+  %end;
 
 
   /* order pk fields at start of table.  The delete flag is NOT added here:
@@ -297,9 +339,9 @@ select upcase(loadtype)
                             ,Str2=&pk
                           );
   %put sourcevars=&sourcevars;
-  /* the payload is capped here, at the row limit - the limits above are
-    checked against the whole selection, this is what goes to the browser */
-  options obs=&DC_MAXOBS_WEBEDIT;
+  /* the payload is capped at &maxrows - whichever limit that works out to be
+    - this is what goes to the browser */
+  options obs=&maxrows;
   data outdata;
     /* pk fields come first */
     %if %length(&pk)>0 %then %do;
@@ -513,13 +555,28 @@ select upcase(loadtype)
 /* extract col info */
 %mp_getcols(&libds, outds=cols1)
 
+/* derived variables are added to the view, so take their attributes from
+  there - this keeps type, length, format and label identical to the data */
+%if &derived_nobs>0 %then %do;
+  %mp_getcols(work.vw_derived, outds=derived_cols_all)
+  proc sql;
+  create table work.derived_cols as
+    select a.*
+    from work.derived_cols_all a
+    where a.NAME in (select NAME from work.derived_vars);
+  quit;
+  data work.cols1;
+    set work.cols1 work.derived_cols;
+  run;
+%end;
+
 /* join with cls rules.  Trim COLS payload in the SQL below - DDTYPE
   C=CHARACTER N=NUMERIC (dates kept in full), VARNUM dropped (client
   relies on column order, not position) */
 proc sql;
 create table work.cols as
   select a.NAME
-    ,coalesce(c.desc,a.NAME) as LABEL
+    ,coalesce(c.desc,a.LABEL,a.NAME) as LABEL
     ,a.FMTNAME
     ,case a.DDTYPE
       when 'CHARACTER' then 'C'
@@ -572,7 +629,7 @@ select count(*) into: ismap from &mpelib..mpe_xlmap_info
   where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
 
 data sasparams;
-  length colHeaders $20000 filter_text $32767;
+  length colHeaders $20000 filter_text $32767 trunc_msg $200;
   colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
   pkCnt=countw("&pk");
   pk="&pk";
@@ -583,6 +640,10 @@ data sasparams;
   if trim(symget('rk_underlying')) ne '' then rk_flag=1;
   else rk_flag=0;
   filter_text=symget('filter_text');
+  /* set when the selection was cut down to the row / cell limit, so the EDIT
+    screen can say so against the table name */
+  truncated=&truncated;
+  trunc_msg="&trunc_msg";
   if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
   else cls_flag=1;
   put (_all_)(=);

sas/sasjs/services/editors/getdata.test.1.sas

@@ -2,17 +2,19 @@
   @file
   @brief testing getdata row and cell limits
 
-  @details The EDIT screen refuses a selection that is too big on either
-  count - rows (DC_MAXOBS_WEBEDIT) or cells (DC_MAXCELLS_WEBEDIT).  The row
-  check runs first, so a selection over the row limit reports "Table is too
-  big" and one under the row limit but over the cell limit reports
-  "Selection is too wide".  Both aborts are proven by the child job state
-  (an aborted service registers no webout).
+  @details The EDIT screen is bounded by two limits - rows
+  (DC_MAXOBS_WEBEDIT) and cells, rows x columns (DC_MAXCELLS_WEBEDIT) -
+  whichever is reached first.  A selection over either is SERVED TRUNCATED:
+  the grid gets the first rows that fit, and sasparams carries TRUNCATED=1
+  plus a TRUNC_MSG naming the limit, which the EDIT screen shows as a warning
+  against the table name.
 
-  Fixture: dctest.wideboy (1000 rows x 1004 columns), created and registered
-  by testsetup.  Filtered to 300 rows it is over the 250 row limit; filtered
-  to 200 rows it is under the row limit but 200 x 1004 = 200,800 cells, over
-  the 200,000 cell limit.
+  Test 1 uses MPE_X_TEST (several hundred rows, few columns) so the row
+  limit alone decides the cap.  Test 2 uses the fixture dctest.wideboy (1000
+  rows x 1004 columns, created and registered by testsetup) filtered to 200
+  rows: that is under the row limit but 200 x 1004 = 200,800 cells, over the
+  200,000 cell limit, so the cell limit decides the cap -
+  floor(200000 / 1004) = 199 rows.
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
@@ -25,7 +27,7 @@
 %let _program=&appLoc/services/editors/getdata;
 
 /**
-  * build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
+  * Build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
   * in &filter_rk
   */
 %macro wideboy_filter(maxrow);
@@ -57,32 +59,56 @@
 %mend wideboy_filter;
 
 /**
-  * Test 1 - row limit (300 rows > 250)
+  * Call getdata as a child job and expose the webout as the &outlib JSON
+  * libname, so the test can read what the browser would have received.
   */
-%wideboy_filter(300)
-%let f1=%mf_getuniquefileref();
-data _null_;
-  file &f1 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%macro call_getdata(libds,filter_rk,outlib);
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref termstr=crlf;
+    put 'LIBDS:$43. FILTER_RK:8.';
+    put "&libds,&filter_rk";
+  run;
+  %mx_execute(&_program,
+    viyacontext=&defaultcontext,
+    inputfiles=&fref:sascontroltable,
+    outlib=&outlib
+  )
+  %let syscc=0;
+%mend call_getdata;
 
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f1:sascontroltable,
-  outref=web1
-)
+/**
+  * Test 1 - row limit.  MPE_X_TEST is well under the cell limit, so the
+  * 250 row default is what caps the payload.
+  */
+%call_getdata(&dclib..MPE_X_TEST,-1,web1)
 
-%let abort1=0;
+%let t1_nobs=-1;
+%let t1_trunc=-1;
+%let t1_msg=;
+proc sql noprint;
+select count(*) into: t1_nobs from web1.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
+  set web1.sasparams;
+  call symputx('t1_trunc',truncated);
+  call symputx('t1_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
-%let syscc=0;
 
 %mp_assert(
-  iftrue=(&abort1=1),
-  desc=Test 1 - a selection over the row limit is refused,
+  iftrue=(&t1_trunc=1),
+  desc=Test 1 - a selection over the row limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t1_nobs=250),
+  desc=Test 1 - the payload is capped at DC_MAXOBS_WEBEDIT (250) rows,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t1_msg,DC_MAXOBS_WEBEDIT)>0),
+  desc=Test 1 - the truncation message names the DC_MAXOBS_WEBEDIT limit,
   outds=work.test_results
 )
 
@@ -90,28 +116,34 @@ run;
   * Test 2 - cell limit (200 rows x 1004 cols = 200,800 > 200,000)
   */
 %wideboy_filter(200)
-%let f2=%mf_getuniquefileref();
-data _null_;
-  file &f2 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%call_getdata(dctest.wideboy,&filter_rk,web2)
 
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f2:sascontroltable,
-  outref=web2
-)
-
-%let abort2=0;
+%let t2_nobs=-1;
+%let t2_trunc=-1;
+%let t2_msg=;
+proc sql noprint;
+select count(*) into: t2_nobs from web2.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
+  set web2.sasparams;
+  call symputx('t2_trunc',truncated);
+  call symputx('t2_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
 
 %mp_assert(
-  iftrue=(&abort2=1),
-  desc=Test 2 - a selection over the cell limit is refused,
+  iftrue=(&t2_trunc=1),
+  desc=Test 2 - a selection over the cell limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t2_nobs=199),
+  desc=Test 2 - the payload is capped at floor(DC_MAXCELLS_WEBEDIT / cols),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t2_msg,DC_MAXCELLS_WEBEDIT)>0),
+  desc=Test 2 - the truncation message names the DC_MAXCELLS_WEBEDIT limit,
   outds=work.test_results
 )
 

sas/sasjs/services/editors/getdata.test.2.sas

@@ -21,10 +21,10 @@
   editing MPE_GROUPS alone - Row Level Security is skipped for them - so the
   test reports a skip row instead of running the assertions.
 
-  The call passes a filter (PRIMARY_KEY_FIELD < 100) because getdata.sas also
-  enforces the row / cell limits, and the unfiltered MPE_X_TEST is above the
-  row limit (642 rows against DC_MAXOBS_WEBEDIT:250) - the limits test
-  (getdata.test.1.sas) covers that abort separately.
+  The call passes a filter (PRIMARY_KEY_FIELD < 100) so the whole selection is
+  served - the unfiltered MPE_X_TEST is above the row limit (642 rows against
+  DC_MAXOBS_WEBEDIT:250) and would come back truncated.  The limits test
+  (getdata.test.1.sas) covers that truncation separately.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas

sas/sasjs/services/editors/getdata.test.sas

@@ -3,12 +3,14 @@
   @brief testing gethistory service
 
   <h4> SAS Macros </h4>
+  @li mp_assert.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
   @li mp_filterstore.sas
+  @li mpe_derived.sas
   @li mx_execute.sas
 
 **/
@@ -240,3 +242,179 @@ run;
   test=EQUALS 10,
   outds=work.test_results
 )
+
+
+/**
+  * Test 4 - derived variables
+  */
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_deps='LIBREF DSN'
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
+    ,active=1;
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE2'
+    ,var_deps='DERIVED_NOTE'
+    ,var_type='C'
+    ,var_len=45
+    ,var_fmt=''
+    ,var_logic="cats(DERIVED_NOTE,' (2)')"
+    ,active=1;
+quit;
+
+/* a filter may reference the derived variable, so validate against the view */
+%mpe_derived(libds=&dc_libref..MPE_TABLES,
+  dclib=&dc_libref,
+  outref=dercode4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.inquery4;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
+;;;;
+run;
+%mp_filterstore(
+  libds=&dc_libref..MPE_TABLES,
+  targetds=work.vw_derived4,
+  filter_summary=&dc_libref..mpe_filteranytable,
+  filter_detail=&dc_libref..mpe_filtersource,
+  lock_table=&dc_libref..mpe_lockanytable,
+  maxkeytable=&dc_libref..mpe_maxkeyvalues,
+  queryds=work.inquery4,
+  outresult=work.result4,
+  outquery=work.query4
+)
+data _null_;
+  set work.result4;
+  call symputx('filter_rk4',filter_rk);
+run;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "&dclib..MPE_TABLES,&filter_rk4";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:sascontroltable,
+  outlib=web4,
+  mdebug=&sasjs_mdebug
+)
+
+/* a cancelled child returns no webout, so the outlib keeps whatever an
+  earlier call left in it - check the job state before reading its members,
+  otherwise the assertion failures below describe the wrong problem */
+%let abort4=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort4',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort4=0),
+  desc=Test4 - the derived variable call completes,
+  outds=work.test_results
+)
+
+%if &abort4=0 %then %do;
+  data sasdata4;
+    set web4.sasdata;
+  run;
+  data cols4;
+    set web4.cols;
+  run;
+%end;
+%else %do;
+  data cols4;
+    length name $32;
+    stop;
+  run;
+%end;
+
+%mp_assertcols(work.cols4,
+  desc=Test4 - derived column is in the COLS payload,
+  cols=DERIVED_NOTE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk4a;
+  set work.cols4;
+  where name='DERIVED_NOTE' and cls_rule='READ';
+run;
+%mp_assertdsobs(work.chk4a,
+  desc=Test4 - derived column is read only,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4b;
+  set work.cols4;
+  where name='DERIVED_NOTE' and label=:'DERIVED_NOTE=';
+run;
+%mp_assertdsobs(work.chk4b,
+  desc=Test4 - derived column is labelled,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+%mp_assertdsobs(work.sasdata4,
+  desc=Test4 - filter on the derived column is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4c;
+  set work.sasdata4;
+  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCK_ANYTABLE');
+run;
+%mp_assertdsobs(work.chk4c,
+  desc=Test4 - derived value is correct,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* the second variable is built on the first, so this proves statement order */
+data work.chk4d;
+  set work.sasdata4;
+  if derived_note2=cats(derived_note,' (2)');
+run;
+%mp_assertdsobs(work.chk4d,
+  desc=Test4 - a derived variable can reference an earlier one,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* tidy up the config so repeated runs start clean.  Delete every row for the
+  table, not just the first variable: leaving DERIVED_NOTE2 behind means the
+  next run starts with a dependency on a variable that is no longer declared,
+  and every call aborts */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;

sas/sasjs/services/editors/stagedata.sas

@@ -22,6 +22,7 @@
   @li mf_getuniquefileref.sas
   @li mpe_loader.sas
   @li mpe_filtermaster.sas
+  @li mpe_derived.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_cntlout.sas
@@ -106,6 +107,37 @@ run;
 )
 
 %put Verify that the upload does not violate Row Level Security checks:;
+
+/* derived variables are display-only and must not be submitted, so drop them
+  from the data before the security check runs - otherwise a crafted
+  submission could satisfy an RLS clause that references a derived column */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outvars=work.derived_vars
+)
+%let derived_droplist=;
+%if %mf_nobs(work.derived_vars)>0 %then %do;
+  proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
+  run;
+  proc sql noprint;
+  select a.name into: derived_droplist separated by ' '
+    from work.derived_vars a
+    inner join work.jsdata_vars b
+    on a.name=b.name;
+  quit;
+%end;
+
+/* drop them from the submitted data itself - a DROP on the output of a later
+  step would leave the columns in that step's input, and the security clause
+  is evaluated against the input */
+%if %length(&derived_droplist)>0 %then %do;
+  data work.jsdata;
+    set work.jsdata;
+    drop &derived_droplist;
+  run;
+%end;
+
 %mpe_filtermaster(ULOAD,&libds,
   dclib=&mpelib,
   outref=filtref,
@@ -220,6 +252,9 @@ data _null_;
   put '%let ' name '=' value ';';
 run;
 
+/* derived variables are display-only - they never reach the approval package
+  or the target table.  work.jsdata was rewritten without them before the
+  security check, so there is nothing left to drop here */
 data approve.jsdset;
   length _____DELETE__THIS__RECORD_____ $3;
   set jsdata;

sas/sasjs/services/editors/stagedata.test.3.sas

@@ -132,6 +132,88 @@ quit;
   outds=work.test_results
 )
 
+/**
+  * Test 3 - an RLS rule naming a DERIVED variable must be rejected at save
+  * time: the rule is evaluated against the submitted data, where a derived
+  * column does not exist
+  */
+%let t3=%sysfunc(datetime());
+
+/* define a derived variable on the target table */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_REPLACE'
+    ,var_nm='DERIVED_X'
+    ,var_deps='X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='RLS rule on a derived variable should abort';
+  libds="&dc_libref..MPE_ROW_LEVEL_SECURITY";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length RLS_SCOPE $8 RLS_GROUP $128 RLS_LIBREF $8 RLS_TABLE $32
+    RLS_GROUP_LOGIC $3 RLS_SUBGROUP_LOGIC $3 RLS_VARIABLE_NM $32
+    RLS_OPERATOR_NM $12 RLS_RAW_VALUE $4000
+    _____DELETE__THIS__RECORD_____ $3;
+  RLS_SCOPE='EDIT';
+  RLS_GROUP='SASAdministrators';
+  RLS_LIBREF='DCTEST';
+  RLS_TABLE='DC_REPLACE';
+  RLS_GROUP_LOGIC='AND';
+  RLS_SUBGROUP_LOGIC='OR';
+  RLS_SUBGROUP_ID=0;
+  RLS_VARIABLE_NM='DERIVED_X';
+  RLS_OPERATOR_NM='NE';
+  RLS_RAW_VALUE='1';
+  RLS_ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort3=0;
+proc sql noprint;
+select count(*) into: abort3
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t3
+    and reason_txt="RLS rule on a derived variable should abort"
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort3>0),
+  desc=Checking an RLS rule on a derived variable is rejected,
+  outds=work.test_results
+)
+
+/* tidy up the derived configuration */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+quit;
+
 /**
   * Test 2 - applying REPLACE loadtype to a table with RLS EDIT security
   * should abort.  The active EDIT rule was inserted in the prep section,

sas/sasjs/services/editors/stagedata.test.4.sas

@@ -0,0 +1,413 @@
+/**
+  @file
+  @brief testing the derived-variable submit guards (MPE_DERIVED post edit hook)
+  @details Backend validations should abort when:
+    1. an active row's VAR_DEPS names an inactive (active=0) row of the same
+      table - the engine reads only active rows, so the dependency can never
+      resolve at run time
+    2. a row is staged with active=0 while a surviving active row depends on it
+
+  They should NOT abort when:
+    3. a new active row is added to a table that also holds an inactive row -
+      the inactive row is not part of the dependency graph, so it cannot
+      create a cycle or a stale reference
+    4. a delete-marked RLS rule naming a derived variable is submitted - the
+      rule is being removed, so its variables are not validated
+
+  The staged loads are aborted by the post edit hook inside the child service.
+  An aborted load leaves its MPE_LOADS row IN PROGRESS, so each assertion
+  matches on the unique reason_txt (the submitted message) rather than any load
+  in the time window - the suite runs in parallel sessions.
+
+  <h4> SAS Macros </h4>
+  @li mx_execute.sas
+  @li mp_assert.sas
+
+
+**/
+
+%let _program=&appLoc/services/editors/stagedata;
+
+/**
+  * Prep - a target table registration for the derived configuration, and the
+  * hook registration (some estates were initialised before the hook existed)
+  */
+proc sql noprint;
+update &dc_libref..mpe_tables
+  set post_edit_hook='services/hooks/mpe_derived_postedit'
+  where upcase(dsn)='MPE_DERIVED' and post_edit_hook='';
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+quit;
+
+/**
+  * Test 1 - a new active row depending on an inactive row must be rejected
+  */
+%let t1=%sysfunc(datetime());
+
+proc sql noprint;
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='INACTIVE_DEP'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=0;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Derived var depending on an inactive row should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
+    VAR_LOGIC $4000 VAR_FMT $32
+    _____DELETE__THIS__RECORD_____ $3;
+  LIBREF='DCTEST';
+  DSN='DC_DERIV';
+  VAR_NM='USES_INACTIVE';
+  VAR_DEPS='INACTIVE_DEP';
+  VAR_TYPE='N';
+  VAR_LEN=8;
+  VAR_FMT='';
+  VAR_LOGIC='INACTIVE_DEP + 1';
+  ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web1,
+  outref=wb1,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort1=0;
+proc sql noprint;
+select count(*) into: abort1
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t1
+    and reason_txt='Derived var depending on an inactive row should abort'
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort1>0),
+  desc=Checking a dependency on an inactive derived row is rejected,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - deactivating a row that a surviving active row depends on must be
+  * rejected: the control table still shows the row as active, so the check has
+  * to read the staged row too
+  */
+%let t2=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='BASE_X'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=1;
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='USES_BASE'
+    ,var_deps='BASE_X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='BASE_X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Deactivating a relied-on derived row should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dc_libref..mpe_derived(where=(upcase(libref)='DCTEST'
+    and upcase(var_nm)='BASE_X'));
+  length _____DELETE__THIS__RECORD_____ $3;
+  ACTIVE=0;
+  _____DELETE__THIS__RECORD_____='No';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web2,
+  outref=wb2,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort2=0;
+proc sql noprint;
+select count(*) into: abort2
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t2
+    and reason_txt='Deactivating a relied-on derived row should abort'
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort2>0),
+  desc=Checking deactivating a relied-on derived row is rejected,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - an inactive row must not block a submission that does not depend
+  * on it: with INACTIVE_DEP present (active=0) a new independent row loads
+  */
+%let t3=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='INACTIVE_DEP'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=0;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Active row next to an inactive row should load';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
+    VAR_LOGIC $4000 VAR_FMT $32
+    _____DELETE__THIS__RECORD_____ $3;
+  LIBREF='DCTEST';
+  DSN='DC_DERIV';
+  VAR_NM='FRESH';
+  VAR_DEPS='X';
+  VAR_TYPE='N';
+  VAR_LEN=8;
+  VAR_FMT='';
+  VAR_LOGIC='X + 1';
+  ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let status3=0;
+data _null_;
+  set web3.sasparams;
+  putlog (_all_)(=);
+  if status='SUCCESS' then call symputx('status3',1);
+run;
+
+%mp_assert(
+  iftrue=(&status3=1),
+  desc=Checking an inactive row does not block an unrelated derived variable,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - a delete-marked RLS rule naming a derived variable is a removal,
+  * so it must not abort: the rule is gone by the time the guards run
+  */
+%let t4=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='DERIVED_X'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=1;
+delete from &dc_libref..mpe_row_level_security
+  where upcase(rls_libref)='DCTEST' and upcase(rls_table)='DC_DERIV';
+select coalesce(max(rls_rk),0)+1 into: rlsrk
+  from &dc_libref..mpe_row_level_security;
+insert into &dc_libref..mpe_row_level_security
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,rls_rk=&rlsrk
+    ,rls_scope='EDIT'
+    ,rls_group='SASAdministrators'
+    ,rls_libref='DCTEST'
+    ,rls_table='DC_DERIV'
+    ,rls_group_logic='AND'
+    ,rls_subgroup_logic='OR'
+    ,rls_subgroup_id=0
+    ,rls_variable_nm='DERIVED_X'
+    ,rls_operator_nm='NE'
+    ,rls_raw_value='1'
+    ,rls_active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Delete-marked RLS rule on a derived variable should load';
+  libds="&dc_libref..MPE_ROW_LEVEL_SECURITY";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dc_libref..mpe_row_level_security(where=(upcase(rls_libref)='DCTEST'
+    and upcase(rls_table)='DC_DERIV'));
+  length _____DELETE__THIS__RECORD_____ $3;
+  _____DELETE__THIS__RECORD_____='Yes';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web4,
+  outref=wb4,
+  mdebug=&sasjs_mdebug
+)
+
+%let status4=0;
+data _null_;
+  set web4.sasparams;
+  putlog (_all_)(=);
+  if status='SUCCESS' then call symputx('status4',1);
+run;
+
+%mp_assert(
+  iftrue=(&status4=1),
+  desc=Checking a delete-marked RLS rule on a derived variable is not rejected,
+  outds=work.test_results
+)
+
+/**
+  * Test 5 - deleting a derived row that a surviving active row depends on must
+  * be rejected: the delete is not in the control table yet and the staged copy
+  * is delete-marked, so the inactive set has to admit it
+  */
+%let t5=%sysfunc(datetime());
+
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='BASE_X'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='1'
+    ,active=1;
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_DERIV'
+    ,var_nm='USES_BASE'
+    ,var_deps='BASE_X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='BASE_X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='Deleting a relied-on derived row should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dc_libref..mpe_derived(where=(upcase(libref)='DCTEST'
+    and upcase(var_nm)='BASE_X'));
+  length _____DELETE__THIS__RECORD_____ $3;
+  _____DELETE__THIS__RECORD_____='Yes';
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web5,
+  outref=wb5,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort5=0;
+proc sql noprint;
+select count(*) into: abort5
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t5
+    and reason_txt='Deleting a relied-on derived row should abort'
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort5>0),
+  desc=Checking deleting a relied-on derived row is rejected,
+  outds=work.test_results
+)
+
+/* tidy up so repeated runs start clean */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_DERIV';
+delete from &dc_libref..mpe_row_level_security
+  where upcase(rls_libref)='DCTEST' and upcase(rls_table)='DC_DERIV';
+quit;

sas/sasjs/services/hooks/mpe_derived_postedit.sas

@@ -0,0 +1,433 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_DERIVED table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+  This hook normalises the incoming rows and applies the configuration rules
+  at submit time, so a bad derived variable is rejected while the submitter is
+  still watching - rather than breaking the EDIT screen later, when the view
+  is built.
+
+  VAR_DEPS must list every input the expression uses, physical columns
+  and derived variables alike.  The hook checks the part that does not need
+  the target library: a derived variable referenced by an expression must be
+  declared, a variable may not depend on itself, and the resulting graph must
+  be acyclic - across the incoming rows and the current rows they do not
+  replace.
+
+  The target table is not assigned in this context, so the collision between
+  VAR_NM and a physical column, and the resolution of a physical dependency,
+  are left to %mpe_derived, which runs with the target library assigned.
+  Uniqueness on the business key (LIBREF DSN VAR_NM) is checked by mpe_loader
+  before this hook runs, but on the raw submitted values, so it is case
+  sensitive: 'Region' and 'REGION' for the same table both pass the loader and
+  collide only at approve time.  The post-normalisation check below rejects
+  the second one at submit time, naming the variable.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mpe_derived.sas
+  @li mpe_loader.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let der_errflag=0;
+%let der_errmsg=;
+
+/* a hook is included at execution time, so this is open code - the macro
+  language has to be inside a macro definition for %local to be valid. The
+  control variables are set here, in the including scope (which %mp_include
+  makes local): a %global inside the macro would conflict with them and abort
+  the step. */
+%macro _mpe_derived_postedit();
+  %local left nready cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
+    cyc_n cyc_found;
+
+/* normalise the incoming values */
+data work.staging_ds;
+  set work.staging_ds;
+  if upcase(_____DELETE__THIS__RECORD_____) ne 'YES' then do;
+    libref=upcase(strip(libref));
+    dsn=upcase(strip(dsn));
+    var_nm=upcase(strip(var_nm));
+    var_type=upcase(strip(var_type));
+    var_fmt=upcase(strip(var_fmt));
+    var_label=strip(var_label);
+    var_deps=upcase(strip(var_deps));
+    var_logic=strip(var_logic);
+  end;
+run;
+
+/**
+  * Duplicate names after normalisation - mpe_loader compares the raw values,
+  * so a case variant reaches us here
+  */
+proc sql;
+create table work.derived_dup as
+  select libref, dsn, var_nm, count(*) as n
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  group by libref, dsn, var_nm
+  having count(*)>1;
+quit;
+
+data _null_;
+  set work.derived_dup(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',var_nm,' is declared more than once for '
+      ,libref,'.',dsn));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_dup))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/**
+  * Field level checks - the same rules %mpe_derived applies at read time
+  */
+data work.derived_check;
+  set work.staging_ds;
+  length reason $200;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  call missing(reason);
+
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if var_nm='' then reason='VAR_NM is required';
+  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VAR_NM must be a valid SAS name';
+  else if verify(substr(var_nm,1,1),'0123456789')=0
+    then reason='VAR_NM must not start with a digit';
+  else if var_nm='_____DELETE__THIS__RECORD_____'
+    then reason='VAR_NM is reserved';
+  else if var_type not in ('C','N')
+    then reason='VAR_TYPE must be C or N';
+  else if missing(var_len) or var_len ne int(var_len) or var_len<1
+    then reason='VAR_LEN must be a positive integer';
+  else if var_type='C' and var_len>32767
+    then reason='VAR_LEN exceeds 32767 for a character variable';
+  else if var_type='N' and var_len>8
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
+  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
+    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
+    then reason='VAR_FMT must be a format name, without ; & % or //*';
+  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VAR_FMT must be a format name';
+  else if missing(var_logic)
+    then reason='VAR_LOGIC is required';
+  else if index(var_logic,';')>0
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
+  else if indexc(var_logic,'&%')>0
+    then reason='VAR_LOGIC must not contain macro triggers';
+  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
+    then reason='VAR_LOGIC must not contain comments';
+
+  if not missing(reason) then output;
+run;
+
+%if %mf_nobs(work.derived_check)>0 %then %do;
+  data _null_;
+    set work.derived_check(obs=1);
+    call symputx('der_errmsg',cats('MPE_DERIVED: ',var_nm,': ',reason));
+    call symputx('der_errflag',1);
+  run;
+%end;
+
+%mp_abort(iftrue= (&der_errflag=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/**
+  * The graph has to hold for the resulting configuration: the incoming rows
+  * plus the current rows of the same tables that this submission does not
+  * replace. Only active rows are part of it - the engine reads nothing else
+  * (mpe_derived.sas), so an inactive row can neither resolve a dependency nor
+  * be resolved by one.
+  */
+proc sql;
+create table work.derived_rows as
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_deps)) as deps
+    ,strip(var_logic) as logic
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+    and active=1
+  union all
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_deps)) as deps
+    ,strip(var_logic) as logic
+  from &DC_LIBREF..mpe_derived
+  where &dc_dttmtfmt. lt tx_to
+    and active=1
+    and catx('.',libref,dsn,var_nm) not in
+      (select catx('.',libref,dsn,var_nm) from work.staging_ds)
+    and catx('.',libref,dsn) in
+      (select catx('.',libref,dsn) from work.staging_ds);
+quit;
+
+data work.derived_dep;
+  set work.derived_rows;
+  length dep_nm $32;
+  do i=1 to countw(deps,' ');
+    dep_nm=scan(deps,i,' ');
+    if not missing(dep_nm) then output;
+  end;
+  keep libref dsn name dep_nm;
+run;
+
+/* a dependency on a row the engine will not read cannot resolve: it aborts
+  every read of the table with "VAR_DEPS lists X, which is not a column of the
+  table or a derived variable". Catch it here, covering a row this submission
+  deactivates or deletes - neither shows in the control table yet. */
+proc sql;
+create table work.derived_baddep as
+  select d.libref, d.dsn, d.name, d.dep_nm
+  from work.derived_dep d
+  inner join (
+    select upcase(strip(libref)) as libref
+      ,upcase(strip(dsn)) as dsn
+      ,upcase(strip(var_nm)) as name
+    from work.staging_ds
+    where upcase(_____DELETE__THIS__RECORD_____) = 'YES'
+      or active ne 1
+    union all
+    select upcase(strip(libref)) as libref
+      ,upcase(strip(dsn)) as dsn
+      ,upcase(strip(var_nm)) as name
+    from &DC_LIBREF..mpe_derived
+    where &dc_dttmtfmt. lt tx_to
+      and active ne 1
+  ) inactive
+    on inactive.libref=d.libref
+    and inactive.dsn=d.dsn
+    and inactive.name=d.dep_nm;
+quit;
+
+data _null_;
+  set work.derived_baddep(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS lists ',dep_nm
+      ,', which is not active after this load - the engine cannot resolve it'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_baddep))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+data work.derived_ref;
+  set work.derived_rows;
+  length l $4000 tok $32;
+  l=prxchange("s/'([^']|'')*'//",-1,logic);
+  l=prxchange('s/"([^"]|"")*"//',-1,l);
+  l=prxchange('s/[^A-Za-z0-9_]/ /',-1,upcase(l));
+  do i=1 to countw(l,' ');
+    tok=scan(l,i,' ');
+    if not missing(tok) then output;
+  end;
+  keep libref dsn name tok;
+run;
+
+proc sql;
+create table work.derived_self as
+  select libref, dsn, name, dep_nm
+  from work.derived_dep
+  where name=dep_nm;
+
+create table work.derived_undeclared as
+  select r.libref, r.dsn, r.name, r.tok as dep_nm
+  from work.derived_ref r
+  where exists (
+      select 1 from work.derived_rows n
+      where n.libref=r.libref and n.dsn=r.dsn and n.name=r.tok
+    )
+    and r.tok ne r.name
+    and not exists (
+      select 1 from work.derived_dep d
+      where d.libref=r.libref and d.dsn=r.dsn and d.name=r.name
+        and d.dep_nm=r.tok
+    );
+quit;
+
+/* the message is always prepared, the abort is driven by the count - an
+  iftrue=1 inside a wrapping %if leaves the step alive on some engines */
+data _null_;
+  set work.derived_self(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_self))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+data _null_;
+  set work.derived_undeclared(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
+      ,', which is not in VAR_DEPS'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_undeclared))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/* the inverse of the check above: a declared name the expression never uses
+  is stale, and the read-time engine rejects it too */
+proc sql;
+create table work.derived_stale as
+  select d.libref, d.dsn, d.name, d.dep_nm
+  from work.derived_dep d
+  where not exists (
+    select 1 from work.derived_ref r
+    where r.libref=d.libref and r.dsn=d.dsn and r.name=d.name
+      and r.tok=d.dep_nm
+  );
+quit;
+
+data _null_;
+  set work.derived_stale(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS lists ',dep_nm
+      ,', which VAR_LOGIC does not reference'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_stale))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/* the same invariant the RLS hook enforces from its side: a derived variable
+  cannot be named by an active row level security rule of any scope. Only the
+  VIEW scope could evaluate it (on the view the EDIT screen reads), but the RLS
+  hook validates every rule against the physical table, where the column does
+  not exist, so the pairing is refused up front rather than failing there. */
+proc sql;
+create table work.derived_rls as
+  select d.libref, d.dsn, d.name
+  from work.derived_rows d
+  inner join &DC_LIBREF..mpe_row_level_security r
+    on upcase(r.rls_libref)=upcase(d.libref)
+    and upcase(r.rls_table)=upcase(d.dsn)
+    and upcase(r.rls_variable_nm)=upcase(d.name)
+  where r.rls_active=1
+    and &dc_dttmtfmt lt r.tx_to;
+quit;
+
+data _null_;
+  set work.derived_rls(obs=1);
+  call symputx('der_errmsg'
+    ,cats('MPE_DERIVED: ',name,' is named by a row level security rule on '
+      ,libref,'.',dsn,' - derived variables cannot be used there'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_rls))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(der_errmsg)
+)
+
+/**
+  * Cycle check - repeatedly take the variables whose derived inputs have all
+  * been placed; a round that places nothing leaves a cycle
+  */
+proc sql;
+create table work.derived_edge as
+  select d.libref, d.dsn, d.name as child, d.dep_nm as parent
+  from work.derived_dep d
+  where d.dep_nm in (select name from work.derived_rows);
+quit;
+
+data work.derived_todo;
+  set work.derived_rows(keep=libref dsn name);
+run;
+
+%let left=%mf_nobs(work.derived_todo);
+%let cyc_found=0;
+
+%do %while (&left>0);
+  proc sql;
+  create table work.derived_ready as
+    select t.libref, t.dsn, t.name
+    from work.derived_todo t
+    where not exists (
+      select 1 from work.derived_edge e
+      inner join work.derived_todo t2
+        on e.libref=t2.libref and e.dsn=t2.dsn and e.parent=t2.name
+      where e.libref=t.libref and e.dsn=t.dsn and e.child=t.name
+    );
+  quit;
+
+  %let nready=%mf_nobs(work.derived_ready);
+  %if &nready=0 %then %do;
+    /* walk the edges, so the message names the cycle rather than just the
+      names that could not be placed */
+    %let cyc_n=%mf_nobs(work.derived_todo);
+    data _null_;
+      set work.derived_todo(obs=1);
+      call symputx('cyc_start',name);
+    run;
+    %let cyc_path=&cyc_start;
+    %let cyc_cur=&cyc_start;
+    %let cyc_seen=%str(.&cyc_start..);
+    %do cyc_i=1 %to &cyc_n;
+      %let cyc_next=;
+      proc sql noprint;
+        select e.parent into: cyc_next
+        from work.derived_edge e
+        where e.child="&cyc_cur"
+          and catx('.',e.libref,e.dsn,e.parent) in
+            (select catx('.',libref,dsn,name) from work.derived_todo);
+      quit;
+      %if %length(&cyc_next)=0 %then %let cyc_i=&cyc_n;
+      %else %do;
+        %let cyc_path=&cyc_path. -> &cyc_next;
+        %if %index(&cyc_seen,%str(.&cyc_next..))>0 %then %let cyc_i=&cyc_n;
+        %else %do;
+          %let cyc_seen=&cyc_seen.&cyc_next..;
+          %let cyc_cur=&cyc_next;
+        %end;
+      %end;
+    %end;
+    /* leave the loop before aborting - an abort that only cancels the step
+      would re-enter it with the ready count still zero */
+    %let cyc_found=1;
+    %let left=0;
+  %end;
+
+  %if &cyc_found=0 %then %do;
+    proc sql;
+    delete from work.derived_todo
+      where catx('.',libref,dsn,name) in
+        (select catx('.',libref,dsn,name) from work.derived_ready);
+    quit;
+
+    %let left=%mf_nobs(work.derived_todo);
+  %end;
+%end;
+
+%mp_abort(iftrue= (&cyc_found=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
+)
+
+%mend _mpe_derived_postedit;
+
+%_mpe_derived_postedit()

sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas

@@ -18,6 +18,7 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_filtercheck.sas
 
@@ -54,7 +55,7 @@ run;
   ) change the loadtype.)
 )
 
-/* ignore scope and group for validation */
+/* group and subgroup are not used for validation */
 proc sql;
 create table work.batches as
   select distinct upcase(rls_libref) as rls_libref,
@@ -67,6 +68,7 @@ create table work.batches as
     rls_raw_value as raw_value
   from work.staging_ds
   where rls_active=1
+    and upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
   order by rls_libref, rls_table;
 
 %let cnt=0;
@@ -81,6 +83,37 @@ data _null_;
   end;
 run;
 
+/* derived variables exist only in the view the EDIT screen reads - a rule on
+  one cannot be validated against the physical table, so reject it here rather
+  than let mp_filtercheck fail with a variable-not-found error. Applies to
+  every scope: the VIEW scope could evaluate the column, but this hook checks
+  rules against the table they are submitted for. */
+proc sql;
+create table work.rls_derived as
+  select b.rls_libref, b.rls_table, b.variable_nm
+  from work.batches b
+  inner join &DC_LIBREF..mpe_derived d
+    on upcase(d.libref)=b.rls_libref
+    and upcase(d.dsn)=b.rls_table
+    and upcase(d.var_nm)=upcase(b.variable_nm)
+  where &dc_dttmtfmt lt d.tx_to
+    and d.active=1;
+quit;
+
+%let rls_msg=;
+data _null_;
+  set work.rls_derived(obs=1);
+  call symputx('rls_msg'
+    ,cats('MPE_ROW_LEVEL_SECURITY: ',variable_nm
+      ,' is a derived variable in ',rls_libref,'.',rls_table
+      ,' - derived variables cannot be used in a row level security rule'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.rls_derived))
+  ,mac=mpe_row_level_security_postedit
+  ,msg=%superq(rls_msg)
+)
+
 %macro quickloop();
 %do i=1 %to &cnt;
   data work.inds&i;

sas/sasjs/services/public/getcolvals.sas

@@ -35,6 +35,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_nobs.sas
   @li mf_verifymacvars.sas
   @li dc_assignlib.sas
   @li mf_getvarformat.sas
@@ -42,6 +43,7 @@
   @li mp_cntlout.sas
   @li mp_filtercheck.sas
   @li mp_filtergenerate.sas
+  @li mpe_derived.sas
   @li mp_validatecol.sas
   @li mpe_validatecol.sas
 
@@ -150,10 +152,23 @@ run;
 )
 
 
+/**
+  * Derived (computed) variables - these exist only in a view, so the filter
+  * must be validated and applied against that view
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&libds));
+
 /**
   * Validate the filter query
   */
-%mp_filtercheck(work.filterquery,targetds=&libds,abort=YES)
+%mp_filtercheck(work.filterquery,targetds=&filterds,abort=YES)
 
 /**
   * Prepare the query
@@ -162,14 +177,14 @@ run;
 
 /* cannot %inc in a sql where clause, only data step, so - use a view */
 data work.vw_vals/view=work.vw_vals;
-  set &libds;
+  set &filterds;
   where %inc myfilter;;
 run;
 
 proc sql;
 create view work.vw_vals_sorted as
   select distinct
-      put(&col2,%mf_getVarFormat(&libds,&col2,force=1)) as formatted,
+      put(&col2,%mf_getVarFormat(work.vw_vals,&col2,force=1)) as formatted,
       &col2 as unformatted
   from work.vw_vals;
 
@@ -194,7 +209,7 @@ proc sql noprint;
 select count(*) into: nobs from work.vw_vals_sorted;
 data meta;
   column="&col2";
-  sasformat="%mf_getVarFormat(&libds,&col2)";
+  sasformat="%mf_getVarFormat(work.vw_vals,&col2)";
   startrow=&startrow;
   rows=&rows;
   nobs=&nobs;

sas/sasjs/services/public/getcolvals.test.1.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
   @li mx_execute.sas
   @li mf_getuniquefileref.sas
 
@@ -36,3 +37,54 @@ run;
   desc=DCLIB found in unfiltered getgolvals response,
   test=ANYVAL
 )
+
+/**
+  * derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
+    ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
+    ,var_nm='DERIVED_KEY'
+    ,var_deps='PRIMARY_KEY_FIELD'
+    ,var_type='C'
+    ,var_len=20
+    ,var_fmt=''
+    ,var_logic='cats(primary_key_field)'
+    ,active=1;
+quit;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. COL:$32.';
+  put "&dclib..MPE_X_TEST,DERIVED_KEY";
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:iwant,
+  outlib=web4
+)
+
+data work.vals4;
+  set web4.vals;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.vals4,
+  desc=Values are returned for a derived column,
+  test=HASOBS
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where var_nm='DERIVED_KEY'
+    and upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+quit;

sas/sasjs/services/public/getrawdata.sas

@@ -21,6 +21,8 @@
   @li mp_ds2csv.sas
   @li mp_streamfile.sas
   @li mp_validatecol.sas
+  @li mf_nobs.sas
+  @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li mpe_filtermaster.sas
 
@@ -143,10 +145,32 @@ data _null_;
   putlog _infile_;
 run;
 
+/* derived variables exist only in a view, so a filter that references one
+  must be applied against it.  They are dropped from the download - they are
+  an EDIT screen feature. */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let derived_droplist=;
+%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&libds));
+%if &derived_nobs>0 %then %do;
+  proc sql noprint;
+  select name into: derived_droplist separated by ' '
+    from work.derived_vars;
+  quit;
+%end;
+
 options obs=200000;/* stop limit */
 data staged(drop=&txfrom &txto);
-  set &libds;
+  set &filterds;
   where %inc filtref;;
+  %if %length(&derived_droplist)>0 %then %do;
+    drop &derived_droplist;
+  %end;
 run;
 options obs=max;
 

sas/sasjs/services/public/validatefilter.sas

@@ -26,8 +26,10 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_filterstore.sas
+  @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li removecolsfromwork.sas
 
@@ -69,8 +71,19 @@ run;
 
 %dc_assignlib(WRITE,%scan(&ds,1,.))
 
+/* derived variables exist only in a view, so validate against that */
+%mpe_derived(libds=&ds,
+  dclib=&dc_libref,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&ds));
+
 %mp_filterstore(
   libds=&ds,
+  targetds=&filterds,
   queryds=work.filterquery,
   filter_summary=&dc_libref..mpe_filteranytable,
   filter_detail=&dc_libref..mpe_filtersource,

sas/sasjs/services/public/validatefilter.test.sas

@@ -48,6 +48,8 @@ run;
 
 data work.iwant;
   filter_table='DCTEST.DCFMTS-FC';
+  /* mx_execute reads the program to run from the input dataset */
+  _program="&_program";
 run;
 data work.filterquery;
 GROUP_LOGIC='AND';
@@ -77,3 +79,66 @@ run;
   desc=Test2 result filter,
   test=EQUALS 1
 )
+
+/**
+  * Test 3 - filter on a derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_deps='LIBREF DSN'
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
+    ,active=1;
+quit;
+
+data work.iwant;
+  filter_table="&dclib..MPE_TABLES";
+run;
+data work.filterquery;
+  GROUP_LOGIC='AND';
+  SUBGROUP_LOGIC='AND';
+  SUBGROUP_ID=1;
+  VARIABLE_NM='DERIVED_NOTE';
+  OPERATOR_NM='CONTAINS';
+  RAW_VALUE="'MPE_LOCK'";
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.iwant work.filterquery,
+  outlib=web3,
+  outref=web3ref
+)
+data _null_;
+  infile web3ref;
+  input;
+  putlog _infile_;
+run;
+data result;
+  set web3.result;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.result,
+  desc=Test3 derived filter,
+  test=EQUALS 1
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where var_nm='DERIVED_NOTE'
+    and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;

sas/sasjs/services/public/viewdata.sas

@@ -59,6 +59,7 @@
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getversions.sas
   @li mpe_filtermaster.sas
@@ -218,6 +219,25 @@ run;
     putlog _infile_;
   run;
 
+  /* derived variables exist only in a view, so a filter that references one
+    must be applied against it.  The columns themselves are dropped from the
+    output - they are an EDIT screen feature. */
+  %mpe_derived(libds=&libds,
+    dclib=&mpelib,
+    outref=dercode,
+    outview=work.vw_derived,
+    outvars=work.derived_vars
+  )
+  %let derived_nobs=%mf_nobs(work.derived_vars);
+  %let derived_droplist=;
+  %let filterds=%sysfunc(ifc(&derived_nobs>0,work.vw_derived,&libds));
+  %if &derived_nobs>0 %then %do;
+    proc sql noprint;
+    select name into: derived_droplist separated by ' '
+      from work.derived_vars;
+    quit;
+  %end;
+
   %if &searchtype=NONE or "%trim(&searchtype) " = " " %then %do;
     /* get row count */
     filename rows temp;
@@ -227,7 +247,7 @@ run;
       input;
       if _n_=1 then do;
         put 'proc sql;';
-        put "select count(*) into: dsobs from &libds where";
+        put "select count(*) into: dsobs from &filterds where";
       end;
       put _infile_;
       if eof then put ';';
@@ -242,8 +262,11 @@ run;
 
     /* send actual data, filtered and row-capped */
     data work.viewdata;
-      set &libds;
+      set &filterds;
       where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
       if _n_>&DC_MAXOBS_WEBVIEW then stop;
     run;
     %if %mf_nobs(work.viewdata)=0 %then %do;
@@ -256,8 +279,11 @@ run;
   %end;
   %else %do;
     data work.vwsearch/view=work.vwsearch;
-      set &libds;
+      set &filterds;
       where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
     run;
     %if %upcase(&searchtype)=CHAR %then %do;
       %mp_searchdata(lib=work
@@ -275,6 +301,8 @@ run;
     %end;
     %if %mf_existds(libds=MPSEARCH.vwsearch) %then %do;
       %let dsobs=%mf_nobs(MPSEARCH.vwsearch);
+      /* the derived columns were dropped from work.vwsearch above, so
+        mp_searchdata's output cannot carry them */
       data viewdata;
         set MPSEARCH.vwsearch;
         if _n_<=&DC_MAXOBS_WEBVIEW;
@@ -283,7 +311,7 @@ run;
     %else %do;
       %let dsobs=0;
       data viewdata;
-        set &libds;
+        set &filterds;
         stop;
       run;
     %end;