chore: merge feat/derived-vars into the startupservice launcher branch

.gitea/workflows/build.yaml

@@ -198,7 +198,7 @@ jobs:
           # Start frontend and run cypress
           # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts,cypress/e2e/startupservice-launcher.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

client/cypress/e2e/derived-variables.cy.ts

@@ -0,0 +1,162 @@
+export {}
+//
+// Derived (computed) variables are configured per table in MPE_DERIVED and
+// created at read time, ahead of the filter, so that they can be filtered on.
+//
+// TESTDATA.DERIVED_DEMO carries two of them, neither of which is a physical
+// column of the table:
+//   REGION      ifc(STATE in ('CA','OR','WA'),'West','East')
+//   LINE_VALUE  QTY * PRICE
+//
+// The mock implements the derivation in getdata.js / getcolvals.js (see
+// mpeDerived in dcMockUtils.js), so the EDIT screen can be exercised without a
+// SAS server. The real evaluation is done by SAS.
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('derived variables: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.viewport(1440, 900)
+    bootApp()
+  })
+
+  it('1 | derived columns are shown in the EDIT grid with computed values', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    getCellByHeaderAndRow(0, 'REGION').should('have.text', 'West')
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'East')
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '21')
+    getCellByHeaderAndRow(1, 'LINE_VALUE').should('have.text', '99')
+    // REGION_LABEL is built on REGION, so it only resolves if the derived
+    // statements run in VARIABLE_ORDER sequence
+    getCellByHeaderAndRow(0, 'REGION_LABEL').should('have.text', 'West-CA')
+    getCellByHeaderAndRow(1, 'REGION_LABEL').should('have.text', 'East-NY')
+
+    cy.screenshot('derived-1-grid', { capture: 'viewport' })
+  })
+
+  it('2 | filters on a character derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('REGION')
+    setFilterValue('West')
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', "REGION = 'West'")
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 2)
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'West')
+
+    cy.screenshot('derived-2-filter-region', { capture: 'viewport' })
+  })
+
+  it('3 | filters on a numeric derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('LINE_VALUE')
+    setFilterOperator('>')
+    setFilterValue('50')
+    submitFilter()
+
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 1)
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '99')
+
+    cy.screenshot('derived-3-filter-value', { capture: 'viewport' })
+  })
+})
+
+/**
+ * Cypress clears cookies between tests, which drops the SASjs Server session
+ * and lands the app back on the evaluation agreement card. Accept it when it is
+ * shown - a no-op when the session survived.
+ */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: longerCommandTimeout })
+  cy.get('body').then(($body: any) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Opens a table straight in the EDIT screen and waits for the grid. */
+const openEditor = (libMem: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/editor/${libMem}`, {
+    timeout: longerCommandTimeout
+  })
+  cy.get('.ht_clone_top .htCore thead tr th', {
+    timeout: longerCommandTimeout
+  }).should('exist')
+  cy.wait(2500)
+}
+
+const openFilterPopup = () => {
+  cy.contains('button.btnView', 'Filter', {
+    timeout: longerCommandTimeout
+  }).click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/**
+ * The soft-select inputs drop a transparent .overlay click-catcher over the
+ * modal while their suggestion list is open, so they need force.
+ */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterOperator = (operator: string) => {
+  cy.get('.filter-modal .operator-col select').first().select(operator)
+  cy.wait(500)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+const getCellByHeaderAndRow = (rowIndex: number, headerText: string) => {
+  return cy
+    .get('.ht_clone_top .htCore thead tr th')
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => (th as HTMLElement).innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => (th as HTMLElement).innerText.trim() === headerText
+      )
+
+      return cy
+        .get('.ht_master tbody tr')
+        .then((rows: any) => rows[rowIndex].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}

client/cypress/e2e/excel.cy.ts

@@ -399,6 +399,26 @@ context('excel tests: ', function () {
     })
   })
 
+  it('25 | Keeps decimals when the upload preview becomes a manual edit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    // Entering manual edit mode discards the source file but keeps the values
+    // parsed from it. SOME_NUM holds 0.901045951946194 in the first data row
+    // of the fixture, so a truncated or zeroed cell fails the assertion.
+    attachExcelFile('regular_excel.xlsx', () => {
+      getFirstRowCells().then((previewCells) => {
+        expect(previewCells).to.include(decimalSomeNum)
+
+        cy.get('.btn-upload-preview.btn-primary').click()
+        cy.get('.modal-footer .btn-success-outline').click()
+
+        getFirstRowCells().then((editCells) => {
+          expect(editCells).to.include(decimalSomeNum)
+        })
+      })
+    })
+  })
+
   // Large files break Cypress
 
   // it ('? | Uploads Excel with size of 5MB', (done) => {
@@ -476,6 +496,19 @@ const attachExcelFile = (excelFilename: string, callback?: any) => {
     })
 }
 
+const decimalSomeNum = '0.901045951946194'
+
+// The row's cell texts, without depending on column positions: the preview
+// grid renders row-status and delete cells in an order that varies.
+const getFirstRowCells = () =>
+  cy
+    .get('#hotTable', { timeout: longerCommandTimeout })
+    .find('div.ht_master.handsontable tbody tr')
+    .eq(0)
+    .then(($row) =>
+      Array.from($row[0].children).map((cell: any) => cell.innerText.trim())
+    )
+
 const submitExcel = (callback?: any) => {
   cy.get('.buttonBar button.preview-submit', { timeout: longerCommandTimeout })
     .click()

client/cypress/e2e/row-cell-limits.cy.ts

@@ -1,13 +1,15 @@
-// The EDIT screen refuses a selection that is too large - DC_MAXOBS_WEBEDIT
-// rows, or DC_MAXCELLS_WEBEDIT cells (rows x columns), whichever is reached
-// first - and the VIEW screen caps at DC_MAXOBS_WEBVIEW rows.
+// The EDIT screen is bounded by DC_MAXOBS_WEBEDIT rows and DC_MAXCELLS_WEBEDIT
+// cells (rows x columns), whichever is reached first, and the VIEW screen caps
+// at DC_MAXOBS_WEBVIEW rows.
 //
+// A selection over an EDIT limit is served TRUNCATED - the grid carries the
+// first rows that fit, and a warning triangle against the table name says why.
 // This spec pins those limits against the mock, which mirrors the shipped
 // defaults (250 rows / 200000 cells for EDIT, 2000 rows for VIEW).  It uses
 // two existing fixtures rather than new ones:
 //   MPE_X_SEARCH - 1000 rows x 9 cols, so it trips the ROW limit
-//   MPE_X_WIDE   - 200 rows x 1001 cols (200,400 cells), under the row limit
-//                  but over the CELL limit
+//   MPE_X_WIDE   - 200 rows x 1001 cols (200,200 cells), under the row limit
+//                  but over the CELL limit, so the row cap is 199
 export {}
 
 const hostUrl = Cypress.env('hosturl')
@@ -60,27 +62,51 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
     })
   })
 
-  it('2 | EDIT refuses a table over the row limit', () => {
+  it('2 | EDIT serves a table over the row limit, truncated, with a warning', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
     openEdit()
 
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 1000 rows x 9 columns is over the 250 row limit, so the grid carries the
+    // first 250 rows and the header says how many it is showing.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
+      '250 rows'
     )
+    cy.get('#hotTable .ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length.at.least', 1)
+
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXOBS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
   })
 
-  it('3 | EDIT refuses a table over the cell limit', () => {
+  it('3 | EDIT serves a table over the cell limit, truncated, with a warning', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_wide')
     openEdit()
 
-    // 200 rows x 1002 columns (the 1001 stored columns plus the housekeeping
-    // delete column) = 200,400 cells, over the 200,000 cell limit, while the
-    // row count is under the 250 row limit.
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 200 rows x 1001 stored columns = 200,200 cells, over the 200,000 cell
+    // limit, while the row count is under the 250 row limit - so the cell
+    // limit sets the cap: floor(200000 / 1001) = 199 rows.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      'Selection is too wide (200 rows x 1002 cols) - please filter and try again!'
+      '199 rows'
     )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
   })
 
   it('4 | the EDIT limits follow MPE_CONFIG, not the shipped defaults', () => {
@@ -90,20 +116,29 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
     // all.  And MPE_CONFIG lives in the control library, so reading it from the
     // table's own library would fail for this table in particular.
     //
-    // The cell limit is the one to move: 7 rows x 18 columns is 126 cells, so
-    // at 110 the TESTDATA.MPE_X_NEW edit must be refused, where a hard-coded
-    // 200000 - or a config read that fails and falls back to it - lets it
-    // through.  110 is still above MPE_CONFIG's own 17 x 6 = 102, so the option
-    // can be put back afterwards.
+    // The cell limit is the one to move: 7 rows x 17 columns is 119 cells, so
+    // at 110 the TESTDATA.MPE_X_NEW edit must be truncated to
+    // floor(110 / 17) = 6 rows, where a hard-coded 200000 - or a config read
+    // that fails and falls back to it - lets it through.  110 is still above
+    // MPE_CONFIG's own 6 x 17 = 102, so the option can be put back afterwards.
     setConfigValue('DC_MAXCELLS_WEBEDIT', '110', () => {
       visitPage('view/data/TESTDATA.MPE_X_NEW')
       closeAbortModalIfPresent()
       openEdit()
 
-      cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
         'contain.text',
-        'Selection is too wide (7 rows x 18 cols) - please filter and try again!'
+        '6 rows'
       )
+      cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+        .should('be.visible')
+        .realHover()
+      cy.get('.editor-title clr-tooltip-content', {
+        timeout: longerCommandTimeout
+      })
+        .should('be.visible')
+        .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+        .should(($el) => expect($el.css('opacity')).to.eq('1'))
     })
 
     // Put the option back, so the rest of the estate sees the shipped default.
@@ -112,28 +147,31 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
       closeAbortModalIfPresent()
       openEdit()
 
-      // 126 cells under a 200000 cell limit: the editor loads, no abort.
-      cy.get('#hotTable .ht_master .htCore tbody tr', {
-        timeout: longerCommandTimeout
-      }).should('exist')
+      // 119 cells under a 200000 cell limit: all 7 rows load, no warning.
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+        'contain.text',
+        '7 rows'
+      )
+      cy.get('.truncation-warning').should('not.exist')
     })
   })
 
-  it('5 | the filter picker still lists the variables after the abort', () => {
+  it('5 | the warning says to filter, and filtering narrows the grid', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
     openEdit()
 
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
-      'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
-    )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'Filter the table to work with the rest')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
 
-    closeAbortModal()
-
-    // The abort returns no table, and the picker's variable list normally comes
-    // from the table's own columns - so without the public/getcols fallback the
-    // "please filter and try again" advice has nothing to pick from.  Every
-    // column of MPE_X_SEARCH must be on offer.
+    // The advice has to be actionable: the picker must offer the table's
+    // columns and the filter must reload the grid.
     openFilterPopup()
 
     filterVariables().should(($options) => {
@@ -155,20 +193,20 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
       ])
     })
 
-    // and the suggestion list is usable, not just populated
     setFilterVariable('SITE_NAME')
-
-    // Submitting is what exercises TYPE.  The picker keys its operator set and
-    // its value quoting on it, and the fallback list never passes through
-    // mergeColsRules, so without TYPE on this payload a character value would be
-    // submitted unquoted and the backend's %mp_filtercheck would reject it.
-    setFilterValue('Bristol')
+    // 100 of the 1000 rows: under the limit, so nothing is truncated.
+    setFilterValue('Nereid Trench')
     submitFilter()
 
     cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      "SITE_NAME = 'Bristol'"
+      "SITE_NAME = 'Nereid Trench'"
     )
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      '100 rows'
+    )
+    cy.get('.truncation-warning').should('not.exist')
   })
 })
 
@@ -185,13 +223,6 @@ const openEdit = () => {
   }).click()
 }
 
-/** The abort modal offers Close, and the filter button sits behind it. */
-const closeAbortModal = () => {
-  cy.get('app-info-modal', { timeout: longerCommandTimeout })
-    .contains('button', 'Close')
-    .click()
-}
-
 /**
  * The EDIT screen's Filter button opens the same .filter-modal the viewer uses.
  * It only renders once the editor is in its read-only (non-editing) state,

client/package-lock.json

@@ -7,15 +7,15 @@
       "name": "data_controller-client",
       "hasInstallScript": true,
       "dependencies": {
-        "@angular/animations": "20.3.33",
+        "@angular/animations": "20.3.34",
         "@angular/cdk": "^20.2.14",
-        "@angular/common": "20.3.33",
-        "@angular/compiler": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/forms": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
-        "@angular/platform-browser-dynamic": "20.3.33",
-        "@angular/router": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/compiler": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/forms": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
+        "@angular/platform-browser-dynamic": "20.3.34",
+        "@angular/router": "20.3.34",
         "@cds/core": "^6.15.1",
         "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
         "@clr/icons": "^13.0.2",
@@ -65,7 +65,7 @@
         "@angular-eslint/schematics": "19.8.1",
         "@angular-eslint/template-parser": "19.8.1",
         "@angular/cli": "20.3.37",
-        "@angular/compiler-cli": "20.3.33",
+        "@angular/compiler-cli": "20.3.34",
         "@babel/plugin-proposal-private-methods": "^7.18.6",
         "@compodoc/compodoc": "^2.0.0",
         "@cypress/webpack-preprocessor": "^5.17.1",
@@ -1439,9 +1439,9 @@
       }
     },
     "node_modules/@angular/animations": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.3.33.tgz",
-      "integrity": "sha512-z9gPYm1byXYE0tMGzjjDXE4OprxvjUhV4jOXVfLLlU0Sk5ChM3GwR0m2Vh+hDWDrJ2f7nfKKTeoGlLriP7NPZA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.3.34.tgz",
+      "integrity": "sha512-HdDfbLwpxzx49wAWz4oOUQKSkZcFNRpWEGsDEgufZ3ExwxRk1/2QPJuvZ2CbhKeEcLPrLKUorQ8XlGrNsJmNiQ==",
       "deprecated": "@angular/animations is deprecated. Use `animate.enter` and `animate.leave` instead. For more information see: https://v22.angular.dev/guide/animations.",
       "license": "MIT",
       "dependencies": {
@@ -1451,7 +1451,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/core": "20.3.33"
+        "@angular/core": "20.3.34"
       }
     },
     "node_modules/@angular/cdk": {
@@ -1987,9 +1987,9 @@
       }
     },
     "node_modules/@angular/common": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.33.tgz",
-      "integrity": "sha512-PMfSF2j/C3gGHPp6O6c/BL+32msNzhmkzavrxtmBSxIGAOB5Q4KNktnIuolVrvVvu/0UrZj1/zwT91r/GKRaeA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.34.tgz",
+      "integrity": "sha512-OhB/l0cNldOvQeICqbWqRUzK5fF6zNYy44Q4cIGJZql8HXL/invGiqWLE9f+kU8z4jRisofHIGTBtyJQYYOudg==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -1998,14 +1998,14 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/core": "20.3.33",
+        "@angular/core": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/@angular/compiler": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.33.tgz",
-      "integrity": "sha512-tqKZaapDV05r2SXA6bOzTSpiNhSoqZR9Usuacs5myMpbpt4gwNMAZkgaIoIcPutnhbd3qT+Og54VCVvCOXF8pg==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.34.tgz",
+      "integrity": "sha512-8agPW384c/JER1ersPTmJJfcCOnlYcBbdj0MYRxNdiYe+7gtnspF3ORuJmxqCEpMS17AljSmk2skwUcYnHB6xA==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2015,9 +2015,9 @@
       }
     },
     "node_modules/@angular/compiler-cli": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.3.33.tgz",
-      "integrity": "sha512-NKE4N4ssXysHLJQ7xVqvAv/yf3JfJxaFz5H+Ue+EnYToAmTxXe7uyt2MPRIydef/H63n7O9uphVp5P5GkRxVuA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.3.34.tgz",
+      "integrity": "sha512-PKHzhSOyDUFh64Ull3fUwgFlxnsQNEAuRyz4KqQ8FsTGZ4pOpUaZntO+2Xn1nX/gT2whSyGBJhD3H37zA8ZGEQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -2038,7 +2038,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/compiler": "20.3.33",
+        "@angular/compiler": "20.3.34",
         "typescript": ">=5.8 <6.0"
       },
       "peerDependenciesMeta": {
@@ -2048,9 +2048,9 @@
       }
     },
     "node_modules/@angular/core": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.33.tgz",
-      "integrity": "sha512-Ld/AhODjPf8c14xUOWUnJygaZb2TsRy99ZhwFkEsRJOzqauzEFovb8Hb2MEaB1e4RolADN+HWxPvE4n+u3HFmA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.34.tgz",
+      "integrity": "sha512-xOvSmGoao3SVPz//R7zTrq7+WdrIGi+WNpAdxJEvJJXuSpLGS3kSUqukjY4kOMochg2v0NCEPIdCyH+yPs8JSw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2059,7 +2059,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/compiler": "20.3.33",
+        "@angular/compiler": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0",
         "zone.js": "~0.15.0"
       },
@@ -2073,9 +2073,9 @@
       }
     },
     "node_modules/@angular/forms": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.3.33.tgz",
-      "integrity": "sha512-UmwKS9toyYhgq8LEbvFxNBfD9dUXr+B6Y5OTwBGwIRy8RhUrOiLwrXTRr+2y/yREGgDtGAWl8zQRVMVk2BNybw==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.3.34.tgz",
+      "integrity": "sha512-DndQJEoORsdthFFVRvT9WHn8YcA85jkjMxCLtXyf+TDVk0zbqObbFDfrloRnJwsINqfDNsJrz23ko3YU6sjnTw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2084,16 +2084,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/@angular/platform-browser": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.33.tgz",
-      "integrity": "sha512-od2uw6ee68r5TczjoYDsJuIDWoH4op3/req3sdeFtVHWdQgzf/AhM0B5ZBVRya8THon/b/SDHdhxOrUTgcszTA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.34.tgz",
+      "integrity": "sha512-0bd2Bj05WPoBTFM0A13IwbzxKVQPstx/N+7X0ZWKCifliyydIc7DBHu+r37ybJpl8ZsCVBe7DoR9Gon4lZ+zlw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2102,9 +2102,9 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/animations": "20.3.33",
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33"
+        "@angular/animations": "20.3.34",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34"
       },
       "peerDependenciesMeta": {
         "@angular/animations": {
@@ -2113,9 +2113,9 @@
       }
     },
     "node_modules/@angular/platform-browser-dynamic": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.3.33.tgz",
-      "integrity": "sha512-IwElnTCUW1ZZRr/pw6IRb9dRyNF8KBOh9QoGcmTk6uRAWynenYAe5yICFvgOKozqOHBarFNv8h4jXXJ/W8TcfA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.3.34.tgz",
+      "integrity": "sha512-vqKzExmUDf5jpIlNZRuICsLzaLWY5yTvuXsJhe97KTbC29BD71LhRGhEu7fjmR5uYy+LForpWYm5IsYlJcJAGQ==",
       "deprecated": "@angular/platform-browser-dynamic is deprecated. Use `@angular/platform-browser` instead.",
       "license": "MIT",
       "dependencies": {
@@ -2125,16 +2125,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/compiler": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33"
+        "@angular/common": "20.3.34",
+        "@angular/compiler": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34"
       }
     },
     "node_modules/@angular/router": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.3.33.tgz",
-      "integrity": "sha512-M55ibvrJSWGP7sGf+uZFvC66cxRkFlS4TjtI4Owus4gDCqW69/jhUo8pNGXf0T3QpO3iorbHScZfGwVmLATLLA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.3.34.tgz",
+      "integrity": "sha512-vQo2f/94nUoNMZaxGY0+fK1t9Oe7zJuxHhoMVYhDbCqKxINVmxMX/9uEHlHeu9UIE7JMazepQjRzBLo7bf9xCw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2143,9 +2143,9 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
@@ -15420,9 +15420,9 @@
       "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
-      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
+      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
       "funding": [
         {
           "type": "individual",

client/package.json

@@ -41,15 +41,15 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "20.3.33",
+    "@angular/animations": "20.3.34",
     "@angular/cdk": "^20.2.14",
-    "@angular/common": "20.3.33",
-    "@angular/compiler": "20.3.33",
-    "@angular/core": "20.3.33",
-    "@angular/forms": "20.3.33",
-    "@angular/platform-browser": "20.3.33",
-    "@angular/platform-browser-dynamic": "20.3.33",
-    "@angular/router": "20.3.33",
+    "@angular/common": "20.3.34",
+    "@angular/compiler": "20.3.34",
+    "@angular/core": "20.3.34",
+    "@angular/forms": "20.3.34",
+    "@angular/platform-browser": "20.3.34",
+    "@angular/platform-browser-dynamic": "20.3.34",
+    "@angular/router": "20.3.34",
     "@cds/core": "^6.15.1",
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
@@ -99,7 +99,7 @@
     "@angular-eslint/schematics": "19.8.1",
     "@angular-eslint/template-parser": "19.8.1",
     "@angular/cli": "20.3.37",
-    "@angular/compiler-cli": "20.3.33",
+    "@angular/compiler-cli": "20.3.34",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@compodoc/compodoc": "^2.0.0",
     "@cypress/webpack-preprocessor": "^5.17.1",

client/src/app/editor/editor.component.html

@@ -245,6 +245,25 @@
               }
             </clr-tooltip>
 
+            @if (truncationNote) {
+              <clr-tooltip>
+                <clr-icon
+                  clrTooltipTrigger
+                  shape="warning"
+                  aria-label="The selection was truncated"
+                  class="color-orange truncation-warning"
+                  size="20"
+                ></clr-icon>
+                <clr-tooltip-content
+                  clrPosition="right"
+                  clrSize="lg"
+                  *clrIfOpen
+                >
+                  {{ truncationNote }}
+                </clr-tooltip-content>
+              </clr-tooltip>
+            }
+
             @if (dataSource) {
               @if (!zeroFilterRows) {
                 ({{ dataSource.length | thousandSeparator: ',' }}

client/src/app/editor/editor.component.ts

@@ -160,6 +160,9 @@ export class EditorComponent
   dsmeta: DSMeta[] = []
   versions: Version[] = []
   dsNote = ''
+  /** Set when getdata served a truncated selection - shown as a warning
+   * against the table name, with the reason in a tooltip. */
+  truncationNote = ''
 
   viewboxes = false
 
@@ -1360,8 +1363,14 @@ export class EditorComponent
             colSpecs.TYPE === 'num' &&
             !colSpecs.DDTYPE.includes('TIME') &&
             !colSpecs.DDTYPE.includes('DATE')
-          )
-            row[colKey] = parseInt(row[colKey])
+          ) {
+            const numericValue = parseFloat(row[colKey])
+
+            // A cell the file parser could not read as a number stays as it
+            // was: the editor's numeric validation then flags it as text,
+            // where a NaN would render as an empty cell.
+            if (!Number.isNaN(numericValue)) row[colKey] = numericValue
+          }
         }
       }
     }
@@ -3930,6 +3939,11 @@ export class EditorComponent
       this.dsNote = ''
     }
 
+    // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
+    // truncated to the first rows within the limit, and carries the reason
+    // here.  Empty means the whole selection is in the grid.
+    this.truncationNote = response.data.sasparams[0].TRUNC_MSG || ''
+
     const hot = this.hotInstance
 
     const approvers: Approver[] = response.data.approvers

client/src/app/models/TableData.ts

@@ -47,6 +47,10 @@ export interface SASParam {
   PKCNT: number
   RK_FLAG: number
   TMVARS: string
+  /** 1 when getdata cut the selection down to the row / cell limit */
+  TRUNCATED?: number
+  /** Reason shown in the tooltip when TRUNCATED is 1, else empty */
+  TRUNC_MSG?: string
 }
 
 export interface ExcelRule {

sas/mocks/sasjs/services/admin/makedata.js

@@ -281,6 +281,53 @@ const tables = {
       DD_SENSITIVITY: 'Low'
     }
   ],
+  // Derived (computed) variables.  Values are display-time only - the logic is
+  // applied by mpe_derived in the mock's getdata/getcolvals, and must use the
+  // expression subset that compileSasExpression supports (the real evaluation
+  // is done by SAS).
+  MPE_DERIVED: [
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'REGION',
+      var_deps: 'STATE',
+      var_type: 'C',
+      var_len: 8,
+      var_fmt: '',
+      var_logic: "ifc(STATE in ('CA','OR','WA'),'West','East')",
+      active: 1
+    },
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'LINE_VALUE',
+      var_deps: 'QTY PRICE',
+      var_type: 'N',
+      var_len: 8,
+      var_fmt: 'dollar12.2',
+      var_logic: 'QTY * PRICE',
+      active: 1
+    },
+    {
+      // built on REGION, so it proves the dependency order is applied
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      var_nm: 'REGION_LABEL',
+      var_deps: 'REGION STATE',
+      var_type: 'C',
+      var_len: 14,
+      var_fmt: '',
+      var_label: 'Region and state',
+      var_logic: "cats(REGION,'-',STATE)",
+      active: 1
+    }
+  ],
   MPE_DATALOADS: [],
   MPE_EMAILS: [],
   MPE_EXCEL_CONFIG: [],
@@ -2226,7 +2273,7 @@ tables.MPE_TABLES.push({
 
 // ─── MPE_X_WIDE: wide table for the EDIT cell limit ─────────────────────────
 // 200 rows x 1001 columns (ROW_ID + 500 single-character + 500 numeric) is
-// 200,400 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
+// 200,200 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
 // DC_MAXOBS_WEBEDIT (250), so it exercises the CELL limit on its own and the
 // row limit spec can use MPE_X_SEARCH (1000 rows) instead.  Values are one
 // character / one digit on purpose: the point is width, and small values keep
@@ -3262,6 +3309,20 @@ const schema = {
       notnull: false
     }
   ],
+  MPE_DERIVED: [
+    { name: 'TX_FROM', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true },
+    { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
+    { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_DEPS', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
+    { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_LABEL', type: 'C', length: 256, format: '', label: '', notnull: false },
+    { name: 'VAR_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'ACTIVE', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'TX_TO', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true }
+  ],
   MPE_DATALOADS: [
     {
       name: 'LIBREF',
@@ -5349,7 +5410,15 @@ const testTables = {
     { ORDER_ID: 1003, CUSTOMER: 'Cobalt GmbH', REGION: 'EU', AMOUNT: 620, STATUS: 'CLOSED' },
     { ORDER_ID: 1004, CUSTOMER: 'Delta Systems', REGION: 'US', AMOUNT: 4500, STATUS: 'NEW' }
   ],
-  DEMO_MIRROR: []
+  DEMO_MIRROR: [],
+  // Derived variable demo - REGION and LINE_VALUE are not columns of this
+  // table, they are computed at read time from MPE_DERIVED rows.
+  DERIVED_DEMO: [
+    { ID: 1, STATE: 'CA', QTY: 2, PRICE: 10.5 },
+    { ID: 2, STATE: 'NY', QTY: 1, PRICE: 99 },
+    { ID: 3, STATE: 'OR', QTY: 5, PRICE: 3 },
+    { ID: 4, STATE: 'TX', QTY: 3, PRICE: 7.25 }
+  ]
 }
 
 // Schemas for the TESTDATA tables, matching the old inline getdata.js payload
@@ -5691,6 +5760,12 @@ const testSchema = {
       label: 'Last reviewed',
       notnull: false
     }
+  ],
+  DERIVED_DEMO: [
+    { name: 'ID', type: 'N', length: 8, format: '', label: 'ID', notnull: true },
+    { name: 'STATE', type: 'C', length: 2, format: '', label: 'State', notnull: false },
+    { name: 'QTY', type: 'N', length: 8, format: '', label: 'Quantity', notnull: false },
+    { name: 'PRICE', type: 'N', length: 8, format: 'dollar12.2', label: 'Price', notnull: false }
   ]
 }
 
@@ -5739,6 +5814,16 @@ tables.MPE_TABLES.push(
     buskey: 'ID',
     notes: 'Compact rule demo table (eight columns)'
   },
+  {
+    tx_from: 0,
+    tx_to: 127490111999,
+    libref: testDataLibref,
+    dsn: 'DERIVED_DEMO',
+    num_of_approvals_required: 1,
+    loadtype: 'UPDATE',
+    buskey: 'ID',
+    notes: 'Derived variable demo table'
+  },
   {
     tx_from: 0,
     tx_to: 127490111999,

sas/mocks/sasjs/services/dcMockUtils.js

@@ -1154,6 +1154,289 @@ function mpFilterStore(opts) {
     }
 }
 
+// ─── mpe_derived.sas ────────────────────────────────────────────────────────
+
+/**
+ * Compiles a SAS expression into a JS function - for the mock only.
+ *
+ * The real evaluation is done by SAS. This covers the subset used by the mock
+ * and demo configs, so the EDIT screen can be exercised (display, filtering,
+ * submission) without a SAS server:
+ *
+ *   - column names, numbers, quoted strings
+ *   - + - * / and comparisons  (= ne < > <= >=)
+ *   - AND OR, and `X IN (a, b, c)`
+ *   - ifc(), cats(), catx(), upcase(), lowcase(), strip(), trim(), length(),
+ *     substr(), index()
+ *
+ * Anything outside that subset throws when a row is derived, and the column is
+ * left missing for that row - the same way a failing SAS expression would.
+ */
+function compileSasExpression(expr, columnNames) {
+    // Tokenise first, so operators are only rewritten outside quoted strings
+    // (a literal such as 'OR' must never be read as an operator).
+    const tokens = []
+    const s = String(expr)
+    let i = 0
+    while (i < s.length) {
+        const c = s[i]
+        if (/\s/.test(c)) { i++; continue }
+        if (c === "'" || c === '"') {
+            const q = c
+            let j = i + 1
+            let val = ''
+            while (j < s.length) {
+                if (s[j] === q) {
+                    if (s[j + 1] === q) { val += q; j += 2; continue }
+                    break
+                }
+                val += s[j]
+                j++
+            }
+            tokens.push({ type: 'str', value: val })
+            i = j + 1
+            continue
+        }
+        if (/[0-9]/.test(c)) {
+            let j = i
+            while (j < s.length && /[0-9.]/.test(s[j])) j++
+            tokens.push({ type: 'num', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        if (/[A-Za-z_]/.test(c)) {
+            let j = i
+            while (j < s.length && /[A-Za-z0-9_]/.test(s[j])) j++
+            tokens.push({ type: 'word', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        const two = s.slice(i, i + 2)
+        if (['<=', '>=', '!=', '<>'].includes(two)) {
+            tokens.push({ type: 'punct', value: two })
+            i += 2
+            continue
+        }
+        // only the operators in the documented subset may reach the generated
+        // function - anything else is rejected rather than executed
+        if (!'+-*/(),<>='.includes(c)) {
+            throw new Error('Unsupported character in derived expression: ' + c)
+        }
+        tokens.push({ type: 'punct', value: c })
+        i++
+    }
+
+    // X IN (a, b, c)  ->  __in([a, b, c], X)
+    const out = []
+    for (let t = 0; t < tokens.length; t++) {
+        const tok = tokens[t]
+        if (
+            tok.type === 'word' &&
+            tok.value.toLowerCase() === 'in' &&
+            out.length > 0 &&
+            tokens[t + 1] &&
+            tokens[t + 1].value === '('
+        ) {
+            let depth = 0
+            let j = t + 1
+            for (; j < tokens.length; j++) {
+                if (tokens[j].value === '(') depth++
+                else if (tokens[j].value === ')') {
+                    depth--
+                    if (depth === 0) break
+                }
+            }
+            if (j < tokens.length) {
+                const left = out.pop()
+                const inner = tokens.slice(t + 2, j)
+                out.push(
+                    { type: 'word', value: '__in' },
+                    { type: 'punct', value: '(' },
+                    { type: 'punct', value: '[' },
+                    ...inner,
+                    { type: 'punct', value: ']' },
+                    { type: 'punct', value: ',' },
+                    left,
+                    { type: 'punct', value: ')' }
+                )
+                t = j
+                continue
+            }
+        }
+        out.push(tok)
+    }
+
+    const OPS = {
+        and: '&&', or: '||',
+        eq: '==', ne: '!=', lt: '<', gt: '>', le: '<=', ge: '>='
+    }
+    const js = out
+        .map(tok => {
+            if (tok.type === 'str') return JSON.stringify(tok.value)
+            if (tok.type === 'num') return tok.value
+            if (tok.type === 'word') return OPS[tok.value.toLowerCase()] || tok.value
+            if (tok.value === '=') return '=='
+            if (tok.value === '<>') return '!='
+            return tok.value
+        })
+        .join(' ')
+
+    const sasTrue = v => {
+        if (typeof v === 'boolean') return v
+        if (typeof v === 'number') return v !== 0 && !isNaN(v)
+        if (v === null || v === undefined) return false
+        const str = String(v).trim()
+        return str !== '' && str !== '0'
+    }
+    const toStr = v => {
+        if (v === null || v === undefined) return ''
+        if (typeof v === 'number') return isNaN(v) ? '' : String(v)
+        return String(v).trim()
+    }
+    const fns = {
+        ifc: (c, a, b) => (sasTrue(c) ? a : b),
+        cats: (...args) => args.map(toStr).join(''),
+        catx: (sep, ...args) => args.map(toStr).filter(v => v !== '').join(toStr(sep)),
+        upcase: v => toStr(v).toUpperCase(),
+        lowcase: v => toStr(v).toLowerCase(),
+        strip: v => toStr(v),
+        trim: v => toStr(v),
+        length: v => toStr(v).length,
+        substr: (v, p, n) => {
+            const str = toStr(v)
+            const start = Number(p) - 1
+            return n === undefined
+                ? str.substring(start)
+                : str.substring(start, start + Number(n))
+        },
+        index: (v, sub) => toStr(v).indexOf(toStr(sub)) + 1,
+        __in: (list, v) => list.some(item => String(item) === String(v) || Number(item) === Number(v))
+    }
+
+    const names = Object.keys(fns)
+    const fn = new Function(...columnNames, ...names, 'return (' + js + ');')
+    return row => fn(...columnNames.map(n => row[n]), ...names.map(n => fns[n]))
+}
+
+/**
+ * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
+ * variables with a compiled row function, in dependency order (ties by name),
+ * the same sequence the real macro emits.
+ *
+ * @param {Object} opts
+ * @param {string} opts.libref - target libref
+ * @param {string} opts.table - target dataset
+ * @param {string} opts.dataDir - data directory of the DC control library
+ * @param {Array<Object>} opts.columns - physical columns of the target table
+ */
+function mpeDerived(opts) {
+    const { libref, table, dataDir, columns } = opts
+    const loadTableData = makeTableLoader(dataDir)
+    const cfg = loadTableData('MPE_DERIVED')
+    if (!cfg || !Array.isArray(cfg.rows) || cfg.rows.length === 0) return []
+
+    const now = Date.now() / 1000
+    const lib = String(libref || '').toUpperCase()
+    const dsn = String(table || '').toUpperCase()
+    const columnNames = (columns || []).map(c => c.name)
+
+    const vars = cfg.rows
+        .filter(r =>
+            String(colVal(r, 'LIBREF') || '').toUpperCase() === lib &&
+            String(colVal(r, 'DSN') || '').toUpperCase() === dsn &&
+            Number(colVal(r, 'ACTIVE')) === 1 &&
+            (!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
+        )
+        .map(r => {
+            const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
+            const nm = String(colVal(r, 'VAR_NM') || '').toUpperCase()
+            return {
+                name: nm,
+                // every input the expression uses - columns and derived vars
+                deps: String(colVal(r, 'VAR_DEPS') || '')
+                    .toUpperCase()
+                    .split(/\s+/)
+                    .filter(Boolean),
+                type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
+                length: Number(colVal(r, 'VAR_LEN')) || 8,
+                format: String(colVal(r, 'VAR_FMT') || ''),
+                logic,
+                label: String(colVal(r, 'VAR_LABEL') || '').trim() || nm + '=' + logic
+            }
+        })
+        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
+
+    // dependency order: repeatedly take the variables whose derived inputs are
+    // already placed, alphabetically within a round
+    const byName = new Map(vars.map(v => [v.name, v]))
+    const placed = []
+    const done = new Set()
+    while (placed.length < vars.length) {
+        const ready = vars
+            .filter(v => !done.has(v.name))
+            .filter(v => v.deps.every(d => !byName.has(d) || done.has(d)))
+            .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+        if (ready.length === 0) {
+            console.log('derived variables for ' + lib + '.' + dsn + ' contain a cycle')
+            return []
+        }
+        for (const v of ready) {
+            placed.push(v)
+            done.add(v.name)
+        }
+    }
+    vars.length = 0
+    vars.push.apply(vars, placed)
+
+    // compile each against the physical columns plus the variables defined
+    // earlier, so a derived variable can build on another one
+    const compileCols = columnNames.slice()
+    for (const d of vars) {
+        try {
+            // slice(): the compiled function captures the column list it was
+            // built with, not the array as it grows
+            d.fn = compileSasExpression(d.logic, compileCols.slice())
+        } catch (err) {
+            console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
+            d.fn = () => (d.type === 'N' ? null : '')
+        }
+        compileCols.push(d.name)
+    }
+    return vars
+}
+
+/**
+ * Adds the derived variables to every row and to the column list, so that the
+ * filter (which runs afterwards) and the grid both see them.
+ */
+function mpeDeriveRows(tableData, derived) {
+    if (!derived || derived.length === 0) return tableData
+    const rows = tableData.rows.map(row => {
+        const copy = Object.assign({}, row)
+        for (const d of derived) {
+            try {
+                copy[d.name] = d.fn(copy)
+            } catch (err) {
+                copy[d.name] = d.type === 'N' ? null : ''
+            }
+        }
+        return copy
+    })
+    const derivedCols = derived.map(d => ({
+        name: d.name,
+        type: d.type,
+        length: d.length,
+        format: d.format,
+        label: d.label,
+        notnull: false
+    }))
+    return {
+        name: tableData.name,
+        columns: (tableData.columns || []).concat(derivedCols),
+        rows
+    }
+}
+
 // ─── DC settings (services/settings.js) ──────────────────────────────────────
 
 /**

sas/mocks/sasjs/services/editors/getdata.js

@@ -77,6 +77,21 @@ if (mpeTablesData && mpeTablesData.rows) {
     tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
 }
 
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Applied before the filter, so a filter can reference them.  The derived
+// columns are added to the schema too, which is what puts them in COLS and in
+// the grid (read only, since the mock marks every column READ).
+
+const derived = mpeDerived({
+    libref: requestedLibref,
+    table: requestedTable,
+    dataDir: mpeDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
 // Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
 // NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
@@ -191,9 +206,8 @@ if (tableData && tableData.rows && tableData.columns) {
 // ─── Build response ──────────────────────────────────────────────────────────
 
 let response
-// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
-// getdata.sas, and is emitted as sasjsAbort below.
-let abortMsg = ''
+// Set when the row or cell limit cut the selection down - mirrors the
+// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.
 
 if (tableData && tableData.columns && tableData.rows) {
     // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
@@ -236,15 +250,15 @@ if (tableData && tableData.columns && tableData.rows) {
     }
 
     // ─── Row and cell limits ─────────────────────────────────────────────────
-    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
-    // cells (rows x columns), whichever is reached first.  Both are read from
-    // MPE_CONFIG, the way the real service picks them up from the settings
-    // program it generates - so editing the option in DC changes what the mock
-    // allows, and an option that is absent (or switched off, var_active=0)
-    // falls back to the shipped default.  The check runs after the
-    // PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
-    // replace the data with a larger table.  The cell count includes the
-    // housekeeping delete column, which is what getdata.sas counts.
+    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
+    // cells (rows x columns), whichever is reached first - the selection is
+    // cut to min(rows, cells/columns) and flagged in sasparams.
+    // Both options are read from MPE_CONFIG, the way the real service picks
+    // them up from the settings program it generates - so editing the option
+    // in DC changes what the mock serves, and an option that is absent (or
+    // switched off, var_active=0) falls back to the shipped default.  The
+    // check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
+    // does - a hook may replace the data with a larger table.
     // MPE_CONFIG lives in the control library, not in the table's own library,
     // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
     // uses.  Passing the requested table's dataDir made the read fail for every
@@ -252,20 +266,32 @@ if (tableData && tableData.columns && tableData.rows) {
     const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
     const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
     const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
-    const editColCount = visibleColumns.length + 1
-    const cellCount = visibleRows.length * editColCount
-    if (visibleRows.length > maxObsWebEdit) {
-        abortMsg =
-            'Table is too big (' +
-            visibleRows.length +
-            ' rows) - please filter and try again!'
-    } else if (cellCount > maxCellsWebEdit) {
-        abortMsg =
-            'Selection is too wide (' +
-            visibleRows.length +
-            ' rows x ' +
-            editColCount +
-            ' cols) - please filter and try again!'
+    // getdata.sas counts the columns of work.out, i.e. the table's own
+    // columns - the delete column is client-owned and not part of it.
+    const editColCount = visibleColumns.length
+    let maxRows = maxObsWebEdit
+    let limitVar = 'DC_MAXOBS_WEBEDIT'
+    if (editColCount > 0) {
+        const cellRows = Math.floor(maxCellsWebEdit / editColCount)
+        if (cellRows < maxRows) {
+            maxRows = cellRows
+            limitVar = 'DC_MAXCELLS_WEBEDIT'
+        }
+    }
+    if (maxRows < 1) maxRows = 1
+    let truncated = 0
+    let truncMsg = ''
+    if (visibleRows.length > maxRows) {
+        truncated = 1
+        // Same wording as the macro getdata.sas sets, so the spec can assert
+        // on the text the real service would send.
+        truncMsg =
+            'Only the first ' +
+            maxRows +
+            ' rows are loaded - the ' +
+            limitVar +
+            ' limit has been reached.  Filter the table to work with the rest.'
+        visibleRows = visibleRows.slice(0, maxRows)
     }
 
     // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
@@ -371,7 +397,9 @@ if (tableData && tableData.columns && tableData.rows) {
             LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
             RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
             CLS_FLAG: 0,
-            ISMAP: 0
+            ISMAP: 0,
+            TRUNCATED: truncated,
+            TRUNC_MSG: truncMsg
         }],
         versions: [],
         xl_rules: []
@@ -386,26 +414,21 @@ if (tableData && tableData.columns && tableData.rows) {
         ],
         maxvarlengths: [], query: [], sasdata: [],
         $sasdata: { vars: {} },
-        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0 }],
+        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
         versions: [], xl_rules: []
     }
 }
 
 webOutOpen()
-if (abortMsg) {
-    webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
-    webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
-} else {
-    webOutObj(response.approvers, 'approvers')
-    webOutObj(response.cols, 'cols')
-    webOutObj(response.dqdata, 'dqdata')
-    webOutObj(response.dqrules, 'dqrules')
-    webOutObj(response.dsmeta, 'dsmeta')
-    webOutObj(response.maxvarlengths, 'maxvarlengths')
-    webOutObj(response.query, 'query')
-    webOutObj(response.sasdata, 'sasdata', response.$sasdata)
-    webOutObj(response.sasparams, 'sasparams')
-    webOutObj(response.versions, 'versions')
-    webOutObj(response.xl_rules, 'xl_rules')
-}
+webOutObj(response.approvers, 'approvers')
+webOutObj(response.cols, 'cols')
+webOutObj(response.dqdata, 'dqdata')
+webOutObj(response.dqrules, 'dqrules')
+webOutObj(response.dsmeta, 'dsmeta')
+webOutObj(response.maxvarlengths, 'maxvarlengths')
+webOutObj(response.query, 'query')
+webOutObj(response.sasdata, 'sasdata', response.$sasdata)
+webOutObj(response.sasparams, 'sasparams')
+webOutObj(response.versions, 'versions')
+webOutObj(response.xl_rules, 'xl_rules')
 webOutClose()

sas/mocks/sasjs/services/public/getcolvals.js

@@ -21,9 +21,25 @@ const tableName = libds.split('.')[1] || ''
 // ─── Load table data (searches all libref data folders) ──────────────────────
 
 const _loaded = loadTableAnyLib(tableName) || {}
-const tableData = _loaded.data
+let tableData = _loaded.data
 const tableLibref = _loaded.libref || libref
 
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Present in the schema and in the rows before the filter is applied, so the
+// filter may reference them and the values below are computed from the same
+// row set the grid shows.
+
+const controlDataDir = libDataDir(dcLibref)
+const derived = mpeDerived({
+    libref: tableLibref,
+    table: tableName,
+    dataDir: controlDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── Resolve column metadata ─────────────────────────────────────────────────
 
 let colMeta = null

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "4.20.9",
-        "@sasjs/core": "5.5.2"
+        "@sasjs/core": "5.7.2"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -240,9 +240,9 @@
       "license": "MIT"
     },
     "node_modules/@sasjs/core": {
-      "version": "5.5.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.5.2.tgz",
-      "integrity": "sha512-puojJhAKAKg5o0uHdGSkpQ3cXgzajMY9AXAm3R22l6bZ4GKvC03qRq1ueGjBBF3npNxviwIKCABTsNEtjs1LAw==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.7.2.tgz",
+      "integrity": "sha512-1cjzI5Gg6vLTDX6jqZFQbGjbscXtZRa9pR6Ck+29cVa9VW3Av1FE/tQD4grxXrhEHMv/VJlBHq/odDJeaLtRHg==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {

sas/package.json

@@ -31,7 +31,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "4.20.9",
-    "@sasjs/core": "5.5.2"
+    "@sasjs/core": "5.7.2"
   },
   "overrides": {
     "nanoid": "3.3.18"

sas/sasjs/db/datactrl/mpe_derived.ddl

@@ -0,0 +1,32 @@
+/**
+  @file mpe_derived.ddl
+  @brief ddl file
+  @details Defines the derived (computed) variable configuration table.  This
+  drives MPE_DERIVED rows in MPE_TABLES, so that the table itself can be
+  maintained through the EDIT screen with the usual approve workflow.
+
+  Derived values are not persisted - the logic is applied at read time, before
+  the filter, so that the variables can be filtered on.
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+
+create table &curlib..MPE_DERIVED
+   (
+    TX_FROM num not null format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VAR_NM varchar(32),
+    VAR_DEPS varchar(4000),
+    VAR_TYPE varchar(1),
+    VAR_LEN int,
+    VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
+    VAR_LOGIC varchar(4000),
+    ACTIVE int,
+    TX_TO num not null format=datetime19.3,
+    constraint pk_mpe_derived
+        primary key(tx_from, libref, dsn, var_nm));

sas/sasjs/db/datactrl/mpe_tables.ddl

@@ -118,3 +118,16 @@ insert into &mpelib..mpe_tables
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
     ;
+  insert into &mpelib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&mpelib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;

sas/sasjs/db/migrations/20261019_v7.19_release.sas

@@ -0,0 +1,169 @@
+/**
+  @file
+  @brief migration script to move to v7.19 of Data Controller
+
+  REQUIRED CHANGE - adds the MPE_DERIVED table, which holds derived (computed)
+  variable definitions for the EDIT screen, and registers it in MPE_TABLES so
+  that it can be maintained through the app with the usual approve workflow.
+
+  Replace YOURDCLIB and the library path below, then run the script.  Any
+  MPE_DERIVED rows added before this script runs will be lost when the table
+  is created, so run it before configuring any derived variables.
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+proc sql;
+create table &dclib..mpe_derived
+  (
+    tx_from num not null format=datetime19.3,
+    libref varchar(8),
+    dsn varchar(32),
+    var_nm varchar(32),
+    var_deps varchar(4000),
+    var_type varchar(1),
+    var_len num,
+    var_fmt varchar(32),
+    var_label varchar(256),
+    var_logic varchar(4000),
+    active num,
+    tx_to num not null format=datetime19.3
+  );
+quit;
+
+proc datasets lib=&dclib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn var_nm)
+    /nomiss unique;
+quit;
+
+proc sql;
+insert into &dclib..mpe_tables
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="&dclib"
+    ,dsn='MPE_DERIVED'
+    ,num_of_approvals_required=1
+    ,loadtype='TXTEMPORAL'
+    ,buskey='LIBREF DSN VAR_NM'
+    ,post_edit_hook='services/hooks/mpe_derived_postedit'
+    ,notes='Derived (computed) variables for the EDIT screen'
+    ,var_txfrom='TX_FROM'
+    ,var_txto='TX_TO'
+  ;
+quit;
+
+/* Dropdown values for VAR_TYPE */
+proc sql noprint;
+  select max(selectbox_rk) into: maxrk from &dclib..mpe_selectbox;
+quit;
+
+%let rk1=%eval(&maxrk+1);
+%let rk2=%eval(&maxrk+2);
+
+proc sql;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk1
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='C'
+    ,selectbox_order=1
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk2
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='N'
+    ,selectbox_order=2
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+quit;
+
+/* Field rules for the EDIT screen.  These are client side - the server side
+  * enforcement is the post edit hook registered above. */
+proc sql;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+quit;

sas/sasjs/macros/mpe_derived.sas

@@ -0,0 +1,103 @@
+/**
+  @file
+  @brief Generates the SAS statements that create Data Controller derived variables
+  @details Derived (computed) variables are configured in MPE_DERIVED.  Each
+  row defines one variable, which is created in a DATA step view ahead of the
+  filter, so that it can be filtered on in the EDIT screen.
+
+  The engine lives in @sasjs/core as mp_derivedvars - this macro only supplies
+  what is specific to Data Controller: the control table name, the bitemporal
+  filter that picks the current version of a row, and the list of names that
+  belong to the metadata and may not be used as a variable.
+
+  The macro writes the generated statements to a fileref, so that callers can
+  compose them into a view:
+
+      %mpe_derived(libds=&libds,dclib=&mpelib,outref=dercode)
+      data work.vw_derived / view=work.vw_derived;
+        set &libds;
+        %include dercode;
+      run;
+
+  Alternatively, pass outview= and the macro will build the view itself.
+
+  The filter must be applied by the caller, in the step that reads the view:
+
+      data out;
+        set work.vw_derived;
+        where %inc filtref;;
+      run;
+
+  A WHERE statement applies to the data set named on the SET statement, so it
+  cannot reference a variable that is created in the same data step.  Applying
+  it to the view puts the derived variables on the input, which is what makes
+  them filterable.
+
+  Each statement block is of the form:
+
+      attrib REGION length=$21 label='REGION=<expression>' format=$21.;
+      REGION = <expression>;
+
+  Type and length are declared by the administrator, so the result is
+  deterministic.  VAR_LABEL is optional: when it is missing or blank the label
+  is the variable name, an equals sign and the expression, otherwise the
+  configured label is used as given.
+
+  VAR_DEPS lists every input the expression uses - physical columns and
+  derived variables alike.  The statements are generated in dependency order,
+  so a derived variable may build on another one without a sequence number
+  being maintained.  A cycle is rejected.
+
+  Semicolons, macro triggers and comments are rejected, so a row can only ever
+  be an expression.  A derived variable may not reuse the name of a physical
+  column, and may not reference itself: an intra-row calculation would imply a
+  retained value, which depends on the order rows happen to arrive in.
+
+  <h4> SAS Macros </h4>
+  @li mp_derivedvars.sas
+  @li mp_abort.sas
+
+  @param [in] libds= The target table (libref.dataset, lib must be assigned)
+  @param [in] dclib= The Data Controller control library
+  @param [out] outref= (0) Fileref to receive the generated statements
+  @param [out] outvars= (work.derived_vars) Dataset of derived variable names
+  @param [in] outview= (0) Optional two-level name for the derived view
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%macro mpe_derived(libds=,
+  dclib=,
+  outref=0,
+  outvars=work.derived_vars,
+  outview=0
+);
+
+%local reserved;
+
+/* the only name that has to be refused outright - the metadata columns of the
+  target table are physical columns, so the engine's collision check covers
+  them whatever MPE_TABLES calls them */
+%let reserved=%str(_____DELETE__THIS__RECORD_____);
+
+%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing: libds dclib)
+)
+
+%mp_derivedvars(
+  libds=&libds
+  ,control=&dclib..mpe_derived
+  ,where=%str(&dc_dttmtfmt lt tx_to and active=1)
+  ,outref=&outref
+  ,outvars=&outvars
+  ,outview=&outview
+  ,reserved=&reserved
+)
+
+%mend mpe_derived;

sas/sasjs/macros/mpe_derived.test.sas

@@ -0,0 +1,296 @@
+/**
+  @file
+  @brief testing the mpe_derived macro
+
+  @details Covers the generated statements, the view, the metadata the callers
+  consume, and the dependency rules.
+
+  The configuration checks abort the run when they fail, so they are not
+  exercised here - this suite covers the behaviour that callers depend on.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+  @li mpe_derived.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+
+/* a source table, and a table with no derived variables */
+data work.class;
+  set sashelp.class;
+run;
+data work.noconfig;
+  set sashelp.class;
+run;
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+/**
+  * Test 1 - statements, view, chaining
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='BAND'
+    ,var_deps='AGE'
+    ,var_type='C'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_label='Age band'
+    ,var_logic="ifc(AGE>13,'TEEN','JUNIOR')"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='BAND_AGE'
+    ,var_deps='BAND AGE'
+    ,var_type='C'
+    ,var_len=16
+    ,var_fmt=''
+    ,var_logic="cats(BAND,'-',AGE)"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='DOUBLE_AGE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE * 2'
+    ,active=1;
+quit;
+
+/* lower case libds on purpose - the config match must be case insensitive */
+/* the setup SQL above creates the SQL automatic macro variables, so the
+  snapshot is taken here - the check is for the macro's own leakage */
+%mp_assertscope(SNAPSHOT)
+
+%mpe_derived(libds=work.class,
+  dclib=&dclib,
+  outref=derref1,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+
+%mp_assertdsobs(work.derived_vars,
+  desc=Test1 - one row per derived variable,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+%mp_assertcols(work.vw_derived,
+  desc=Test1 - the view carries the derived columns,
+  cols=BAND BAND_AGE DOUBLE_AGE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk1;
+  set work.vw_derived;
+  where BAND='TEEN';
+run;
+%mp_assertdsobs(work.chk1,
+  desc=Test1 - the filter on a derived column selects rows,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* the chained value only resolves if the statements run in order */
+data work.chk1b;
+  set work.vw_derived;
+  where BAND='TEEN';
+  if BAND_AGE ne cats(BAND,'-',AGE) then output;
+run;
+%mp_assertdsobs(work.chk1b,
+  desc=Test1 - a derived variable can reference an earlier one,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+data work.chk1c;
+  set work.vw_derived;
+  if DOUBLE_AGE ne AGE*2 then output;
+run;
+%mp_assertdsobs(work.chk1c,
+  desc=Test1 - the numeric derived value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* proc contents on a view reports a format name without its width, so the
+  format is compared on the name only */
+proc contents noprint data=work.vw_derived
+  out=work.vw_cols(keep=name label format);
+run;
+data work.chk1d;
+  set work.vw_cols;
+  where name='BAND' and label='Age band' and format=:'$';
+run;
+%mp_assertdsobs(work.chk1d,
+  desc=Test1 - the label and default format are applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - a configured format is used as-is
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='FLAG_DT'
+    ,var_deps=''
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt='DATE9.'
+    ,var_logic='mdy(1,1,2020)'
+    ,active=1;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref2,
+  outview=work.vw_derived2,
+  outvars=work.derived_vars2
+)
+
+/* asserted on the emitted statement - proc contents on a view does not
+  report the format in full */
+data work.chk2;
+  infile derref2 truncover;
+  input line $4000.;
+  if index(line,'format=DATE9.')>0 then output;
+run;
+%mp_assertdsobs(work.chk2,
+  desc=Test2 - the configured format is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk2b;
+  set work.vw_derived2;
+  if flag_dt ne mdy(1,1,2020) then output;
+  stop;
+run;
+%mp_assertdsobs(work.chk2b,
+  desc=Test2 - the numeric value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - no configuration
+  */
+
+%mpe_derived(libds=WORK.NOCONFIG,
+  dclib=&dclib,
+  outref=derref3,
+  outview=work.vw_derived3,
+  outvars=work.derived_vars3
+)
+
+%mp_assertdsobs(work.derived_vars3,
+  desc=Test3 - no derived variables returned,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+%mp_assert(iftrue=(%mf_existds(libds=work.vw_derived3)=0),
+  desc=Test3 - no view is built when there is no config,
+  outds=work.test_results
+)
+
+data _null_;
+  infile derref3 end=eof;
+  input;
+  if not eof or cats(_infile_) ne '' then do;
+    putlog 'ERR' 'OR: derived statement file is not empty';
+    call symputx('emptycheck','0','l');
+  end;
+run;
+%mp_assert(iftrue=(%symexist(emptycheck)=0),
+  desc=Test3 - no statements are generated,
+  outds=work.test_results
+)
+
+/**
+  * Test 4 - the wrapper scopes the config: only the current, active row of a
+  * table is used
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='01JAN2020:00:00:00'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='CLOSED_OFF'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 1'
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='NOT_ACTIVE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 2'
+    ,active=0;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.chk4;
+  set work.derived_vars4;
+  where name in ('CLOSED_OFF','NOT_ACTIVE');
+run;
+
+%mp_assertdsobs(work.chk4,
+  desc=Test4 - a closed version and an inactive row are ignored,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* tidy up so repeated runs start clean */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+%mp_assertscope(COMPARE,ignorelist=_SASJS_WEBOUT_HEADERS,outds=work.test_results)

sas/sasjs/macros/mpe_makedata.sas

@@ -679,6 +679,24 @@ insert into &lib..mpe_security set
       ,selectbox_value='UNLOCKED'
       ,selectbox_order=2
       ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='C'
+      ,selectbox_order=1
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='N'
+      ,selectbox_order=2
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
   insert into &lib..mpe_selectbox set
       selectbox_rk=%mf_increment(rk)
       ,ver_from_dttm=0
@@ -1495,6 +1513,19 @@ insert into &lib..mpe_selectbox set
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
     ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;
   insert into &lib..mpe_tables
     set tx_from=0
       ,tx_to='31DEC5999:23:59:59'dt
@@ -1944,6 +1975,87 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/columns_in_libds"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+
+/* MPE_DERIVED - the field rules are also enforced server-side, in
+  * services/hooks/mpe_derived_postedit.sas.  These rows give the user
+  * immediate feedback in the EDIT screen. */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"

sas/sasjs/macros/mpe_makedatamodel.sas

@@ -692,6 +692,29 @@ proc datasets lib=&lib noprint;
     pk_mpe_validations=(tx_from base_lib base_ds base_col rule_type)
     /nomiss unique;
 quit;
+/* mpe_derived - computed variables for the EDIT screen */
+proc sql;
+create table &lib..MPE_DERIVED
+  (
+    TX_FROM num &notnull format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VAR_NM varchar(32),
+    VAR_DEPS varchar(4000),
+    VAR_TYPE varchar(1),
+    VAR_LEN num,
+    VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
+    VAR_LOGIC varchar(4000),
+    ACTIVE num,
+    TX_TO num &notnull format=datetime19.3
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn var_nm)
+    /nomiss unique;
+quit;
 proc sql;
 create table &lib..mpe_x_test(
           primary_key_field num &notnull,

sas/sasjs/services/editors/getdata.sas

@@ -17,6 +17,10 @@
   @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
     else set to 1 (CLS rules exist)
   @li ISMAP - set to 1 if the target DS is an excel map target, else 0
+  @li TRUNCATED - set to 1 if the selection was cut down to the
+    DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
+  @li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
+    else empty
 
   <h5> approvers </h5>
   <h5> dqrules </h5>
@@ -68,11 +72,14 @@
   @li mp_validatecol.sas
   @li mpe_accesscheck.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getlabels.sas
   @li mpe_getversions.sas
   @li mpe_filtermaster.sas
   @li mpe_runhook.sas
+  @li mpeinit.sas
+  @li mpeterm.sas
 
   @version 9.2
   @author 4GL Apps Ltd
@@ -201,6 +208,21 @@ select upcase(loadtype)
   outds=work.query
 )
 
+/**
+  * Derived (computed) variables, configured in MPE_DERIVED.  They are added
+  * via a view, so that the filter (applied by the step that reads the view)
+  * can reference them - a WHERE clause cannot see a variable that is created
+  * in the same data step.
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%put &=derived_nobs;
+
 %macro mpestp_getdata();
   %if not %symexist(DC_MAXOBS_WEBEDIT) %then %do;
     %put NOTE:;%put NOTE- DC_MAXOBS_WEBEDIT not found!;
@@ -237,7 +259,12 @@ select upcase(loadtype)
 
   %if &loadtype=BITEMPORAL %then %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
       where %inc filtref;;
     run;
     proc sort data=out;
@@ -251,7 +278,12 @@ select upcase(loadtype)
   %end;
   %else %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
       where %inc filtref;;
     run;
   %end;
@@ -262,30 +294,40 @@ select upcase(loadtype)
   )
 
   /* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
-    then run the hook, then check the row and cell limits.  Checking once,
-    after both, means the counts cover everything that could reach the
-    browser: the rows the selection matched, and anything the hook has added
-    to work.out.  A wide table exhausts the cell budget long before it
-    reaches the row limit - 250 rows of a 1000 column table is a quarter of
-    a million cells - so capping rows alone does not bound the work the
-    browser is asked to do.  The payload itself is capped further down, when
-    work.outdata is built. */
+    then run the hook, then apply the row and cell limits (below). */
   %let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
   proc sort data=work.out; by &sortPK; run;
 
   %mpe_runhook(PRE_EDIT_HOOK)
 
+  /* Serve the selection truncated, at whichever limit binds first, and carry
+    the reason in sasparams for the EDIT screen to show against the table
+    name.  Bounding the browser's work is the point: a wide table exhausts the
+    cell budget long before it reaches the row limit - 250 rows of a 1000
+    column table is a quarter of a million cells - so the row cap is
+    min(DC_MAXOBS_WEBEDIT, DC_MAXCELLS_WEBEDIT / columns).  Counting after the
+    PRE_EDIT_HOOK means the cap covers what the hook added to work.out. */
   %let obscnt=%mf_getattrn(work.out,NLOBS);
   %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
-  %let cellcnt=%eval(&obscnt * &colcnt);
-  %mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Table is too big (&obscnt rows) - please filter and try again!
-  )
-  %mp_abort(iftrue=(&cellcnt>&DC_MAXCELLS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Selection is too wide (&obscnt rows x &colcnt cols) - please filter and try again!
-  )
+  %let maxrows=&DC_MAXOBS_WEBEDIT;
+  %let limitvar=DC_MAXOBS_WEBEDIT;
+  %if &colcnt>0 %then %do;
+    %let cellrows=%sysfunc(floor(&DC_MAXCELLS_WEBEDIT/&colcnt));
+    %if %sysevalf(&cellrows lt &maxrows) %then %do;
+      %let maxrows=&cellrows;
+      %let limitvar=DC_MAXCELLS_WEBEDIT;
+    %end;
+  %end;
+  %if %sysevalf(&maxrows lt 1) %then %let maxrows=1;
+  %global truncated trunc_msg;
+  %let truncated=0;
+  %if %sysevalf(&obscnt gt &maxrows) %then %let truncated=1;
+  %let trunc_msg=;
+  %if &truncated=1 %then %do;
+    %let trunc_msg=Only the first &maxrows rows are loaded;
+    %let trunc_msg=&trunc_msg - the &limitvar limit has been reached.;
+    %let trunc_msg=&trunc_msg  Filter the table to work with the rest.;
+  %end;
 
 
   /* order pk fields at start of table.  The delete flag is NOT added here:
@@ -297,9 +339,9 @@ select upcase(loadtype)
                             ,Str2=&pk
                           );
   %put sourcevars=&sourcevars;
-  /* the payload is capped here, at the row limit - the limits above are
-    checked against the whole selection, this is what goes to the browser */
-  options obs=&DC_MAXOBS_WEBEDIT;
+  /* the payload is capped at &maxrows - whichever limit that works out to be
+    - this is what goes to the browser */
+  options obs=&maxrows;
   data outdata;
     /* pk fields come first */
     %if %length(&pk)>0 %then %do;
@@ -513,13 +555,28 @@ select upcase(loadtype)
 /* extract col info */
 %mp_getcols(&libds, outds=cols1)
 
+/* derived variables are added to the view, so take their attributes from
+  there - this keeps type, length, format and label identical to the data */
+%if &derived_nobs>0 %then %do;
+  %mp_getcols(work.vw_derived, outds=derived_cols_all)
+  proc sql;
+  create table work.derived_cols as
+    select a.*
+    from work.derived_cols_all a
+    where a.NAME in (select NAME from work.derived_vars);
+  quit;
+  data work.cols1;
+    set work.cols1 work.derived_cols;
+  run;
+%end;
+
 /* join with cls rules.  Trim COLS payload in the SQL below - DDTYPE
   C=CHARACTER N=NUMERIC (dates kept in full), VARNUM dropped (client
   relies on column order, not position) */
 proc sql;
 create table work.cols as
   select a.NAME
-    ,coalesce(c.desc,a.NAME) as LABEL
+    ,coalesce(c.desc,a.LABEL,a.NAME) as LABEL
     ,a.FMTNAME
     ,case a.DDTYPE
       when 'CHARACTER' then 'C'
@@ -572,7 +629,7 @@ select count(*) into: ismap from &mpelib..mpe_xlmap_info
   where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
 
 data sasparams;
-  length colHeaders $20000 filter_text $32767;
+  length colHeaders $20000 filter_text $32767 trunc_msg $200;
   colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
   pkCnt=countw("&pk");
   pk="&pk";
@@ -583,6 +640,10 @@ data sasparams;
   if trim(symget('rk_underlying')) ne '' then rk_flag=1;
   else rk_flag=0;
   filter_text=symget('filter_text');
+  /* set when the selection was cut down to the row / cell limit, so the EDIT
+    screen can say so against the table name */
+  truncated=&truncated;
+  trunc_msg="&trunc_msg";
   if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
   else cls_flag=1;
   put (_all_)(=);

sas/sasjs/services/editors/getdata.test.1.sas

@@ -2,17 +2,19 @@
   @file
   @brief testing getdata row and cell limits
 
-  @details The EDIT screen refuses a selection that is too big on either
-  count - rows (DC_MAXOBS_WEBEDIT) or cells (DC_MAXCELLS_WEBEDIT).  The row
-  check runs first, so a selection over the row limit reports "Table is too
-  big" and one under the row limit but over the cell limit reports
-  "Selection is too wide".  Both aborts are proven by the child job state
-  (an aborted service registers no webout).
+  @details The EDIT screen is bounded by two limits - rows
+  (DC_MAXOBS_WEBEDIT) and cells, rows x columns (DC_MAXCELLS_WEBEDIT) -
+  whichever is reached first.  A selection over either is SERVED TRUNCATED:
+  the grid gets the first rows that fit, and sasparams carries TRUNCATED=1
+  plus a TRUNC_MSG naming the limit, which the EDIT screen shows as a warning
+  against the table name.
 
-  Fixture: dctest.wideboy (1000 rows x 1004 columns), created and registered
-  by testsetup.  Filtered to 300 rows it is over the 250 row limit; filtered
-  to 200 rows it is under the row limit but 200 x 1004 = 200,800 cells, over
-  the 200,000 cell limit.
+  Test 1 uses MPE_X_TEST (several hundred rows, few columns) so the row
+  limit alone decides the cap.  Test 2 uses the fixture dctest.wideboy (1000
+  rows x 1004 columns, created and registered by testsetup) filtered to 200
+  rows: that is under the row limit but 200 x 1004 = 200,800 cells, over the
+  200,000 cell limit, so the cell limit decides the cap -
+  floor(200000 / 1004) = 199 rows.
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
@@ -25,7 +27,7 @@
 %let _program=&appLoc/services/editors/getdata;
 
 /**
-  * build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
+  * Build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
   * in &filter_rk
   */
 %macro wideboy_filter(maxrow);
@@ -57,32 +59,56 @@
 %mend wideboy_filter;
 
 /**
-  * Test 1 - row limit (300 rows > 250)
+  * Call getdata as a child job and expose the webout as the &outlib JSON
+  * libname, so the test can read what the browser would have received.
   */
-%wideboy_filter(300)
-%let f1=%mf_getuniquefileref();
-data _null_;
-  file &f1 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%macro call_getdata(libds,filter_rk,outlib);
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref termstr=crlf;
+    put 'LIBDS:$43. FILTER_RK:8.';
+    put "&libds,&filter_rk";
+  run;
+  %mx_execute(&_program,
+    viyacontext=&defaultcontext,
+    inputfiles=&fref:sascontroltable,
+    outlib=&outlib
+  )
+  %let syscc=0;
+%mend call_getdata;
 
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f1:sascontroltable,
-  outref=web1
-)
+/**
+  * Test 1 - row limit.  MPE_X_TEST is well under the cell limit, so the
+  * 250 row default is what caps the payload.
+  */
+%call_getdata(&dclib..MPE_X_TEST,-1,web1)
 
-%let abort1=0;
+%let t1_nobs=-1;
+%let t1_trunc=-1;
+%let t1_msg=;
+proc sql noprint;
+select count(*) into: t1_nobs from web1.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
+  set web1.sasparams;
+  call symputx('t1_trunc',truncated);
+  call symputx('t1_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
-%let syscc=0;
 
 %mp_assert(
-  iftrue=(&abort1=1),
-  desc=Test 1 - a selection over the row limit is refused,
+  iftrue=(&t1_trunc=1),
+  desc=Test 1 - a selection over the row limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t1_nobs=250),
+  desc=Test 1 - the payload is capped at DC_MAXOBS_WEBEDIT (250) rows,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t1_msg,DC_MAXOBS_WEBEDIT)>0),
+  desc=Test 1 - the truncation message names the DC_MAXOBS_WEBEDIT limit,
   outds=work.test_results
 )
 
@@ -90,28 +116,34 @@ run;
   * Test 2 - cell limit (200 rows x 1004 cols = 200,800 > 200,000)
   */
 %wideboy_filter(200)
-%let f2=%mf_getuniquefileref();
-data _null_;
-  file &f2 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%call_getdata(dctest.wideboy,&filter_rk,web2)
 
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f2:sascontroltable,
-  outref=web2
-)
-
-%let abort2=0;
+%let t2_nobs=-1;
+%let t2_trunc=-1;
+%let t2_msg=;
+proc sql noprint;
+select count(*) into: t2_nobs from web2.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
+  set web2.sasparams;
+  call symputx('t2_trunc',truncated);
+  call symputx('t2_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
 
 %mp_assert(
-  iftrue=(&abort2=1),
-  desc=Test 2 - a selection over the cell limit is refused,
+  iftrue=(&t2_trunc=1),
+  desc=Test 2 - a selection over the cell limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t2_nobs=199),
+  desc=Test 2 - the payload is capped at floor(DC_MAXCELLS_WEBEDIT / cols),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t2_msg,DC_MAXCELLS_WEBEDIT)>0),
+  desc=Test 2 - the truncation message names the DC_MAXCELLS_WEBEDIT limit,
   outds=work.test_results
 )
 

sas/sasjs/services/editors/getdata.test.2.sas

@@ -21,10 +21,10 @@
   editing MPE_GROUPS alone - Row Level Security is skipped for them - so the
   test reports a skip row instead of running the assertions.
 
-  The call passes a filter (PRIMARY_KEY_FIELD < 100) because getdata.sas also
-  enforces the row / cell limits, and the unfiltered MPE_X_TEST is above the
-  row limit (642 rows against DC_MAXOBS_WEBEDIT:250) - the limits test
-  (getdata.test.1.sas) covers that abort separately.
+  The call passes a filter (PRIMARY_KEY_FIELD < 100) so the whole selection is
+  served - the unfiltered MPE_X_TEST is above the row limit (642 rows against
+  DC_MAXOBS_WEBEDIT:250) and would come back truncated.  The limits test
+  (getdata.test.1.sas) covers that truncation separately.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas

sas/sasjs/services/editors/getdata.test.sas

@@ -3,12 +3,14 @@
   @brief testing gethistory service
 
   <h4> SAS Macros </h4>
+  @li mp_assert.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
   @li mp_filterstore.sas
+  @li mpe_derived.sas
   @li mx_execute.sas
 
 **/
@@ -240,3 +242,179 @@ run;
   test=EQUALS 10,
   outds=work.test_results
 )
+
+
+/**
+  * Test 4 - derived variables
+  */
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_deps='LIBREF DSN'
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
+    ,active=1;
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE2'
+    ,var_deps='DERIVED_NOTE'
+    ,var_type='C'
+    ,var_len=45
+    ,var_fmt=''
+    ,var_logic="cats(DERIVED_NOTE,' (2)')"
+    ,active=1;
+quit;
+
+/* a filter may reference the derived variable, so validate against the view */
+%mpe_derived(libds=&dc_libref..MPE_TABLES,
+  dclib=&dc_libref,
+  outref=dercode4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.inquery4;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
+;;;;
+run;
+%mp_filterstore(
+  libds=&dc_libref..MPE_TABLES,
+  targetds=work.vw_derived4,
+  filter_summary=&dc_libref..mpe_filteranytable,
+  filter_detail=&dc_libref..mpe_filtersource,
+  lock_table=&dc_libref..mpe_lockanytable,
+  maxkeytable=&dc_libref..mpe_maxkeyvalues,
+  queryds=work.inquery4,
+  outresult=work.result4,
+  outquery=work.query4
+)
+data _null_;
+  set work.result4;
+  call symputx('filter_rk4',filter_rk);
+run;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "&dclib..MPE_TABLES,&filter_rk4";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:sascontroltable,
+  outlib=web4,
+  mdebug=&sasjs_mdebug
+)
+
+/* a cancelled child returns no webout, so the outlib keeps whatever an
+  earlier call left in it - check the job state before reading its members,
+  otherwise the assertion failures below describe the wrong problem */
+%let abort4=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort4',1);
+run;
+
+%mp_assert(
+  iftrue=(&abort4=0),
+  desc=Test4 - the derived variable call completes,
+  outds=work.test_results
+)
+
+%if &abort4=0 %then %do;
+  data sasdata4;
+    set web4.sasdata;
+  run;
+  data cols4;
+    set web4.cols;
+  run;
+%end;
+%else %do;
+  data cols4;
+    length name $32;
+    stop;
+  run;
+%end;
+
+%mp_assertcols(work.cols4,
+  desc=Test4 - derived column is in the COLS payload,
+  cols=DERIVED_NOTE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk4a;
+  set work.cols4;
+  where name='DERIVED_NOTE' and cls_rule='READ';
+run;
+%mp_assertdsobs(work.chk4a,
+  desc=Test4 - derived column is read only,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4b;
+  set work.cols4;
+  where name='DERIVED_NOTE' and label=:'DERIVED_NOTE=';
+run;
+%mp_assertdsobs(work.chk4b,
+  desc=Test4 - derived column is labelled,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+%mp_assertdsobs(work.sasdata4,
+  desc=Test4 - filter on the derived column is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4c;
+  set work.sasdata4;
+  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCK_ANYTABLE');
+run;
+%mp_assertdsobs(work.chk4c,
+  desc=Test4 - derived value is correct,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* the second variable is built on the first, so this proves statement order */
+data work.chk4d;
+  set work.sasdata4;
+  if derived_note2=cats(derived_note,' (2)');
+run;
+%mp_assertdsobs(work.chk4d,
+  desc=Test4 - a derived variable can reference an earlier one,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* tidy up the config so repeated runs start clean.  Delete every row for the
+  table, not just the first variable: leaving DERIVED_NOTE2 behind means the
+  next run starts with a dependency on a variable that is no longer declared,
+  and every call aborts */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;

sas/sasjs/services/editors/stagedata.sas

@@ -22,6 +22,7 @@
   @li mf_getuniquefileref.sas
   @li mpe_loader.sas
   @li mpe_filtermaster.sas
+  @li mpe_derived.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_cntlout.sas
@@ -106,6 +107,37 @@ run;
 )
 
 %put Verify that the upload does not violate Row Level Security checks:;
+
+/* derived variables are display-only and must not be submitted, so drop them
+  from the data before the security check runs - otherwise a crafted
+  submission could satisfy an RLS clause that references a derived column */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outvars=work.derived_vars
+)
+%let derived_droplist=;
+%if %mf_nobs(work.derived_vars)>0 %then %do;
+  proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
+  run;
+  proc sql noprint;
+  select a.name into: derived_droplist separated by ' '
+    from work.derived_vars a
+    inner join work.jsdata_vars b
+    on a.name=b.name;
+  quit;
+%end;
+
+/* drop them from the submitted data itself - a DROP on the output of a later
+  step would leave the columns in that step's input, and the security clause
+  is evaluated against the input */
+%if %length(&derived_droplist)>0 %then %do;
+  data work.jsdata;
+    set work.jsdata;
+    drop &derived_droplist;
+  run;
+%end;
+
 %mpe_filtermaster(ULOAD,&libds,
   dclib=&mpelib,
   outref=filtref,
@@ -220,6 +252,9 @@ data _null_;
   put '%let ' name '=' value ';';
 run;
 
+/* derived variables are display-only - they never reach the approval package
+  or the target table.  work.jsdata was rewritten without them before the
+  security check, so there is nothing left to drop here */
 data approve.jsdset;
   length _____DELETE__THIS__RECORD_____ $3;
   set jsdata;

sas/sasjs/services/editors/stagedata.test.3.sas

@@ -132,6 +132,88 @@ quit;
   outds=work.test_results
 )
 
+/**
+  * Test 3 - an RLS rule naming a DERIVED variable must be rejected at save
+  * time: the rule is evaluated against the submitted data, where a derived
+  * column does not exist
+  */
+%let t3=%sysfunc(datetime());
+
+/* define a derived variable on the target table */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_REPLACE'
+    ,var_nm='DERIVED_X'
+    ,var_deps='X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='RLS rule on a derived variable should abort';
+  libds="&dc_libref..MPE_ROW_LEVEL_SECURITY";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length RLS_SCOPE $8 RLS_GROUP $128 RLS_LIBREF $8 RLS_TABLE $32
+    RLS_GROUP_LOGIC $3 RLS_SUBGROUP_LOGIC $3 RLS_VARIABLE_NM $32
+    RLS_OPERATOR_NM $12 RLS_RAW_VALUE $4000
+    _____DELETE__THIS__RECORD_____ $3;
+  RLS_SCOPE='EDIT';
+  RLS_GROUP='SASAdministrators';
+  RLS_LIBREF='DCTEST';
+  RLS_TABLE='DC_REPLACE';
+  RLS_GROUP_LOGIC='AND';
+  RLS_SUBGROUP_LOGIC='OR';
+  RLS_SUBGROUP_ID=0;
+  RLS_VARIABLE_NM='DERIVED_X';
+  RLS_OPERATOR_NM='NE';
+  RLS_RAW_VALUE='1';
+  RLS_ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort3=0;
+proc sql noprint;
+select count(*) into: abort3
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t3
+    and reason_txt="RLS rule on a derived variable should abort"
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort3>0),
+  desc=Checking an RLS rule on a derived variable is rejected,
+  outds=work.test_results
+)
+
+/* tidy up the derived configuration */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+quit;
+
 /**
   * Test 2 - applying REPLACE loadtype to a table with RLS EDIT security
   * should abort.  The active EDIT rule was inserted in the prep section,

sas/sasjs/services/hooks/mpe_derived_postedit.sas

@@ -0,0 +1,385 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_DERIVED table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+  This hook normalises the incoming rows and applies the configuration rules
+  at submit time, so a bad derived variable is rejected while the submitter is
+  still watching - rather than breaking the EDIT screen later, when the view
+  is built.
+
+  VAR_DEPS must list every input the expression uses, physical columns
+  and derived variables alike.  The hook checks the part that does not need
+  the target library: a derived variable referenced by an expression must be
+  declared, a variable may not depend on itself, and the resulting graph must
+  be acyclic - across the incoming rows and the current rows they do not
+  replace.
+
+  The target table is not assigned in this context, so the collision between
+  VAR_NM and a physical column, and the resolution of a physical dependency,
+  are left to %mpe_derived, which runs with the target library assigned.
+  Uniqueness on the business key (LIBREF DSN VAR_NM) is checked by mpe_loader
+  before this hook runs, but on the raw submitted values, so it is case
+  sensitive: 'Region' and 'REGION' for the same table both pass the loader and
+  collide only at approve time.  The post-normalisation check below rejects
+  the second one at submit time, naming the variable.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mpe_derived.sas
+  @li mpe_loader.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let errflag=0;
+%let errmsg=;
+
+/* a hook is included at execution time, so this is open code - the macro
+  language has to be inside a macro definition for %local to be valid */
+%macro _mpe_derived_postedit();
+  %global errflag errmsg;
+  %local left nready cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
+    cyc_n cyc_found;
+
+/* normalise the incoming values */
+data work.staging_ds;
+  set work.staging_ds;
+  if upcase(_____DELETE__THIS__RECORD_____) ne 'YES' then do;
+    libref=upcase(strip(libref));
+    dsn=upcase(strip(dsn));
+    var_nm=upcase(strip(var_nm));
+    var_type=upcase(strip(var_type));
+    var_fmt=upcase(strip(var_fmt));
+    var_label=strip(var_label);
+    var_deps=upcase(strip(var_deps));
+    var_logic=strip(var_logic);
+  end;
+run;
+
+/**
+  * Duplicate names after normalisation - mpe_loader compares the raw values,
+  * so a case variant reaches us here
+  */
+proc sql;
+create table work.derived_dup as
+  select libref, dsn, var_nm, count(*) as n
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  group by libref, dsn, var_nm
+  having count(*)>1;
+quit;
+
+data _null_;
+  set work.derived_dup(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',var_nm,' is declared more than once for '
+      ,libref,'.',dsn));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_dup))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+/**
+  * Field level checks - the same rules %mpe_derived applies at read time
+  */
+data work.derived_check;
+  set work.staging_ds;
+  length reason $200;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  call missing(reason);
+
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if var_nm='' then reason='VAR_NM is required';
+  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VAR_NM must be a valid SAS name';
+  else if verify(substr(var_nm,1,1),'0123456789')=0
+    then reason='VAR_NM must not start with a digit';
+  else if var_nm='_____DELETE__THIS__RECORD_____'
+    then reason='VAR_NM is reserved';
+  else if var_type not in ('C','N')
+    then reason='VAR_TYPE must be C or N';
+  else if missing(var_len) or var_len ne int(var_len) or var_len<1
+    then reason='VAR_LEN must be a positive integer';
+  else if var_type='C' and var_len>32767
+    then reason='VAR_LEN exceeds 32767 for a character variable';
+  else if var_type='N' and var_len>8
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
+  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
+    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
+    then reason='VAR_FMT must be a format name, without ; & % or //*';
+  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VAR_FMT must be a format name';
+  else if missing(var_logic)
+    then reason='VAR_LOGIC is required';
+  else if index(var_logic,';')>0
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
+  else if indexc(var_logic,'&%')>0
+    then reason='VAR_LOGIC must not contain macro triggers';
+  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
+    then reason='VAR_LOGIC must not contain comments';
+
+  if not missing(reason) then output;
+run;
+
+%if %mf_nobs(work.derived_check)>0 %then %do;
+  data _null_;
+    set work.derived_check(obs=1);
+    call symputx('errmsg',cats('MPE_DERIVED: ',var_nm,': ',reason));
+    call symputx('errflag',1);
+  run;
+%end;
+
+%mp_abort(iftrue= (&errflag=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+/**
+  * The graph has to hold for the resulting configuration: the incoming rows
+  * plus the current rows of the same tables that this submission does not
+  * replace.
+  */
+proc sql;
+create table work.derived_rows as
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_deps)) as deps
+    ,strip(var_logic) as logic
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  union all
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_deps)) as deps
+    ,strip(var_logic) as logic
+  from &DC_LIBREF..mpe_derived
+  where &dc_dttmtfmt. lt tx_to
+    and catx('.',libref,dsn,var_nm) not in
+      (select catx('.',libref,dsn,var_nm) from work.staging_ds)
+    and catx('.',libref,dsn) in
+      (select catx('.',libref,dsn) from work.staging_ds);
+quit;
+
+data work.derived_dep;
+  set work.derived_rows;
+  length dep_nm $32;
+  do i=1 to countw(deps,' ');
+    dep_nm=scan(deps,i,' ');
+    if not missing(dep_nm) then output;
+  end;
+  keep libref dsn name dep_nm;
+run;
+
+data work.derived_ref;
+  set work.derived_rows;
+  length l $4000 tok $32;
+  l=prxchange("s/'([^']|'')*'//",-1,logic);
+  l=prxchange('s/"([^"]|"")*"//',-1,l);
+  l=prxchange('s/[^A-Za-z0-9_]/ /',-1,upcase(l));
+  do i=1 to countw(l,' ');
+    tok=scan(l,i,' ');
+    if not missing(tok) then output;
+  end;
+  keep libref dsn name tok;
+run;
+
+proc sql;
+create table work.derived_self as
+  select libref, dsn, name, dep_nm
+  from work.derived_dep
+  where name=dep_nm;
+
+create table work.derived_undeclared as
+  select r.libref, r.dsn, r.name, r.tok as dep_nm
+  from work.derived_ref r
+  where exists (
+      select 1 from work.derived_rows n
+      where n.libref=r.libref and n.dsn=r.dsn and n.name=r.tok
+    )
+    and r.tok ne r.name
+    and not exists (
+      select 1 from work.derived_dep d
+      where d.libref=r.libref and d.dsn=r.dsn and d.name=r.name
+        and d.dep_nm=r.tok
+    );
+quit;
+
+/* the message is always prepared, the abort is driven by the count - an
+  iftrue=1 inside a wrapping %if leaves the step alive on some engines */
+data _null_;
+  set work.derived_self(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_self))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+data _null_;
+  set work.derived_undeclared(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
+      ,', which is not in VAR_DEPS'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_undeclared))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+/* the inverse of the check above: a declared name the expression never uses
+  is stale, and the read-time engine rejects it too */
+proc sql;
+create table work.derived_stale as
+  select d.libref, d.dsn, d.name, d.dep_nm
+  from work.derived_dep d
+  where not exists (
+    select 1 from work.derived_ref r
+    where r.libref=d.libref and r.dsn=d.dsn and r.name=d.name
+      and r.tok=d.dep_nm
+  );
+quit;
+
+data _null_;
+  set work.derived_stale(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS lists ',dep_nm
+      ,', which VAR_LOGIC does not reference'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_stale))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+/* the same invariant the RLS hook enforces from its side: a derived variable
+  cannot be named by an active row level security rule, because the columns
+  are dropped before the security clause is evaluated */
+proc sql;
+create table work.derived_rls as
+  select d.libref, d.dsn, d.name
+  from work.derived_rows d
+  inner join &DC_LIBREF..mpe_row_level_security r
+    on upcase(r.rls_libref)=upcase(d.libref)
+    and upcase(r.rls_table)=upcase(d.dsn)
+    and upcase(r.rls_variable_nm)=upcase(d.name)
+  where r.rls_active=1
+    and &dc_dttmtfmt lt r.tx_to;
+quit;
+
+data _null_;
+  set work.derived_rls(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,' is named by a row level security rule on '
+      ,libref,'.',dsn,' - derived variables cannot be used there'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_rls))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+/**
+  * Cycle check - repeatedly take the variables whose derived inputs have all
+  * been placed; a round that places nothing leaves a cycle
+  */
+proc sql;
+create table work.derived_edge as
+  select d.libref, d.dsn, d.name as child, d.dep_nm as parent
+  from work.derived_dep d
+  where d.dep_nm in (select name from work.derived_rows);
+quit;
+
+data work.derived_todo;
+  set work.derived_rows(keep=libref dsn name);
+run;
+
+%let left=%mf_nobs(work.derived_todo);
+%let cyc_found=0;
+
+%do %while (&left>0);
+  proc sql;
+  create table work.derived_ready as
+    select t.libref, t.dsn, t.name
+    from work.derived_todo t
+    where not exists (
+      select 1 from work.derived_edge e
+      inner join work.derived_todo t2
+        on e.libref=t2.libref and e.dsn=t2.dsn and e.parent=t2.name
+      where e.libref=t.libref and e.dsn=t.dsn and e.child=t.name
+    );
+  quit;
+
+  %let nready=%mf_nobs(work.derived_ready);
+  %if &nready=0 %then %do;
+    /* walk the edges, so the message names the cycle rather than just the
+      names that could not be placed */
+    %let cyc_n=%mf_nobs(work.derived_todo);
+    data _null_;
+      set work.derived_todo(obs=1);
+      call symputx('cyc_start',name);
+    run;
+    %let cyc_path=&cyc_start;
+    %let cyc_cur=&cyc_start;
+    %let cyc_seen=%str(.&cyc_start..);
+    %do cyc_i=1 %to &cyc_n;
+      %let cyc_next=;
+      proc sql noprint;
+        select e.parent into: cyc_next
+        from work.derived_edge e
+        where e.child="&cyc_cur"
+          and catx('.',e.libref,e.dsn,e.parent) in
+            (select catx('.',libref,dsn,name) from work.derived_todo);
+      quit;
+      %if %length(&cyc_next)=0 %then %let cyc_i=&cyc_n;
+      %else %do;
+        %let cyc_path=&cyc_path. -> &cyc_next;
+        %if %index(&cyc_seen,%str(.&cyc_next..))>0 %then %let cyc_i=&cyc_n;
+        %else %do;
+          %let cyc_seen=&cyc_seen.&cyc_next..;
+          %let cyc_cur=&cyc_next;
+        %end;
+      %end;
+    %end;
+    /* leave the loop before aborting - an abort that only cancels the step
+      would re-enter it with the ready count still zero */
+    %let cyc_found=1;
+    %let left=0;
+  %end;
+
+  %if &cyc_found=0 %then %do;
+    proc sql;
+    delete from work.derived_todo
+      where catx('.',libref,dsn,name) in
+        (select catx('.',libref,dsn,name) from work.derived_ready);
+    quit;
+
+    %let left=%mf_nobs(work.derived_todo);
+  %end;
+%end;
+
+%mp_abort(iftrue= (&cyc_found=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
+)
+
+%mend _mpe_derived_postedit;
+
+%_mpe_derived_postedit()

sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas

@@ -18,6 +18,7 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_filtercheck.sas
 
@@ -81,6 +82,34 @@ data _null_;
   end;
 run;
 
+/* derived variables exist only in the view the EDIT screen reads - a rule on
+  one cannot be evaluated against the submitted data, so reject it here rather
+  than let the submit fail later with a variable-not-found error */
+proc sql;
+create table work.rls_derived as
+  select b.rls_libref, b.rls_table, b.variable_nm
+  from work.batches b
+  inner join &DC_LIBREF..mpe_derived d
+    on upcase(d.libref)=b.rls_libref
+    and upcase(d.dsn)=b.rls_table
+    and upcase(d.var_nm)=upcase(b.variable_nm)
+  where &dc_dttmtfmt lt d.tx_to
+    and d.active=1;
+quit;
+
+data _null_;
+  set work.rls_derived(obs=1);
+  call symputx('rls_msg'
+    ,cats('MPE_ROW_LEVEL_SECURITY: ',variable_nm
+      ,' is a derived variable in ',rls_libref,'.',rls_table
+      ,' - derived variables cannot be used in a row level security rule'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.rls_derived))
+  ,mac=mpe_row_level_security_postedit
+  ,msg=%superq(rls_msg)
+)
+
 %macro quickloop();
 %do i=1 %to &cnt;
   data work.inds&i;

sas/sasjs/services/public/getcolvals.sas

@@ -35,6 +35,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_nobs.sas
   @li mf_verifymacvars.sas
   @li dc_assignlib.sas
   @li mf_getvarformat.sas
@@ -42,6 +43,7 @@
   @li mp_cntlout.sas
   @li mp_filtercheck.sas
   @li mp_filtergenerate.sas
+  @li mpe_derived.sas
   @li mp_validatecol.sas
   @li mpe_validatecol.sas
 
@@ -150,10 +152,30 @@ run;
 )
 
 
+/**
+  * Derived (computed) variables - these exist only in a view, so the filter
+  * must be validated and applied against that view
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+/* %do/%end blocks, not '%then %let ...; %else %let ...;' - in open code the
+  %else is not paired and SAS stops with 'Expected %DO not found' */
+%if &derived_nobs>0 %then %do;
+  %let filterds=work.vw_derived;
+%end;
+%else %do;
+  %let filterds=&libds;
+%end;
+
 /**
   * Validate the filter query
   */
-%mp_filtercheck(work.filterquery,targetds=&libds,abort=YES)
+%mp_filtercheck(work.filterquery,targetds=&filterds,abort=YES)
 
 /**
   * Prepare the query
@@ -162,14 +184,14 @@ run;
 
 /* cannot %inc in a sql where clause, only data step, so - use a view */
 data work.vw_vals/view=work.vw_vals;
-  set &libds;
+  set &filterds;
   where %inc myfilter;;
 run;
 
 proc sql;
 create view work.vw_vals_sorted as
   select distinct
-      put(&col2,%mf_getVarFormat(&libds,&col2,force=1)) as formatted,
+      put(&col2,%mf_getVarFormat(work.vw_vals,&col2,force=1)) as formatted,
       &col2 as unformatted
   from work.vw_vals;
 
@@ -194,7 +216,7 @@ proc sql noprint;
 select count(*) into: nobs from work.vw_vals_sorted;
 data meta;
   column="&col2";
-  sasformat="%mf_getVarFormat(&libds,&col2)";
+  sasformat="%mf_getVarFormat(work.vw_vals,&col2)";
   startrow=&startrow;
   rows=&rows;
   nobs=&nobs;

sas/sasjs/services/public/getcolvals.test.1.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
   @li mx_execute.sas
   @li mf_getuniquefileref.sas
 
@@ -36,3 +37,54 @@ run;
   desc=DCLIB found in unfiltered getgolvals response,
   test=ANYVAL
 )
+
+/**
+  * derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
+    ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
+    ,var_nm='DERIVED_KEY'
+    ,var_deps='PRIMARY_KEY_FIELD'
+    ,var_type='C'
+    ,var_len=20
+    ,var_fmt=''
+    ,var_logic='cats(primary_key_field)'
+    ,active=1;
+quit;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. COL:$32.';
+  put "&dclib..MPE_X_TEST,DERIVED_KEY";
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:iwant,
+  outlib=web4
+)
+
+data work.vals4;
+  set web4.vals;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.vals4,
+  desc=Values are returned for a derived column,
+  test=HASOBS
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where var_nm='DERIVED_KEY'
+    and upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+quit;

sas/sasjs/services/public/getrawdata.sas

@@ -21,6 +21,8 @@
   @li mp_ds2csv.sas
   @li mp_streamfile.sas
   @li mp_validatecol.sas
+  @li mf_nobs.sas
+  @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li mpe_filtermaster.sas
 
@@ -143,10 +145,35 @@ data _null_;
   putlog _infile_;
 run;
 
+/* derived variables exist only in a view, so a filter that references one
+  must be applied against it.  They are dropped from the download - they are
+  an EDIT screen feature. */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let derived_droplist=;
+%if &derived_nobs>0 %then %do;
+  %let filterds=work.vw_derived;
+  proc sql noprint;
+  select name into: derived_droplist separated by ' '
+    from work.derived_vars;
+  quit;
+%end;
+%else %do;
+  %let filterds=&libds;
+%end;
+
 options obs=200000;/* stop limit */
 data staged(drop=&txfrom &txto);
-  set &libds;
+  set &filterds;
   where %inc filtref;;
+  %if %length(&derived_droplist)>0 %then %do;
+    drop &derived_droplist;
+  %end;
 run;
 options obs=max;
 

sas/sasjs/services/public/validatefilter.sas

@@ -26,8 +26,10 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_filterstore.sas
+  @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li removecolsfromwork.sas
 
@@ -69,8 +71,26 @@ run;
 
 %dc_assignlib(WRITE,%scan(&ds,1,.))
 
+/* derived variables exist only in a view, so validate against that */
+%mpe_derived(libds=&ds,
+  dclib=&dc_libref,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+/* %do/%end blocks, not '%then %let ...; %else %let ...;' - in open code the
+  %else is not paired and SAS stops with 'Expected %DO not found' */
+%if &derived_nobs>0 %then %do;
+  %let filterds=work.vw_derived;
+%end;
+%else %do;
+  %let filterds=&ds;
+%end;
+
 %mp_filterstore(
   libds=&ds,
+  targetds=&filterds,
   queryds=work.filterquery,
   filter_summary=&dc_libref..mpe_filteranytable,
   filter_detail=&dc_libref..mpe_filtersource,

sas/sasjs/services/public/validatefilter.test.sas

@@ -48,6 +48,8 @@ run;
 
 data work.iwant;
   filter_table='DCTEST.DCFMTS-FC';
+  /* mx_execute reads the program to run from the input dataset */
+  _program="&_program";
 run;
 data work.filterquery;
 GROUP_LOGIC='AND';
@@ -77,3 +79,66 @@ run;
   desc=Test2 result filter,
   test=EQUALS 1
 )
+
+/**
+  * Test 3 - filter on a derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_deps='LIBREF DSN'
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
+    ,active=1;
+quit;
+
+data work.iwant;
+  filter_table="&dclib..MPE_TABLES";
+run;
+data work.filterquery;
+  GROUP_LOGIC='AND';
+  SUBGROUP_LOGIC='AND';
+  SUBGROUP_ID=1;
+  VARIABLE_NM='DERIVED_NOTE';
+  OPERATOR_NM='CONTAINS';
+  RAW_VALUE="'MPE_LOCK'";
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.iwant work.filterquery,
+  outlib=web3,
+  outref=web3ref
+)
+data _null_;
+  infile web3ref;
+  input;
+  putlog _infile_;
+run;
+data result;
+  set web3.result;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.result,
+  desc=Test3 derived filter,
+  test=EQUALS 1
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where var_nm='DERIVED_NOTE'
+    and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;

sas/sasjs/services/public/viewdata.sas

@@ -59,6 +59,7 @@
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getversions.sas
   @li mpe_filtermaster.sas
@@ -218,6 +219,26 @@ run;
     putlog _infile_;
   run;
 
+  /* derived variables exist only in a view, so a filter that references one
+    must be applied against it.  The columns themselves are dropped from the
+    output - they are an EDIT screen feature. */
+  %mpe_derived(libds=&libds,
+    dclib=&mpelib,
+    outref=dercode,
+    outview=work.vw_derived,
+    outvars=work.derived_vars
+  )
+  %let derived_nobs=%mf_nobs(work.derived_vars);
+  %let derived_droplist=;
+  %if &derived_nobs>0 %then %do;
+    %let filterds=work.vw_derived;
+    proc sql noprint;
+    select name into: derived_droplist separated by ' '
+      from work.derived_vars;
+    quit;
+  %end;
+  %else %let filterds=&libds;
+
   %if &searchtype=NONE or "%trim(&searchtype) " = " " %then %do;
     /* get row count */
     filename rows temp;
@@ -227,7 +248,7 @@ run;
       input;
       if _n_=1 then do;
         put 'proc sql;';
-        put "select count(*) into: dsobs from &libds where";
+        put "select count(*) into: dsobs from &filterds where";
       end;
       put _infile_;
       if eof then put ';';
@@ -242,8 +263,11 @@ run;
 
     /* send actual data, filtered and row-capped */
     data work.viewdata;
-      set &libds;
+      set &filterds;
       where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
       if _n_>&DC_MAXOBS_WEBVIEW then stop;
     run;
     %if %mf_nobs(work.viewdata)=0 %then %do;
@@ -256,8 +280,11 @@ run;
   %end;
   %else %do;
     data work.vwsearch/view=work.vwsearch;
-      set &libds;
+      set &filterds;
       where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
     run;
     %if %upcase(&searchtype)=CHAR %then %do;
       %mp_searchdata(lib=work
@@ -275,6 +302,8 @@ run;
     %end;
     %if %mf_existds(libds=MPSEARCH.vwsearch) %then %do;
       %let dsobs=%mf_nobs(MPSEARCH.vwsearch);
+      /* the derived columns were dropped from work.vwsearch above, so
+        mp_searchdata's output cannot carry them */
       data viewdata;
         set MPSEARCH.vwsearch;
         if _n_<=&DC_MAXOBS_WEBVIEW;
@@ -283,7 +312,7 @@ run;
     %else %do;
       %let dsobs=0;
       data viewdata;
-        set &libds;
+        set &filterds;
         stop;
       run;
     %end;