test(download): check for the modal before using it, and click the menu item

Three review points.

attachExcelFile read '.clr-abort-modal .modal-title' unconditionally, so an
upload that raised no abort modal failed on the lookup, and the abort branch
then left the modal open for the next command to click through.  It asks the body
first: no abort modal means confirm the upload; an abort modal means assert its
title is not an Abort Message and stop there.

openDownloadModal clicked the deepest element holding the text.  The click
handler sits on the div[clrDropdownItem], so the item is selected by that
attribute and its text.

The spec list comment claimed download leaves nothing for later specs to trip
over.  It seeds the approval it needs and that approval stays, which the comment
now says.

.agents/skills/dc-estate-testing/SKILL.md

@@ -44,7 +44,7 @@ An unmatched `/SASjsApi/...` path answers `200` with the Swagger page, so check
 
 `npm run serverdata` sends `deploy/makeDataServer.json`: one `fromjs` row with `ADMIN`, `DCPATH` and `OUTFMT`. Two things about it are worth knowing before you trust its result.
 
-- **`ADMIN` must name a group the calling account is actually in.** The shipped value is `AllUsers`, which is not a group on the shared estate - the account belongs to `all-users` (check with a call to `services/public/getgroups`, or read the `MF_GETUSER` the response echoes back). With a group the account is not in, the seed still answers `SUCCESS` and every later `getdata`/`stagedata` call aborts with `<user> is not authorised to edit <lib>.<ds> in the <lib>.MPE_SECURITY table`. Override `ADMIN` to `all-users` for the estate rather than editing the shipped JSON.
+- **`ADMIN` must name a group the calling account is actually in.** The shipped value is `all-users`, the SASjs Server group that every logged-in user is a member of, which is what the shared estate needs. A name the account is not in - a Viya-style group such as `AllUsers`, or one estate's own group carried into another - still answers `SUCCESS` from the seed, and then every later `getdata`/`stagedata` call aborts with `<user> is not authorised to edit <lib>.<ds> in the <lib>.MPE_SECURITY table`. Check what the account belongs to with a call to `services/public/getgroups` before changing the value.
 - **Each run creates a new library.** The response carries `STATUS`, `DCPATH` and `LIBREF` (for example `DC621835`), and the LIBREF is clock-derived, so a libref from an earlier run answers `dataset <lib>.<table> does not exist` - take the current one from the response.
 - **`KEEPME` controls whether the seed can be repeated.** Without it `makedata` deletes itself and the configurator after a successful run (deliberately, so it cannot be executed twice over real data), and a second seed fails with `The Program at (<appLoc>/services/admin/makedata) does not exist` until you run `sasjs cbd` again. A harness that reseeds between runs sets `KEEPME` in the input row.
 - **Seed once per deploy, not before every test run.** The suite builds its own `dctest` library from the seeded control tables; reseeding in between is wasted work, and there is no need to run `makedata` and `sasjs test` in the opposite order.

.gitea/scripts/cypress-specs.sh

@@ -14,16 +14,19 @@
 # known green keeps its shape. metanav is deliberately absent - see
 # https://git.datacontroller.io/dc/dc/issues/395.
 #
-# download is held out for a different reason - its download button is covered by
-# a fixed-position element, see
-# https://git.datacontroller.io/dc/dc/issues/398.
+# download was held out while its selectors were stale - see
+# https://git.datacontroller.io/dc/dc/issues/398. Its first case uploads a fixture
+# to mpe_x_test and submits it, so it creates the approval it downloads instead of
+# depending on what earlier specs leave behind - but the approval stays behind.
+# It is appended because a spec added later goes on the end rather than reordering
+# the sequence that was known green.
 set -euo pipefail
 
-SPECS="csv-limited hook-programs liveness editor excel-multi-load excel csv filtering filter-panel licensing viewer-labels full-table-search viewbox stage browser-info ddl-export row-cell-limits derived-variables startupservice-launcher coltype-delete-record submitted-details embed-va"
+SPECS="csv-limited hook-programs liveness editor excel-multi-load excel csv filtering filter-panel licensing viewer-labels full-table-search viewbox stage browser-info ddl-export row-cell-limits derived-variables startupservice-launcher coltype-delete-record submitted-details embed-va download"
 
 # Specs that exist but are held out of CI, each with the reason. Listed here so
 # the coverage check below can tell a deliberate exclusion from a forgotten file.
-EXCLUDED="metanav download"
+EXCLUDED="metanav"
 
 case "${1:-}" in
   all)

CHANGELOG.md

@@ -1,3 +1,17 @@
+# [7.21.0](https://git.datacontroller.io/dc/dc/compare/v7.20.0...v7.21.0) (2026-10-29)
+
+
+### Bug Fixes
+
+* **client:** parse word-date values from VA parameters ([405b28d](https://git.datacontroller.io/dc/dc/commit/405b28dcccfe5994521bf63f0270d4877f544294))
+* **client:** parse word-date values from VA parameters ([ce5b1aa](https://git.datacontroller.io/dc/dc/commit/ce5b1aa96175d0fd1f048fea6d127cbc743804f2))
+* **makedata:** name the sasjs server group in the estate seed ([eeffa92](https://git.datacontroller.io/dc/dc/commit/eeffa92f9e973b323171bbefe2ef1cbc27e3aee7))
+
+
+### Features
+
+* **derived:** apply the core field scan in the MPE_DERIVED hook ([a4a0f9c](https://git.datacontroller.io/dc/dc/commit/a4a0f9cb2f7c0b6252405b71df64aae912d1b075))
+
 # [7.20.0](https://git.datacontroller.io/dc/dc/compare/v7.19.0...v7.20.0) (2026-10-28)
 
 

client/cypress/e2e/download.cy.ts

@@ -40,18 +40,37 @@ context('download files test: ', function () {
   })
 
   it('1 | downloads audit file', (done) => {
-    visitPage('approve/toapprove')
+    /* The approve grid renders a download button per table that is waiting for
+      approval, and a freshly seeded estate has none, so create one: upload a
+      fixture to the test table and submit it from the stage screen.  The button
+      carries the table id, which the stage URL also carries, so the assertion
+      names the row this test created rather than whichever row comes first.
+      The screen is at review/approve; approve/toapprove is a route from before
+      the review module was namespaced and renders the 404 page. */
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
 
-    cy.get('.app-loading', { timeout: longerCommandTimeout })
-      .should('not.exist')
-      .then(() => {
-        cy.get('.btn.btn-success')
-          .should('be.visible')
-          .then((buttons) => {
-            buttons[0].click()
-            const id = buttons[0].id
+    attachExcelFile('regular_excel.xlsx')
 
-            checkForFileDownloaded(id, 'zip', () => done())
+    submitExcel()
+
+    cy.url({ timeout: longerCommandTimeout })
+      .should('include', '/stage/')
+      .then((url) => {
+        const tableId = url.split('/stage/')[1].split(/[?#]/)[0]
+
+        visitPage('review/approve')
+
+        cy.get('.app-loading', { timeout: longerCommandTimeout })
+          .should('not.exist')
+          .then(() => {
+            cy.get(`[aria-label="Download audit file"][id="${tableId}"]`, {
+              timeout: longerCommandTimeout
+            })
+              .should('have.length', 1)
+              .first()
+              .click()
+
+            checkForFileDownloaded(tableId, 'zip', () => done())
           })
       })
   })
@@ -83,7 +102,7 @@ context('download files test: ', function () {
 
     openDownloadModal(() => {
       cy.get('select')
-        .select('Excel')
+        .select('EXCEL')
         .then(() => {
           cy.get('.btn.btn-sm.btn-success-outline').then((button) => {
             button.trigger('click')
@@ -103,7 +122,7 @@ context('download files test: ', function () {
 
     openDownloadModal(() => {
       cy.get('select')
-        .select('SAS Datalines')
+        .select('SAS')
         .then(() => {
           cy.get('.btn.btn-sm.btn-success-outline').then((button) => {
             button.trigger('click')
@@ -123,7 +142,7 @@ context('download files test: ', function () {
 
     openDownloadModal(() => {
       cy.get('select')
-        .select('SAS DDL')
+        .select('SAS_DDL')
         .then(() => {
           cy.get('.btn.btn-sm.btn-success-outline').then((button) => {
             button.trigger('click')
@@ -143,7 +162,7 @@ context('download files test: ', function () {
 
     openDownloadModal(() => {
       cy.get('select')
-        .select('TSQL DDL')
+        .select('TSQL_DDL')
         .then(() => {
           cy.get('.btn.btn-sm.btn-success-outline').then((button) => {
             button.trigger('click')
@@ -163,7 +182,7 @@ context('download files test: ', function () {
 
     openDownloadModal(() => {
       cy.get('select')
-        .select('PGSQL DDL')
+        .select('PGSQL_DDL')
         .then(() => {
           cy.get('.btn.btn-sm.btn-success-outline').then((button) => {
             button.trigger('click')
@@ -209,18 +228,22 @@ const checkForFileDownloaded = (
 }
 
 const openDownloadModal = (callback?: any) => {
-  cy.get('.btn.btn-sm.btn-outline.filterSide.dropdown-toggle')
+  /* filterSide is the viewer's options dropdown; it no longer carries a
+    dropdown-toggle class - see filter-panel.cy.ts and row-cell-limits.cy.ts.
+    Its entries are div[clrDropdownItem], not buttons, so select Download by its
+    text the way filter-panel.cy.ts selects Filter. */
+  cy.get('.btn.btn-sm.btn-outline.filterSide', {
+    timeout: longerCommandTimeout
+  })
     .click()
     .then(() => {
-      cy.get('clr-dropdown-menu button').then((buttons) => {
-        for (let button of buttons) {
-          if (button.innerText.toLowerCase().includes('download')) {
-            button.click()
+      /* the click handler is on the clrDropdownItem element, not on the span
+        holding the text */
+      cy.contains('clr-dropdown-menu [clrDropdownItem]', 'Download', {
+        timeout: longerCommandTimeout
+      }).click()
 
-            if (callback) callback()
-          }
-        }
-      })
+      if (callback) callback()
     })
 }
 
@@ -259,3 +282,39 @@ const openTableFromTree = (libNameIncludes: string, tablename: string) => {
       })
     })
 }
+
+const fixturePath = 'excels'
+
+/**
+ * Uploads a fixture to the open table.  attachFile drives the editor's file
+ * input directly, so no click on the hidden control is needed and no positional
+ * selector is involved.
+ */
+const attachExcelFile = (excelFilename: string) => {
+  cy.get('input[type="file"]#file-upload')
+    .should('have.length', 1)
+    .attachFile(`/${fixturePath}/${excelFilename}`)
+
+  /* an upload raises a confirmation modal; a failure raises the abort modal
+    instead.  Ask the body which one is there rather than waiting on a modal that
+    may never appear, and fail loudly on the abort rather than clicking through
+    the overlay. */
+  cy.get('body').then(($body) => {
+    const abortModal = $body.find('.clr-abort-modal').length > 0
+
+    if (abortModal) {
+      cy.get('.clr-abort-modal .modal-title')
+        .invoke('text')
+        .should('not.include', 'Abort Message')
+      return
+    }
+
+    cy.get('.modal-footer .btn.btn-primary').click()
+  })
+}
+
+const submitExcel = () => {
+  cy.get('.buttonBar button.preview-submit', {
+    timeout: longerCommandTimeout
+  }).click()
+}

client/src/app/deploy/sections/automatic/automatic.component.html

@@ -8,20 +8,6 @@
       @if (autodeployDone) {
         <p class="m-0 align-self-start">Done</p>
         <hr class="w-100" />
-        @if (autoDeployStatus.deployServicePack !== null) {
-          <div class="deploy-status-row">
-            @if (autoDeployStatus.deployServicePack === true) {
-              <clr-icon
-                class="deploy-success"
-                shape="success-standard"
-              ></clr-icon>
-            }
-            @if (!autoDeployStatus.deployServicePack === false) {
-              <clr-icon class="deploy-error" shape="times-circle"></clr-icon>
-            }
-            <p>Deploy SAS Jobs</p>
-          </div>
-        }
         @if (autoDeployStatus.runMakeData !== null) {
           <div class="deploy-status-row">
             @if (autoDeployStatus.runMakeData) {
@@ -33,16 +19,13 @@
             @if (autoDeployStatus.runMakeData === false) {
               <clr-icon class="deploy-error" shape="times-circle"></clr-icon>
             }
-            <p>Create database</p>
+            <p>Create control library</p>
           </div>
         }
         <hr class="w-100" />
         <div class="buttons">
           <button (click)="navigateToHome()" class="btn btn-primary mt-15 mr-0">
-            @if (
-              autoDeployStatus.deployServicePack === false ||
-              autoDeployStatus.runMakeData === false
-            ) {
+            @if (autoDeployStatus.runMakeData === false) {
               <clr-icon class="deploy-error" shape="times-circle"></clr-icon>
             }
             LAUNCH
@@ -78,9 +61,9 @@
 
 <h4 class="text-center my-15">Viya Deploy</h4>
 <p class="deploy-intro">
-  Configure and deploy Data Controller to your Viya environment. Deployment
-  creates the backend SAS services and the control library used by the
-  application.
+  Configure Data Controller for your Viya environment. The frontend and services
+  are already in place; this runs the one-off job that creates the control
+  library and its configuration data, then removes itself.
 </p>
 <hr />
 
@@ -248,19 +231,8 @@
   </div>
 }
 
-<!-- Keeping this for a reference in case future VIYA changes and starts allowing separate backend and frontend) -->
-
-<!-- <clr-checkbox-wrapper>
-  <input
-    clrCheckbox
-    [(ngModel)]="recreateDatabase"
-    (click)="recreateDatabaseClicked($event)"
-    type="checkbox"
-    checked
-    />
-  <label>Recreate database</label>
-</clr-checkbox-wrapper> -->
-
+<!-- Runs the one-off makedata job. This is the screen's only action: the
+     frontend and services are already deployed. -->
 <hr />
 
 <button
@@ -269,53 +241,3 @@
 >
   Deploy
 </button>
-
-<!-- Keeping this for a reference in case future VIYA changes and starts allowing separate backend and frontend) -->
-
-<!-- <button
-(click)="executeJson()"
-class="btn-autodeploy btn btn-primary d-inline-block mr-10"
-[disabled]="!jsonFile"
->
-Deploy {{ !jsonFile ? '(json file is not available)' : '' }}
-</button> -->
-
-<!-- <button
-(click)="uploadJsonAuto.click()"
-class="btn-autodeploy btn btn-primary d-inline-block mr-10"
->
-Upload different file to deploy
-</button>
-<input
-  #uploadJsonAuto
-  type="file"
-  hidden
-  (click)="clearUploadInput($event)"
-  (change)="onJsonFileChange($event)"
-  /> -->
-
-<clr-modal [(clrModalOpen)]="recreateDatabaseModal" [clrModalClosable]="false">
-  <h3 class="modal-title">Warning</h3>
-  <div class="modal-body">
-    This action will recreate the database (if it exists). For an initial
-    deployment, this is expected. If this is a re-deployment, you will lose any
-    existing tables in <strong>{{ dcPath }}</strong
-    >.
-  </div>
-  <div class="modal-footer">
-    <button
-      type="button"
-      class="btn btn-sm btn-primary"
-      (click)="recreateDatabaseModal = false; recreateDatabase = false"
-    >
-      Cancel
-    </button>
-    <button
-      type="button"
-      class="btn btn-sm btn-primary"
-      (click)="recreateDatabaseModal = false; recreateDatabase = true"
-    >
-      Ok
-    </button>
-  </div>
-</clr-modal>

client/src/app/deploy/sections/automatic/automatic.component.spec.ts

@@ -0,0 +1,153 @@
+import { SASjsConfig } from '@sasjs/adapter'
+import { AutomaticComponent } from './automatic.component'
+
+/**
+ * The Viya setup screen has one job: run `makedata` with the selections the
+ * form collected - the admin group, the DC path, and the compute context the
+ * job runs in. These tests pin that payload and the new-window URL, because a
+ * screen whose only action silently stopped calling the service would still
+ * compile, still render, and still look like it worked.
+ */
+describe('AutomaticComponent (Viya setup)', () => {
+  const build = () => {
+    const requests: any[][] = []
+
+    const sasJs: any = {
+      request: (...args: any[]) => {
+        requests.push(args)
+        return Promise.resolve({ result: [{}] })
+      }
+    }
+
+    const sasService: any = {
+      getSasjsConfig: () => new SASjsConfig(),
+      getExecutionPath: () => '/SASJobExecution/',
+      shouldLogin: { next: () => undefined },
+      viyaMakedataSuccessfull: () => Promise.resolve(true),
+      updateFileContent: () => Promise.resolve()
+    }
+
+    const component = new AutomaticComponent(
+      {
+        showAbortModal: () => undefined,
+        showInfoModal: () => undefined
+      } as any,
+      { downloadFile: () => undefined } as any,
+      sasService,
+      {} as any,
+      { error: () => undefined } as any,
+      {} as any
+    )
+
+    component.sasJs = sasJs
+    const config = {
+      contextName: 'Compute Reusable',
+      serverUrl: 'https://viya.example.com',
+      appLoc: '/Users/demo/dc'
+    } as any
+    component.sasJsConfig = config
+    // The new-window URL is built from the adapter config the component holds,
+    // while the request path reads the @Input one.
+    component.sasjsConfig = config
+    component.appLoc = '/Users/demo/dc'
+    component.dcPath = '/export/dc'
+    component.selectedAdminGroup = 'viyagroup18'
+
+    return { component, requests, sasService }
+  }
+
+  it('posts makedata with the admin group and the DC path', async () => {
+    const { component, requests } = build()
+    component.deployInNewWindow = false
+
+    await component.createDatabase()
+
+    expect(requests[0][0]).toBe('services/admin/makedata')
+    expect(requests[0][1]).toEqual({
+      fromjs: [{ ADMIN: 'viyagroup18', DCPATH: '/export/dc' }]
+    })
+  })
+
+  it('runs the job in the configured context, on the web method', async () => {
+    const { component, requests } = build()
+    component.deployInNewWindow = false
+
+    await component.createDatabase()
+
+    expect(requests[0][2]).toEqual({
+      useComputeApi: null,
+      contextName: 'Compute Reusable',
+      debug: true
+    })
+  })
+
+  it('prefers the context chosen in the form over the configured one', async () => {
+    const { component, requests } = build()
+    component.deployInNewWindow = false
+    component.computeContexts = [
+      { id: 'ctx-1', name: 'Selected Context' } as any
+    ]
+    component.selectedComputeContext = 'ctx-1'
+
+    await component.createDatabase()
+
+    expect(requests[0][2].contextName).toBe('Selected Context')
+  })
+
+  it('opens makedata in a new window with the selections as URL parameters', () => {
+    const { component } = build()
+    const opened: string[] = []
+    spyOn(window, 'open').and.callFake((url?: any) => {
+      opened.push(String(url))
+      return null
+    })
+    spyOn(component as any, 'pollMakedataCompletion')
+    component.deployInNewWindow = true
+
+    component.createDatabase()
+
+    expect(opened.length).toBe(1)
+    expect(opened[0]).toContain('/Users/demo/dc/services/admin/makedata')
+    expect(opened[0]).toContain('_contextname=Compute%20Reusable')
+    expect(opened[0]).toContain('admin=viyagroup18')
+    expect(opened[0]).toContain('dcpath=%2Fexport%2Fdc')
+    expect(opened[0]).toContain('_debug=131')
+  })
+
+  it('percent-encodes the values that need it in the new-window URL', () => {
+    // A context name or admin group can hold characters that would otherwise
+    // split the query string - an ampersand would start a new parameter and a
+    // space would arrive truncated.
+    const { component } = build()
+    const opened: string[] = []
+    spyOn(window, 'open').and.callFake((url?: any) => {
+      opened.push(String(url))
+      return null
+    })
+
+    component.runMakedataInNewWindow({
+      contextName: 'SAS Job Execution & co',
+      admin: 'group 1+2',
+      dcPath: '/export/dc data'
+    })
+
+    expect(opened[0]).toContain('_contextname=SAS%20Job%20Execution%20%26%20co')
+    expect(opened[0]).toContain('admin=group%201%2B2')
+    expect(opened[0]).toContain('dcpath=%2Fexport%2Fdc%20data')
+    // the parameter names themselves are not encoded
+    expect(opened[0]).toContain('&admin=group%201%2B2')
+  })
+
+  it('starts the job when deploy is pressed', () => {
+    // The screen's only button calls this; a version where it stopped doing so
+    // would leave the setup page unable to do anything at all.
+    const { component } = build()
+    const spy = spyOn(component, 'createDatabase').and.returnValue(
+      Promise.resolve() as any
+    )
+
+    component.runAutoDeploy()
+
+    expect(spy).toHaveBeenCalled()
+  })
+})

client/src/app/deploy/sections/automatic/automatic.component.ts

@@ -51,17 +51,8 @@ export class AutomaticComponent implements OnInit {
 
   public selectedComputeContext: string = ''
   public makeDataResponse: string = ''
-  public jsonFile: any = null
   public autodeploying: boolean = false
   public autodeployDone: boolean = false
-  public recreateDatabaseModal: boolean = false
-  public isSubmittingJson: boolean = false
-  public isJsonSubmitted: boolean = false
-  /**
-   * Default was `false` when deploy was done with frontend and backend separately.
-   * Now we are using only streaming app, so we always want to recreate database (makedata)
-   */
-  public recreateDatabase: boolean = true
   public createDatabaseLoading: boolean = false
   public adminGroupsLoading: boolean = false
   public currentUserInfoLoading: boolean = false
@@ -86,18 +77,14 @@ export class AutomaticComponent implements OnInit {
   public currentUserInfo: ViyaApiCurrentUser | null = null
   public computeContexts: ComputeContextItem[] = []
 
-  /** autoDeployStatus
-   * This object presents the status for two steps that we have for deploy.
-   * `deployServicePack` - Creating services based on `viya.json`
-   * `runMakeData` - Running `makedata` service
-   * If any of them is `null` or `false` that means step failed
-   * and will be shown to user on deploy done modal.
+  /**
+   * Status of the one step this screen runs: `makedata`, which creates the
+   * control library and its configuration data and then deletes itself.
+   * `null` means it has not run yet, `false` that it failed.
    */
   public autoDeployStatus: {
-    deployServicePack: any
     runMakeData: any
   } = {
-    deployServicePack: null,
     runMakeData: null
   }
 
@@ -420,61 +407,15 @@ export class AutomaticComponent implements OnInit {
   }
 
   /**
-   * Executes sas.json file to deploy the backend
-   * Method will first try to run the `auto deploy`
-   * If that fails the rest of the code is ignored.
-   * If request is successfull, method will continue to try
-   * to create database if checkbox is toggled on
+   * Runs the deployment. The control library and its configuration data are
+   * created by `makedata`, which self-deletes once it has run; the frontend and
+   * services arrived with the sasjs deploy, so there is nothing else to deploy
+   * from here.
    */
-  public async executeJson() {
-    this.isSubmittingJson = true
-
-    try {
-      let uploadJsonFile = await this.sasJs.deployServicePack(
-        this.jsonFile,
-        this.dcAdapterSettings?.appLoc,
-        undefined,
-        undefined,
-        true
-      )
-
-      this.autoDeployStatus.deployServicePack = true
-      this.isJsonSubmitted = true
-    } catch (ex: any) {
-      let textEx = ''
-
-      if (typeof ex.message !== 'string') {
-        textEx = JSON.stringify(ex).replace(/\\/gm, '')
-      } else {
-        textEx = ex.message
-      }
-
-      this.autoDeployStatus.deployServicePack = false
-      this.eventService.showInfoModal(
-        'Deploy error',
-        `Exception: \n ${textEx !== '' ? textEx : ex}`
-      )
-      this.autodeploying = false
-      this.autodeployDone = false
-
-      return
-    }
-
-    this.isSubmittingJson = false
-  }
-
-  public async runAutoDeploy(executeJson: boolean = false) {
+  public runAutoDeploy() {
     if (!this.deployInNewWindow) this.autodeploying = true
 
-    if (executeJson) {
-      this.executeJson()
-    }
-
-    if (this.recreateDatabase) {
-      this.createDatabase()
-    } else {
-      if (!this.deployInNewWindow) this.autodeployDone = true
-    }
+    this.createDatabase()
   }
 
   /**
@@ -578,9 +519,9 @@ export class AutomaticComponent implements OnInit {
     let serverUrl = this.sasjsConfig.serverUrl
     let appLoc = this.sasjsConfig.appLoc
     const execPath = this.sasService.getExecutionPath()
-    let contextname = `&_contextname=${params.contextName}`
-    let admin = `&admin=${params.admin}`
-    let dcPath = `&dcpath=${params.dcPath}`
+    let contextname = `&_contextname=${encodeURIComponent(params.contextName)}`
+    let admin = `&admin=${encodeURIComponent(params.admin)}`
+    let dcPath = `&dcpath=${encodeURIComponent(params.dcPath)}`
     // Debug is ALWAYS enabled for makedata — it runs only once during
     // deployment and the full log is needed to diagnose any issues.
     let debug = '&_debug=131'
@@ -708,22 +649,6 @@ export class AutomaticComponent implements OnInit {
     this.deployService.downloadFile(content, filename, extension)
   }
 
-  public async onJsonFileChange(event: any) {
-    let file = event.target.files[0]
-
-    this.jsonFile = await this.deployService.readFile(file)
-  }
-
-  public recreateDatabaseClicked(event: Event) {
-    ;(<HTMLInputElement>event.target).checked === true
-      ? (this.recreateDatabaseModal = true)
-      : ''
-  }
-
-  public clearUploadInput(event: Event) {
-    this.deployService.clearUploadInput(event)
-  }
-
   public openSasRequestsModal() {
     this.eventService.openRequestsModal()
   }

client/src/app/deploy/sections/sasjs-configurator/sasjs-configurator.component.spec.ts

@@ -0,0 +1,88 @@
+import { ServerType } from '@sasjs/utils/types/serverType'
+import { SasjsConfiguratorComponent } from './sasjs-configurator.component'
+
+/**
+ * The SAS9 / SASjs Server setup screen cannot deploy anything: it builds the
+ * one-off `makedata` URL and opens it, with the admin group and the directory
+ * the control library will live in. These tests pin the URL shape, because a
+ * wrong parameter name or a lost encode still produces a page that opens and
+ * merely fails inside SAS, where nobody is looking.
+ */
+describe('SasjsConfiguratorComponent (SAS9 / SASjs setup)', () => {
+  const build = (serverType: ServerType) => {
+    const sasService: any = {
+      getExecutionPath: () => '/SASStoredProcess/do',
+      sasjsMakedataChecking: () => Promise.resolve(false),
+      logout: () => undefined
+    }
+
+    const component = new SasjsConfiguratorComponent(
+      sasService,
+      { getServerInfo: () => ({ subscribe: () => undefined }) } as any,
+      { replaceState: () => undefined } as any
+    )
+
+    component.sasJsConfig = {
+      serverType,
+      serverUrl: 'https://sas9.example.com',
+      appLoc: '/User Folders/dc'
+    } as any
+    component.dcDirectory = '/tmp/DataController'
+    component.dcAdminGroup = 'DCADMINS'
+
+    const opened: string[] = []
+    spyOn(window, 'open').and.callFake((url?: any) => {
+      opened.push(String(url))
+      return null
+    })
+
+    return { component, opened, sasService }
+  }
+
+  it('opens makedata on SAS9 with the admin group and the directory', () => {
+    const { component, opened } = build(ServerType.Sas9)
+
+    component.makeData()
+
+    expect(opened.length).toBe(1)
+    expect(opened[0]).toContain(
+      '_program=/User Folders/dc/services/admin/makedata'
+    )
+    expect(opened[0]).toContain('admin=DCADMINS')
+    expect(opened[0]).toContain('path=%2Ftmp%2FDataController')
+    expect(opened[0]).toContain('_debug=131')
+  })
+
+  it('stays on the page origin for SAS9 rather than the configured server', () => {
+    const { component, opened } = build(ServerType.Sas9)
+
+    component.makeData()
+
+    // A SAS9 deployment is reached through the URL the user is already on; the
+    // configured serverUrl would point the browser at a host it may not reach.
+    expect(opened[0].startsWith(window.location.origin)).toBe(true)
+    expect(opened[0]).not.toContain('sas9.example.com')
+  })
+
+  it('keeps the directory readable for SASjs Server', () => {
+    const { component, opened } = build(ServerType.Sasjs)
+
+    component.makeData()
+
+    // The SASjs branch does not encode the path, so a plain directory stays as
+    // written; encoding it there would make makedata look for a folder with %2F
+    // in its name.
+    expect(opened[0]).toContain('path=/tmp/DataController')
+  })
+
+  it('checks the SASjs Server deployment before reloading the app', () => {
+    const { component, sasService } = build(ServerType.Sasjs)
+    const spy = spyOn(sasService, 'sasjsMakedataChecking').and.returnValue(
+      Promise.resolve(false)
+    )
+
+    component.makeData()
+
+    expect(spy).toHaveBeenCalled()
+  })
+})

client/src/app/editor/editor.component.ts

@@ -3647,7 +3647,16 @@ export class EditorComponent
     if (clauses.length === 0) return false
 
     try {
-      const res: any = await this.sasStoreService.saveQuery(libds, clauses)
+      // A `va_parmap` entry is a column the report author typed, so this
+      // request can legitimately fail: the table may not have that column, and
+      // validatefilter aborts on an unknown variable (dc/dc#404). Attempting it
+      // is still worth it - when the mapping is right the very first getdata is
+      // already filtered - but the failure must not stop the page. An abort
+      // modal here covers an editor that is about to load fine; the mapping
+      // banner names the bad entry once the real metadata arrives.
+      const res: any = await this.sasStoreService.saveQuery(libds, clauses, {
+        suppressErrorAbortModal: true
+      })
       const id = res?.result?.[0]?.FILTER_RK
       const table = res?.result?.[0]?.FILTER_TABLE
       if (id === undefined || table === undefined) return false

client/src/app/services/deploy.service.ts

@@ -26,22 +26,4 @@ export class DeployService {
       document.body.removeChild(downloadLink)
     }
   }
-
-  public readFile(file: File): Promise<any> {
-    return new Promise((resolve, reject) => {
-      let JsonFileReader = new FileReader()
-
-      JsonFileReader.onload = () => {
-        if (JsonFileReader.result) {
-          resolve(JSON.parse(JsonFileReader.result.toString()))
-        }
-      }
-
-      JsonFileReader.readAsText(file)
-    })
-  }
-
-  public clearUploadInput(event: Event) {
-    ;(<HTMLInputElement>event.target).value = ''
-  }
 }

client/src/app/services/sas-store.service.spec.ts

@@ -116,6 +116,53 @@ describe('SasStoreService', () => {
       expect(seen.length).toBe(0)
     })
   })
+
+  describe('saveQuery', () => {
+    const capture = (adapterResponse: any = { result: [] }) => {
+      const calls: any[][] = []
+      const store = new SasStoreService(
+        {
+          request: (...args: any[]) => {
+            calls.push(args)
+            return Promise.resolve({ adapterResponse })
+          }
+        } as any,
+        {} as any,
+        {} as any
+      )
+      return { store, calls }
+    }
+
+    it('posts the filter to public/validatefilter', async () => {
+      const { store, calls } = capture()
+      await store.saveQuery('DC.MY_TABLE', [
+        { VARIABLE_NM: 'A', OPERATOR_NM: '=', RAW_VALUE: "'x'" } as any
+      ])
+
+      expect(calls[0][0]).toBe('public/validatefilter')
+      expect(calls[0][1].iwant[0].filter_table).toBe('DC.MY_TABLE')
+      expect(calls[0][1].filterquery.length).toBe(1)
+    })
+
+    it('passes wrapper options through, so a rejected filter can stay quiet', async () => {
+      // The VA deferred load attempts a filter built from a va_parmap entry the
+      // table may not have, and must not cover a page that loads fine with an
+      // abort modal (dc/dc#404).
+      const { store, calls } = capture()
+      await store.saveQuery('DC.MY_TABLE', [], {
+        suppressErrorAbortModal: true
+      })
+
+      expect(calls[0][3]).toEqual({ suppressErrorAbortModal: true })
+    })
+
+    it('sends no wrapper options when the caller gives none', async () => {
+      const { store, calls } = capture()
+      await store.saveQuery('DC.MY_TABLE', [])
+
+      expect(calls[0][3]).toBeUndefined()
+    })
+  })
 })
 
 /**

client/src/app/services/sas-store.service.ts

@@ -8,6 +8,7 @@ import { HelperService } from './helper.service'
 import { QueryClause } from '../models/TableData'
 import { globals } from '../_globals'
 import { FilterClause, FilterGroup, FilterQuery } from '../models/FilterQuery'
+import { RequestWrapperOptions } from '../models/request-wrapper/RequestWrapperOptions'
 import {
   $DataFormats,
   EditorsGetDataSASResponse,
@@ -395,13 +396,30 @@ export class SasStoreService {
     ).adapterResponse
   }
 
-  public async saveQuery(libds: string, filterQuery: QueryClause[]) {
+  /**
+   * Persists a filter and returns its FILTER_RK.
+   *
+   * `wrapperOptions` reaches the request wrapper, so a caller that can live with
+   * a rejected filter (the VA deferred load, whose `va_parmap` entry may name a
+   * column the table does not have) can suppress the abort modal that would
+   * otherwise cover a page which is about to load perfectly well.
+   */
+  public async saveQuery(
+    libds: string,
+    filterQuery: QueryClause[],
+    wrapperOptions?: RequestWrapperOptions
+  ) {
     let tables = {
       iwant: [{ filter_table: libds }],
       filterquery: filterQuery
     }
 
-    const res = await this.sasService.request('public/validatefilter', tables)
+    const res = await this.sasService.request(
+      'public/validatefilter',
+      tables,
+      undefined,
+      wrapperOptions
+    )
 
     this.filter.next(res)
 

client/src/app/services/va-filter.service.spec.ts

@@ -131,6 +131,41 @@ describe('VaFilterService', () => {
       expect(build([{ label: 'SOME_DATE', value: '2025/03/31' }])).toHaveSize(1)
     })
 
+    it('parses word dates, in either order and with an optional comma', () => {
+      const expected = "'01JAN2026'd"
+      // The reported shape: a date parameter displayed as WORDDATE.
+      expect(
+        build([{ label: 'SOME_DATE', value: 'January 1, 2026' }])[0].RAW_VALUE
+      ).toBe(expected)
+      expect(
+        build([{ label: 'SOME_DATE', value: 'Jan 1, 2026' }])[0].RAW_VALUE
+      ).toBe(expected)
+      expect(
+        build([{ label: 'SOME_DATE', value: 'Jan. 1, 2026' }])[0].RAW_VALUE
+      ).toBe(expected)
+      expect(
+        build([{ label: 'SOME_DATE', value: '1 January 2026' }])[0].RAW_VALUE
+      ).toBe(expected)
+      expect(
+        build([{ label: 'SOME_DATE', value: '1-Jan-2026' }])[0].RAW_VALUE
+      ).toBe(expected)
+    })
+
+    it('matches a spelled-out month case-insensitively, and keeps DATE9 working', () => {
+      expect(
+        build([{ label: 'SOME_DATE', value: 'DECEMBER 31, 2025' }])[0].RAW_VALUE
+      ).toBe("'31DEC2025'd")
+      expect(
+        build([{ label: 'SOME_DATE', value: '01AUG1962' }])[0].RAW_VALUE
+      ).toBe("'01AUG1962'd")
+    })
+
+    it('still drops a month-name value that has no day', () => {
+      // A month and year alone is not a date - accepted shapes always carry a
+      // day, so this must not silently become the first of the month.
+      expect(build([{ label: 'SOME_DATE', value: 'January 2026' }])).toEqual([])
+    })
+
     it('resolves ambiguous slash dates day-first (never US), disambiguating by value', () => {
       const iso = build([{ label: 'SOME_DATE', value: '2025-03-31' }])[0]
         .RAW_VALUE

client/src/app/services/va-filter.service.ts

@@ -707,14 +707,42 @@ export class VaFilterService {
     return this.parseMonthNameDate(value) ?? this.parseNumericDate(value, order)
   }
 
-  /** DATE9 "ddMMM[yy]yy" (e.g. 01AUG1962, 01AUG62) -> local Date, or null. */
+  /**
+   * Month-name date shapes -> local Date, or null. Three families:
+   *  - DATE9 "ddMMM[yy]yy" (e.g. 01AUG1962, 01AUG62);
+   *  - word date with the month first, comma optional: "January 1, 2026",
+   *    "Jan 1 2026", "Jan. 1, 2026";
+   *  - word date with the day first: "1 January 2026", "1-Jan-2026".
+   *
+   * A date parameter's display format decides which of these the report author
+   * sees, and VA sends the formatted text, so a report using WORDDATE (or a
+   * long month name in any order) has to parse here rather than being dropped.
+   * The month name may be abbreviated or spelled out; only its first three
+   * letters are matched.
+   */
   private parseMonthNameDate(value: string): Date | null {
-    const m = /^(\d{1,2})([A-Za-z]{3})(\d{2}|\d{4})$/.exec(value)
-    if (!m) return null
-    const [, day, monthAbbr, year] = m
-    const monthIndex = VaFilterService.SAS_MONTHS.indexOf(
-      monthAbbr.toUpperCase()
-    )
+    const monthByName = (name: string): number =>
+      VaFilterService.SAS_MONTHS.indexOf(
+        name.replace(/\./g, '').slice(0, 3).toUpperCase()
+      )
+
+    const date9 = /^(\d{1,2})([A-Za-z]{3})(\d{2}|\d{4})$/.exec(value)
+    if (date9) {
+      const [, day, monthAbbr, year] = date9
+      const monthIndex = monthByName(monthAbbr)
+      if (monthIndex < 0) return null
+      return this.buildDate(this.fullYear(year), monthIndex + 1, Number(day))
+    }
+
+    const monthFirst =
+      /^([A-Za-z]{3,9})\.?[\s-]+(\d{1,2}),?[\s-]+(\d{2}|\d{4})$/.exec(value)
+    const dayFirst =
+      /^(\d{1,2})[\s-]+([A-Za-z]{3,9})\.?,?[\s-]+(\d{2}|\d{4})$/.exec(value)
+    const name = monthFirst ? monthFirst[1] : dayFirst ? dayFirst[2] : null
+    if (!name) return null
+    const day = monthFirst ? monthFirst[2] : dayFirst![1]
+    const year = monthFirst ? monthFirst[3] : dayFirst![3]
+    const monthIndex = monthByName(name)
     if (monthIndex < 0) return null
     return this.buildDate(this.fullYear(year), monthIndex + 1, Number(day))
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "7.20.0",
+  "version": "7.21.0",
   "description": "Data Controller",
   "devDependencies": {
     "@nogoo9/gitleaks": "8.30.1-post.2",

sas/deploy/makeDataServer.json

@@ -1,7 +1,7 @@
 {
   "fromjs": [
     {
-      "ADMIN": "AllUsers",
+      "ADMIN": "all-users",
       "DCPATH": "/tmp/dcdata",
       "OUTFMT": "JSON"
     }

sas/mocks/sasjs/services/hooks/mpe_derived_postedit.js

@@ -13,10 +13,12 @@
 // The rules mirror services/hooks/mpe_derived_postedit.sas, in its order:
 // normalise the incoming rows, reject a duplicate name and a bad field, then
 // check the dependency graph that the resulting configuration would have (the
-// incoming rows plus the current rows they do not replace).  As on the server,
-// the collision between VAR_NM and a physical column is left to the read-time
-// engine, which runs with the target library assigned - the hook does not have
-// it here either.
+// incoming rows plus the current rows they do not replace).  The field rules
+// are core's mp_derivedvarcheck, which the server hook calls and the read-time
+// engine runs too - this file spells them out a second time, so a change there
+// has to be mirrored here.  As on the server, the collision between VAR_NM and
+// a physical column is left to the read-time engine, which runs with the target
+// library assigned - the hook does not have it here either.
 // ─────────────────────────────────────────────────────────────────────────────
 
 ;(function mpeDerivedPostedit() {
@@ -83,9 +85,45 @@
         seen.add(key)
     }
 
-    // ─── Field level checks, the rules the engine applies at read time ─────────
-    // C0 controls and DEL: what a single-line label cannot carry
+    // ─── Field level checks ────────────────────────────────────────────────────
+    // Two passes, in the order the SAS hook runs them.  The first is the fields
+    // this hook owns - the control table key, which the loader normalises but
+    // does not require, and ACTIVE, the picker pair the EDIT screen offers.
+    for (const row of rows) {
+        if (isDeleted(row)) continue
+        const name = field(row, 'VAR_NM')
+        let reason = ''
+        if (!field(row, 'LIBREF') || !field(row, 'DSN'))
+            reason = 'LIBREF and DSN are required'
+        // ACTIVE is a picker in the EDIT screen (HARDSELECT over the
+        // MPE_SELECTBOX pair) - the same pair the hook enforces
+        else if (!['0', '1'].includes(String(colVal(row, 'ACTIVE')).trim()))
+            reason = 'ACTIVE must be 1 or 0'
+        if (reason) return reject(`MPE_DERIVED: ${name}: ${reason}`)
+    }
+
+    // The second pass is the rules for the derived variable itself, which are
+    // core's (mp_derivedvarcheck): the engine runs the same macro at read time,
+    // so a definition is measured against one rule list wherever it is checked.
+
+    // C0 controls and DEL: what a label cannot carry once a line break has been
+    // flattened to a space, which is what the engine does with it
     const CONTROL_CHARS = /[\u0000-\u001F\u007F]/
+    const flatten = (v) => v.replace(/[\r\n]+/g, ' ')
+    const AUTOMATIC_VARS = ['_N_', '_ERROR_', '_IORC_']
+
+    // the capabilities refused by name - a function name cannot be assembled at
+    // run time, so naming them is a control the expression cannot evade
+    const DENY_FUNCS = new RegExp(
+        '\\b(DOSUBL|JAVAOBJ|JAVAMETA|SYMGET|SYMPUTX?|SYSGET|GETOPTION' +
+            '|PATHNAME|FILEEXIST|FOPEN|FREAD|FGET|FCLOSE' +
+            '|DOPEN|DREAD|DNUM|MOPEN|SLEEP' +
+            '|FILENAME|FDELETE|FCOPY|DCREATE|DLCREATE)\\s*\\('
+    )
+
+    // quoted literals are data, not calls
+    const unquoted = (v) =>
+        v.replace(/'([^']|'')*'/g, ' ').replace(/"([^"]|"")*"/g, ' ')
 
     // true when the value carries a character outside the allowed set
     const hasInvalidChar = (value, chars) =>
@@ -93,8 +131,6 @@
 
     for (const row of rows) {
         if (isDeleted(row)) continue
-        const libref = field(row, 'LIBREF')
-        const dsn = field(row, 'DSN')
         const name = field(row, 'VAR_NM')
         const type = field(row, 'VAR_TYPE')
         const len = Number(colVal(row, 'VAR_LEN'))
@@ -103,19 +139,16 @@
         const logic = strip(colVal(row, 'VAR_LOGIC'))
 
         let reason = ''
-        if (!libref || !dsn) reason = 'LIBREF and DSN are required'
-        else if (!name) reason = 'VAR_NM is required'
+        if (!name) reason = 'VAR_NM is required'
         else if (hasInvalidChar(name, NAME_CHARS))
             reason = 'VAR_NM must be a valid SAS name'
         else if ('0123456789'.includes(name[0]))
             reason = 'VAR_NM must not start with a digit'
+        else if (AUTOMATIC_VARS.includes(name))
+            reason = 'VAR_NM is an automatic variable of the data step'
         else if (name === DELETE_MARK) reason = 'VAR_NM is reserved'
         else if (type !== 'C' && type !== 'N')
             reason = 'VAR_TYPE must be C or N'
-        // ACTIVE is a picker in the EDIT screen (HARDSELECT over the
-        // MPE_SELECTBOX pair) - the same pair mpe_derived_check.sas enforces
-        else if (!['0', '1'].includes(String(colVal(row, 'ACTIVE')).trim()))
-            reason = 'ACTIVE must be 1 or 0'
         else if (!Number.isInteger(len) || len < 1)
             reason = 'VAR_LEN must be a positive integer'
         else if (type === 'C' && len > 32767)
@@ -132,20 +165,24 @@
             reason = 'VAR_FMT must be a format name, without ; & % or /* */'
         else if (hasInvalidChar(fmt, FMT_CHARS))
             reason = 'VAR_FMT must be a format name'
-        // free text, in any script - what is refused is anything that cannot be
-        // a single-line column label, and a value the engine would cut to 256
-        else if (CONTROL_CHARS.test(label))
-            reason =
-                'VAR_LABEL must be plain text - no line breaks or control characters'
+        // free text, in any script - what is refused is a control character,
+        // and a value the engine would cut to 256
+        else if (CONTROL_CHARS.test(flatten(label)))
+            reason = 'VAR_LABEL must be plain text - no control characters'
         else if ([...label].length > 256)
             reason = 'VAR_LABEL exceeds 256 characters'
         else if (!logic) reason = 'VAR_LOGIC is required'
+        else if (logic.length > 4000)
+            reason = 'VAR_LOGIC exceeds 4000 characters'
         else if (logic.includes(';'))
             reason = 'VAR_LOGIC must be a single expression - remove the ;'
         else if (logic.includes('&') || logic.includes('%'))
             reason = 'VAR_LOGIC must not contain macro triggers'
         else if (logic.includes('/*') || logic.includes('*/'))
             reason = 'VAR_LOGIC must not contain comments'
+        else if (DENY_FUNCS.test(unquoted(logic).toUpperCase()))
+            reason =
+                'VAR_LOGIC may not call a function that submits code or reaches the file system, the environment or the macro symbol table'
 
         if (reason) return reject(`MPE_DERIVED: ${name}: ${reason}`)
     }

sas/mocks/sasjs/services/public/getddl.js

@@ -1,7 +1,8 @@
 let mimeType = `application/octet-stream`
 let filename = `dc-download`
 
-if (libref) filename = `${libref}.ddl`
+if (libref && ds) filename = `${libref}_${ds}.ddl`
+else if (libref) filename = `${libref}.ddl`
 
 fs.writeFile(_SASJS_WEBOUT_HEADERS, `Content-type: ${mimeType} \n Content-Disposition: attachment;filename=\"${filename}\"`, function (err) {
   if (err) throw err;

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "6.2.0",
-        "@sasjs/core": "6.1.0"
+        "@sasjs/core": "7.0.1"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -199,6 +199,12 @@
         "@sasjs/lint": "^4.3.0"
       }
     },
+    "node_modules/@sasjs/cli/node_modules/@sasjs/core": {
+      "version": "6.1.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.1.4.tgz",
+      "integrity": "sha512-vIF2LIftxPWg4CvO3i5raEfrHFHY/6S+MqVGpjRR71dgjLRJEYNyPJgGqD6BCZwHv5V6/2nDnBi/BIIMiLXfzA==",
+      "license": "MIT"
+    },
     "node_modules/@sasjs/cli/node_modules/@sasjs/utils": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-4.1.0.tgz",
@@ -235,9 +241,9 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-6.1.0.tgz",
-      "integrity": "sha512-U0Ekq/TXVMquYV+AG6urBQSUgqUvV37yduufy68QQVgDj3PJiLUJDcOXKOGbhWGD+Nv877VuPCviGYqrqbjXvg==",
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-7.0.1.tgz",
+      "integrity": "sha512-M75475GuOMTmbmrB8V/Hgcwm2j3PdIHlgmkbjtT88ALSQuzRPBAjEK2IDDYIUn6rsVJG9aZWvUpRV7fzrOjK/w==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {

sas/package.json

@@ -34,7 +34,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "6.2.0",
-    "@sasjs/core": "6.1.0"
+    "@sasjs/core": "7.0.1"
   },
   "overrides": {
     "nanoid": "3.3.18"

sas/sasjs/macros/mpe_derived.sas

@@ -49,9 +49,19 @@
   being maintained.  A cycle is rejected.
 
   Semicolons, macro triggers and comments are rejected, so a row can only ever
-  be an expression.  A derived variable may not reuse the name of a physical
-  column, and may not reference itself: an intra-row calculation would imply a
-  retained value, which depends on the order rows happen to arrive in.
+  be an expression, and an expression that would let the row submit code of its
+  own - DOSUBL, or a function reaching the file system, the environment or the
+  macro symbol table - is refused by name.
+
+  The field level rules themselves are core's: mp_derivedvarcheck scans the
+  definitions and mp_derivedvars calls it, which is the same macro the MPE_DERIVED
+  post edit hook runs at submit time (services/hooks/mpe_derived_postedit).  A
+  row is therefore measured against one rule list, whether it is being staged
+  or read.
+
+  A derived variable may not reuse the name of a physical column, and may not
+  reference itself: an intra-row calculation would imply a retained value,
+  which depends on the order rows happen to arrive in.
 
   <h4> SAS Macros </h4>
   @li mp_derivedvars.sas

sas/sasjs/macros/mpe_derived_check.sas

@@ -1,75 +0,0 @@
-/**
-  @file
-  @brief Field level checks for a staged MPE_DERIVED submission
-  @details Writes one row per staged row that breaks a rule, with the reason in
-  REASON, so a caller can report the first failure and abort.  Rows marked for
-  deletion are skipped - they are being removed, so their fields are not loaded.
-
-  The rules are the ones %mpe_derived applies at read time, so a submission
-  cannot stage a row the engine would then refuse.
-
-  @param inds the staged rows, already normalised by the caller
-  @param outds the dataset to create: one row per failing row, plus REASON
-
-  @author 4GL Apps Ltd
-  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
-    and may not be re-distributed or re-sold without the express permission of
-    4GL Apps Ltd.
-
-**/
-
-%macro mpe_derived_check(inds=work.staging_ds, outds=work.derived_check);
-data &outds;
-  set &inds;
-  length reason $200;
-  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
-  call missing(reason);
-
-  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
-  else if var_nm='' then reason='VAR_NM is required';
-  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
-    then reason='VAR_NM must be a valid SAS name';
-  else if verify(substr(var_nm,1,1),'0123456789')=0
-    then reason='VAR_NM must not start with a digit';
-  else if var_nm='_____DELETE__THIS__RECORD_____'
-    then reason='VAR_NM is reserved';
-  else if var_type not in ('C','N')
-    then reason='VAR_TYPE must be C or N';
-  /* ACTIVE is a picker in the EDIT screen (HARDSELECT over the MPE_SELECTBOX
-    pair) - the same pair is enforced here, so a submission that bypasses the
-    grid cannot land a value the screen would never offer. */
-  else if missing(active) or active not in (0,1)
-    then reason='ACTIVE must be 1 or 0';
-  else if missing(var_len) or var_len ne int(var_len) or var_len<1
-    then reason='VAR_LEN must be a positive integer';
-  else if var_type='C' and var_len>32767
-    then reason='VAR_LEN exceeds 32767 for a character variable';
-  else if var_type='N' and var_len>8
-    then reason='VAR_LEN exceeds 8 for a numeric variable';
-  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
-    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
-    then reason='VAR_FMT must be a format name, without ; & % or /* */';
-  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
-    then reason='VAR_FMT must be a format name';
-  /* VAR_LABEL is free text: any character is allowed, in any script, and the
-    quotes in it are escaped by %mp_derivedvars when the attrib statement is
-    built.  What is refused is anything that cannot be a single-line column
-    label - a control character or a line break - and a value long enough to
-    be cut down silently, since the engine truncates the label to 256. */
-  else if anycntrl(var_label)>0
-    then reason='VAR_LABEL must be plain text - no line breaks or control characters';
-  else if klength(var_label)>256
-    then reason='VAR_LABEL exceeds 256 characters';
-  else if missing(var_logic)
-    then reason='VAR_LOGIC is required';
-  else if index(var_logic,';')>0
-    then reason='VAR_LOGIC must be a single expression - remove the ;';
-  else if indexc(var_logic,'&%')>0
-    then reason='VAR_LOGIC must not contain macro triggers';
-  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
-    then reason='VAR_LOGIC must not contain comments';
-
-  if not missing(reason) then output;
-run;
-
-%mend mpe_derived_check;

sas/sasjs/macros/mpe_derived_check.test.sas

@@ -1,194 +0,0 @@
-/**
-  @file
-  @brief testing the mpe_derived_check macro
-
-  @details Checks each field level rule the MPE_DERIVED post edit hook applies
-  to a staged submission.  The macro is exercised directly so the rules can be
-  probed with values a service cannot carry: VAR_LABEL is varchar(256) in the
-  table and a staging dataset declares the same length, so a longer label is
-  truncated before any check would see it.
-
-  <h4> SAS Macros </h4>
-  @li mf_nobs.sas
-  @li mp_assert.sas
-  @li mpe_derived_check.sas
-
-  @author 4GL Apps Ltd
-  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
-    and may not be re-distributed or re-sold without the express permission of
-    4GL Apps Ltd.
-
-**/
-
-/**
-  * Test 1 - one row per failing rule, and only failing rows
-  */
-data work.staging_ds;
-  length libref $8 dsn $32 var_nm $32 var_type $1 var_len 8
-    var_fmt $32 var_label $4000 var_deps $4000 var_logic $4000
-    active 8 _____DELETE__THIS__RECORD_____ $3;
-  libref='DCTEST';
-  dsn='DC_DERIV';
-  /* valid unless a row says otherwise - ACTIVE is mandatory in the screen and
-    the check enforces the same pair here */
-  active=1;
-  _____DELETE__THIS__RECORD_____='No';
-
-  /* meets every rule */
-  var_nm='GOOD';
-  var_type='N';
-  var_len=8;
-  var_fmt='';
-  var_label='a plain label';
-  var_deps='';
-  var_logic='1';
-  output;
-
-  /* delete marked rows are being removed, so their fields are not loaded */
-  var_nm='DELME';
-  var_label='first line'||'0A'x||'second line';
-  _____DELETE__THIS__RECORD_____='Yes';
-  output;
-  _____DELETE__THIS__RECORD_____='No';
-
-  /* free text is allowed, including quotes and punctuation */
-  var_nm='FREETXT';
-  var_label='It''s "quoted", plain text: 100 percent';
-  output;
-
-  /* a line break cannot be a single line column label */
-  var_nm='LBLBREAK';
-  var_label='first line'||'0A'x||'second line';
-  output;
-
-  /* longer than the table holds, so it would be cut down silently */
-  var_nm='LBLLONG';
-  var_label=repeat('x',300);
-  output;
-
-  /* a format name, not an expression */
-  var_nm='BADFMT';
-  var_fmt='DATE9;';
-  var_label='fmt';
-  output;
-
-  /* one expression, no statement separator, no macro triggers */
-  var_nm='BADLOGIC';
-  var_fmt='';
-  var_logic='x; y';
-  output;
-
-  /* C or N */
-  var_nm='BADTYPE';
-  var_logic='1';
-  var_type='X';
-  output;
-
-  /* ACTIVE is a picker in the screen: only 1 or 0 */
-  var_nm='BADACTIVE';
-  var_type='C';
-  var_logic='1';
-  active=2;
-  output;
-
-  stop;
-run;
-
-%mpe_derived_check(inds=work.staging_ds, outds=work.derived_check)
-
-/**
-  * Test 2 - the reported rows, rule by rule
-  */
-data work.expected;
-  length var_nm $32 reason $200;
-  var_nm='LBLBREAK';
-  reason='VAR_LABEL must be plain text - no line breaks or control characters';
-  output;
-  var_nm='LBLLONG';
-  reason='VAR_LABEL exceeds 256 characters';
-  output;
-  var_nm='BADFMT';
-  reason='VAR_FMT must be a format name, without ; & % or /* */';
-  output;
-  var_nm='BADLOGIC';
-  reason='VAR_LOGIC must be a single expression - remove the ;';
-  output;
-  var_nm='BADTYPE';
-  reason='VAR_TYPE must be C or N';
-  output;
-  var_nm='BADACTIVE';
-  reason='ACTIVE must be 1 or 0';
-  output;
-run;
-
-proc sql;
-create table work.cmp as
-  select e.var_nm
-    ,case when a.reason=e.reason then 1 else 0 end as same
-  from work.expected e
-  left join work.derived_check a
-    on upcase(strip(a.var_nm))=upcase(e.var_nm);
-quit;
-
-data _null_;
-  set work.cmp;
-  call symputx(cats('rule_',var_nm),same,'l');
-run;
-
-%mp_assert(
-  iftrue=(%mf_nobs(work.derived_check)=6),
-  desc=%str(Checking only the six failing rows are reported),
-  outds=work.test_results
-)
-
-%mp_assert(
-  iftrue=(&rule_LBLBREAK=1),
-  desc=%str(Checking a label with a line break is rejected),
-  outds=work.test_results
-)
-
-%mp_assert(
-  iftrue=(&rule_LBLLONG=1),
-  desc=%str(Checking a label over 256 characters is rejected),
-  outds=work.test_results
-)
-
-%mp_assert(
-  iftrue=(&rule_BADFMT=1),
-  desc=%str(Checking a format carrying a statement separator is rejected),
-  outds=work.test_results
-)
-
-%mp_assert(
-  iftrue=(&rule_BADLOGIC=1),
-  desc=%str(Checking a logic value with a statement separator is rejected),
-  outds=work.test_results
-)
-
-%mp_assert(
-  iftrue=(&rule_BADTYPE=1),
-  desc=%str(Checking a VAR_TYPE other than C or N is rejected),
-  outds=work.test_results
-)
-
-%mp_assert(
-  iftrue=(&rule_BADACTIVE=1),
-  desc=%str(Checking an ACTIVE outside 1 or 0 is rejected),
-  outds=work.test_results
-)
-
-/**
-  * Test 3 - a row that meets every rule, a free text label, and a delete
-  * marked row are all left alone
-  */
-proc sql noprint;
-select count(*) into: notchecked
-  from work.derived_check
-  where upcase(strip(var_nm)) in ('GOOD','FREETXT','DELME');
-quit;
-
-%mp_assert(
-  iftrue=(&notchecked=0),
-  desc=%str(Checking a valid row, a free text label and a delete marked row are not reported),
-  outds=work.test_results
-)

sas/sasjs/macros/mpe_execute.sas

@@ -152,7 +152,10 @@
     %mp_abort(
       iftrue=(1=1)
       ,mac=&sysmacroname
-      ,msg=%str(&program did not complete: state=&state &msg)
+      /* %bquote, not %str: the child's message is interpolated here, and a
+        comma in it would otherwise split this call into extra positional
+        parameters before mp_abort ever sees it */
+      ,msg=%bquote(&program did not complete: state=&state &msg)
     )
   %end;
 %end;

sas/sasjs/macros/mpe_makedata.sas

@@ -2355,7 +2355,9 @@ insert into &lib..MPE_VALIDATIONS set
       ,some_char="&x bottles of beer on the wall"
       ,some_dropdown='Option 1'
       ,some_num=ranuni(0)
-      ,some_date=round(ranuni(0)*1000,1)
+      /* First of the month, cycling through two years: a randomly spread date
+      gave every row its own day, so a date filter never returned many rows. */
+      ,some_date=intnx('month','01JAN1960'd,mod(&x,24),'S')
       ,some_datetime=round(ranuni(0)*50000,1)
       ,some_time=round(ranuni(0)*100,1)
       ,some_shortnum=6+round(ranuni(0)*94,1)

sas/sasjs/services/editors/restore.test.2.sas

@@ -0,0 +1,172 @@
+/**
+  @file
+  @brief testing restore of a change to a special missing value
+
+  @details A cell holding a special missing (._ .A-.Z) and changed to an
+    ordinary number must be reverted to that special missing. mp_stripdiffs
+    writes the old value into a generated UPDATE statement, and PUT drops the
+    leading period of a special missing, so the statement used to be
+    "set SOME_BESTNUM =_" - which PROC SQL reads as a column name.
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mx_execute.sas
+
+**/
+
+%let _program=&appLoc/services/editors/restore;
+
+/* seed the base table with a special missing - the audit stores whatever the
+  base holds, so this is the value the revert has to write back */
+proc sql noprint;
+update &dclib..mpe_x_test set some_bestnum=._
+  where primary_key_field=3;
+quit;
+
+/**
+  * load 1 - change that cell to an ordinary number
+  */
+data work.sascontroltable;
+  action='LOAD';
+  message="special missing prep";
+  libds="&dclib..MPE_X_TEST";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  set &dclib..mpe_x_test(where=(primary_key_field=3)
+    rename=(some_date=dt2 SOME_DATETIME=dttm2 SOME_TIME=tm2)
+  );
+  /* the adapter sends these as strings */
+  some_date=put(dt2,date9.);
+  SOME_DATETIME=put(dttm2,datetime19.);
+  some_time=put(tm2,time.);
+  drop dt2 dttm2 tm2;
+  _____DELETE__THIS__RECORD_____='No';
+  some_bestnum=42;
+run;
+
+%mx_execute(&appLoc/services/editors/stagedata,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.jsdata work.sascontroltable,
+  outlib=web1,
+  mdebug=&sasjs_mdebug
+)
+
+%let status=0;
+data _null_;
+  set web1.sasparams;
+  if status='SUCCESS' then call symputx('status',1);
+  call symputx('dsid',dsid);
+run;
+%mp_assert(
+  iftrue=(&status=1 and &syscc=0),
+  desc=Checking the special missing change was staged,
+  outds=work.test_results
+)
+
+data work.sascontroltable;
+  ACTION='APPROVE_TABLE';
+  TABLE="&dsid";
+  DIFFTIME="%sysfunc(datetime())";
+  output;
+  stop;
+run;
+%mx_execute(&appLoc/services/auditors/postdata,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let status=0;
+data _null_;
+  set web3.apparams;
+  if response='SUCCESS!' then call symputx('status',1);
+run;
+%mp_assert(
+  iftrue=(&status=1 and &syscc=0),
+  desc=Checking the change was approved,
+  outds=work.test_results
+)
+
+/* the load reference to revert */
+%let loadref=0;
+data _null_;
+  set &dclib..mpe_submit end=last;
+  if last then call symputx('loadref',table_id);
+run;
+
+/**
+  * revert it - the old value in the diff is the special missing
+  */
+data work.restore_in;
+  load_ref="&loadref";
+  output;
+  stop;
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.restore_in,
+  outlib=web1,
+  mdebug=&sasjs_mdebug
+)
+
+%let newref=0;
+data _null_;
+  set web1.restore_out;
+  call symputx('newref',loadref);
+run;
+%mp_assert(
+  iftrue=(&newref ne 0 and &syscc=0),
+  desc=Checking the revert was submitted,
+  outds=work.test_results
+)
+
+/* approve the reversion */
+data work.sascontroltable;
+  ACTION='APPROVE_TABLE';
+  TABLE="&newref";
+  DIFFTIME="%sysfunc(datetime())";
+  output;
+  stop;
+run;
+%mx_execute(&appLoc/services/auditors/postdata,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let status=0;
+data _null_;
+  set web3.apparams;
+  if response='SUCCESS!' then call symputx('status',1);
+run;
+%mp_assert(
+  iftrue=(&status=1 and &syscc=0),
+  desc=Checking the reversion was approved,
+  outds=work.test_results
+)
+
+/* and the cell is back to the special missing, not a number and not blank */
+%let spmcheck=0;
+data _null_;
+  set &dclib..mpe_x_test(where=(primary_key_field=3));
+  if some_bestnum=._ then call symputx('spmcheck',1);
+  putlog some_bestnum=;
+run;
+%mp_assert(
+  iftrue=(&spmcheck=1),
+  desc=Checking the special missing was restored,
+  outds=work.test_results
+)
+
+/* put the fixture back as makedata left it */
+proc sql noprint;
+update &dclib..mpe_x_test set some_bestnum=44
+  where primary_key_field=3;
+quit;

sas/sasjs/services/editors/stagedata.test.4.sas

@@ -6,16 +6,19 @@
       table - the engine reads only active rows, so the dependency can never
       resolve at run time
     2. a row is staged with active=0 while a surviving active row depends on it
+    3. a staged row breaks a field rule of the shared scan (core's
+      mp_derivedvarcheck), or carries an ACTIVE value the EDIT screen's picker
+      cannot produce
 
   They should NOT abort when:
-    3. a new active row is added to a table that also holds an inactive row -
+    4. a new active row is added to a table that also holds an inactive row -
       the inactive row is not part of the dependency graph, so it cannot
       create a cycle or a stale reference
-    4. a delete-marked RLS rule naming a derived variable is submitted - the
+    5. a delete-marked RLS rule naming a derived variable is submitted - the
       rule is being removed, so its variables are not validated
-    5. a delete and re-add of the same variable is submitted together - the
+    6. a delete and re-add of the same variable is submitted together - the
       re-added active row resolves the dependency
-    6. an inactive row is set active again while another active row depends on
+    7. an inactive row is set active again while another active row depends on
       it - the re-activation is what makes the dependency resolvable
 
   The staged loads are aborted by the post edit hook. The assertion fetches the
@@ -553,6 +556,106 @@ run;
 )
 
 
+/**
+  * Test 8 - a field rule of the shared scan (mp_derivedvarcheck, core) is
+  * applied at submit time: an expression that could submit code of its own is
+  * refused by name, with the reason the scan gives. This is the same scan the
+  * engine runs at read time, so the case pins that the hook calls it rather
+  * than carrying a rule list of its own.
+  */
+data work.sascontroltable;
+  action='LOAD';
+  message='A refused function in VAR_LOGIC should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
+    VAR_LOGIC $4000 VAR_FMT $32
+    _____DELETE__THIS__RECORD_____ $3;
+  LIBREF='DCTEST';
+  DSN='DC_DERIV';
+  VAR_NM='CALLER';
+  VAR_DEPS='';
+  VAR_TYPE='N';
+  VAR_LEN=8;
+  VAR_FMT='';
+  /* no semicolon in the payload: the single-expression rule is tested before
+    the function-name rule, so a quoted semicolon would abort on that first */
+  VAR_LOGIC='dosubl("data _null_")';
+  ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+run;
+
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web8,
+  outref=wb8,
+  /* the child is expected to abort here, so the failure must be asserted
+    rather than aborting this program */
+  failfast=0,
+  mdebug=&sasjs_mdebug
+)
+
+/* the assertion compares the abort's MSG exactly, so the expected value is
+  assembled on one line: a literal wrapped across two source lines carries the
+  newline and the continuation indent with it */
+%let exp8=MPE_DERIVED: CALLER: VAR_LOGIC may not call a function that;
+%let exp8=&exp8 submits code or reaches the file system,;
+%let exp8=&exp8 the environment or the macro symbol table;
+
+%mpe_assert_abort(
+  desc=Checking a refused function in VAR_LOGIC is rejected,
+  outref=wb8,
+  msg=&exp8
+)
+
+/**
+  * Test 9 - the fields the hook owns rather than the scan: ACTIVE is a picker
+  * in the EDIT screen (HARDSELECT over the MPE_SELECTBOX pair), so a submission
+  * that bypasses the grid cannot land a value the screen would never offer.
+  */
+data work.sascontroltable;
+  action='LOAD';
+  message='An ACTIVE value the screen cannot offer should abort';
+  libds="&dc_libref..MPE_DERIVED";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length LIBREF $8 DSN $32 VAR_NM $32 VAR_DEPS $4000 VAR_TYPE $1
+    VAR_LOGIC $4000 VAR_FMT $32
+    _____DELETE__THIS__RECORD_____ $3;
+  LIBREF='DCTEST';
+  DSN='DC_DERIV';
+  VAR_NM='BADACT';
+  VAR_DEPS='';
+  VAR_TYPE='N';
+  VAR_LEN=8;
+  VAR_FMT='';
+  VAR_LOGIC='1';
+  ACTIVE=2;
+  _____DELETE__THIS__RECORD_____='No';
+run;
+
+%mpe_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web9,
+  outref=wb9,
+  mdebug=&sasjs_mdebug
+)
+
+%mpe_assert_abort(
+  desc=Checking an ACTIVE value outside the picker pair is rejected,
+  outref=wb9,
+  msg=MPE_DERIVED: BADACT: ACTIVE must be 1 or 0
+)
+
 /* tidy up so repeated runs start clean */
 proc sql noprint;
 delete from &dc_libref..mpe_derived

sas/sasjs/services/hooks/mpe_derived_postedit.sas

@@ -34,7 +34,7 @@
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
   @li mp_abort.sas
-  @li mpe_derived_check.sas
+  @li mp_derivedvarcheck.sas
 
   <h4> Related Macros </h4>
   @li mpe_derived.sas
@@ -100,14 +100,43 @@ run;
 )
 
 /**
-  * Field level checks - the same rules %mpe_derived applies at read time
+  * Field level checks.  The rules for the derived variable itself - the name,
+  * the type, length, format, label and the expression - are core's, in
+  * mp_derivedvarcheck, so this hook and the read time engine (%mpe_derived ->
+  * mp_derivedvars) scan a row with the same rules and report the same reasons.
+  * What is added here are the two fields that belong to Data Controller rather
+  * than to the expression: the control table key, which the loader normalises
+  * but does not require, and ACTIVE, which the EDIT screen offers as a picker
+  * over the MPE_SELECTBOX pair - so a submission that bypasses the grid cannot
+  * land a value the screen would never offer.
   */
-%mpe_derived_check(inds=work.staging_ds, outds=work.derived_check)
+data work.derived_dcfield(keep=name reason);
+  set work.staging_ds;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  length name $32 reason $200;
+  call missing(reason);
+  name=upcase(strip(var_nm));
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if missing(active) or active not in (0,1)
+    then reason='ACTIVE must be 1 or 0';
+  if not missing(reason) then output;
+run;
+
+%mp_derivedvarcheck(inds=work.staging_ds
+  ,outds=work.derived_field
+  ,reserved=%str(_____DELETE__THIS__RECORD_____)
+)
+
+/* the control table fields are reported before the expression's own rules, so
+  which of the two passes found a row does not change the message it gets */
+data work.derived_check;
+  set work.derived_dcfield work.derived_field;
+run;
 
 %if %mf_nobs(work.derived_check)>0 %then %do;
   data _null_;
     set work.derived_check(obs=1);
-    call symputx('der_errmsg',cats('MPE_DERIVED: ',var_nm,': ',reason));
+    call symputx('der_errmsg',cats('MPE_DERIVED: ',name,': ',reason));
     call symputx('der_errflag',1);
   run;
 %end;

sas/sasjs/services/hooks/mpe_security_postedit.sas

@@ -35,5 +35,7 @@ run;
 
 %mp_abort(iftrue=(&errval=1)
   ,mac=mpe_security_postedit.sas
-  ,msg=%str(&errmsg)
+  /* %bquote: the submitted value is interpolated into this message, and a
+    comma in it would split this call into extra positional parameters */
+  ,msg=%bquote(&errmsg)
 )