2026-11-07 01:09:32 +00:00
17 changed files with 1904 additions and 45 deletions
@@ -1972,6 +1972,23 @@ function loadDcSettings(driveRoot, appLoc) {
    return result
 }
 /**
 * The mock's %mpe_init(): the values every service needs before it does
 * anything, read from the settings program makedata writes.
 *
 * The real backend generates settings.sas in makedata (dc_libref, dcpath,
 * dc_admin_group, dc_staging_area) and %mpe_init reads it into macro variables
 * for every service call. makedata here writes the same values to
 * services/settings.js, and this is the one function services call to get them -
 * so a deployment that puts the control library somewhere else, or names a
 * different admin group, changes one file rather than every service.
 *
 * @returns {{dcLibref: string, dataDir: string, dcAdminGroup: string|null, dcpath: string|null}}
 */
 function dcInit(driveRoot, appLoc) {
    return loadDcSettings(driveRoot, appLoc)
 }
 /**
 * Reads the DC-scope options out of MPE_CONFIG, mirroring the loop in the real
 * backend's generated settings.sas: only rows that are active (var_active=1)
@@ -3,11 +3,35 @@ const nodePath = require('path')
 let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
 const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
 const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
 const dcLibref = 'DC_JSLIB'
 // Load shared DC mock utilities
 eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
 /*
  The control library, the admin group and dcpath come from services/settings.js
  - the JS twin of the settings program the backend generates in %mpe_init, which
  every real service reads. makedata writes it, so a deployment that puts the
  control library somewhere else (a different DCPATH) needs no service change,
  and there is one place the admin group is named.
 */
 const dcSettings = dcInit(driveRoot, appLoc)
 const dcLibref = dcSettings.dcLibref || 'DC_JSLIB'
 const dcAdminGroup = dcSettings.dcAdminGroup || 'dc-admin'
 /**
 * Reads a column from a stored row by name, whatever case the row uses.
 * makedata writes the seeded rows with the column names as the macro spells
 * them (DD_SOURCE), while a row that has been through the app's edit flow comes
 * back with the adapter's lower-cased keys, and a service should not care which.
 */
 function field(row, name) {
    if (!row) return undefined
    if (row[name] !== undefined) return row[name]
    const upper = name.toUpperCase()
    if (row[upper] !== undefined) return row[upper]
    const lower = name.toLowerCase()
    return row[lower]
 }
 // Mirrors SAS cats() for the values a dropdown source can hold: a special
 // missing stored in its period form (".a") becomes the bare uppercase letter
 // ("A") that cats() returns, so the mock's dropdown list matches the real one.
@@ -117,18 +141,47 @@ if (tableData && derived.length > 0) {
 // Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
 // NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
 /**
 * Whether a control-library row is still in force.
 *
 * `tx_to` is epoch seconds on the seeded rows, which is the representation
 * Date.now()/1000 assumes. A row that carries no end date at all - blank,
 * absent, or a value that is not a date - is open-ended rather than expired:
 * reading it as NaN would drop the row with no signal, which for a validation
 * rule silently removes it from the grid and for a security row silently
 * revokes the grant. `tx_from` is not consulted: getdata.sas filters on the end
 * date alone, which is what FILTER_TEXT tells the user.
 */
 function rowInForce(row, now) {
    const raw = field(row, 'tx_to')
    if (raw === undefined || raw === null || String(raw).trim() === '') return true
    const txTo = Number(raw)
    if (!isFinite(txTo)) {
        const parsed = Date.parse(String(raw))
        return isNaN(parsed) ? true : parsed / 1000 > now
    }
    return txTo > now
 }
 const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
 let dqrules = []
 let dqdata = []
 if (validationsData && validationsData.rows) {
-    const pkCols = new Set(pkColumns.map(c => c.toUpperCase()))
+    const now = Math.floor(Date.now() / 1000)
    dqrules = validationsData.rows
-        .filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
+        .filter(r => String(field(r, 'base_lib')).toUpperCase() === requestedLibref)
        .filter(r => String(field(r, 'base_ds')).toUpperCase() === requestedTable)
        // only rules that are switched on and still in force, as getdata.sas's
        // proc sort does (rule_active=1 and tx_to in the future)
        .filter(r => String(field(r, 'rule_active')).trim() === '1' && rowInForce(r, now))
        .map(r => ({
-            BASE_COL: r.base_col,
+            BASE_COL: String(field(r, 'base_col')).toUpperCase(),
-            RULE_TYPE: r.rule_type,
+            RULE_TYPE: String(field(r, 'rule_type')).toUpperCase(),
-            RULE_VALUE: r.rule_value,
+            RULE_VALUE: field(r, 'rule_value'),
-            X: pkCols.has(r.base_col.toUpperCase()) ? 0 : 1
+            // X is the select-rule counter getdata.sas carries on the dataset.
            // This table has no such rule, so the value is SAS missing, which
            // reaches the client as 0
            X: 0
        }))
 }
@@ -155,7 +208,15 @@ if (tableData && tableData.columns) {
    const existingNotNull = new Set(
        dqrules.filter(r => r.RULE_TYPE === 'NOTNULL').map(r => r.BASE_COL.toUpperCase())
    )
-    for (const rule of notNullRules(tableData.columns)) {
+    // getdata.sas limits the NOTNULL union to the columns left in the selection
    // (vars3), and the temporal columns have already been moved out of the data
    // by then - which is why a NOT NULL constraint on tx_from / tx_to never
    // reaches the grid as a rule
    const ruleColumns = tableData.columns.filter(c => {
        const ddtype = getDdType(c)
        return ddtype !== 'DATE' && ddtype !== 'DATETIME' && ddtype !== 'TIME'
    })
    for (const rule of notNullRules(ruleColumns)) {
        if (!existingNotNull.has(rule.BASE_COL)) dqrules.push(rule)
    }
 }
@@ -167,7 +228,33 @@ if (tableData && tableData.columns) {
 // as getdata.sas does.  dqrules is the request-local array the filter/map above
 // built, so appending to it leaves the loaded MPE_VALIDATIONS rows untouched.
 for (const v of derived) {
-    dqrules.push({ BASE_COL: v.name, RULE_TYPE: 'READONLY', RULE_VALUE: '' })
+    dqrules.push({ BASE_COL: String(v.name).toUpperCase(), RULE_TYPE: 'READONLY', RULE_VALUE: '', X: 0 })
 }
 /*
  getdata.sas builds this as a SQL UNION and then reads the result sorted by
  base_col, rule_type, rule_value, keeping only the last row of each
  (base_col, rule_type) pair - so a rule present in MPE_VALIDATIONS wins over the
  schema's NOTNULL for the same column, and the client sees one rule per pair in
  alphabetical order. Doing it here keeps the mock's payload in the same order,
  which is what the client's dropdown-code mapping and the golden compare.
 */
 {
    const byKey = new Map()
    dqrules
        .sort((a, b) => {
            const left = [a.BASE_COL, a.RULE_TYPE, String(a.RULE_VALUE)].join('\u0000')
            const right = [b.BASE_COL, b.RULE_TYPE, String(b.RULE_VALUE)].join('\u0000')
            return left < right ? -1 : left > right ? 1 : 0
        })
        .forEach(rule => {
            // the last value for the pair wins, as `if last.rule_type` does
            byKey.set(rule.BASE_COL + '\u0000' + rule.RULE_TYPE, rule)
        })
    dqrules = Array.from(byKey.values()).sort((a, b) => {
        if (a.BASE_COL !== b.BASE_COL) return a.BASE_COL < b.BASE_COL ? -1 : 1
        return a.RULE_TYPE < b.RULE_TYPE ? -1 : a.RULE_TYPE > b.RULE_TYPE ? 1 : 0
    })
 }
 // ─── DQ dropdown data (from MPE_SELECTBOX + SOFTSELECT from table data) ──────
@@ -269,6 +356,28 @@ if (tableData && tableData.columns && tableData.rows) {
        return copy
    })
    /*
      mpe_filtermaster restricts an EDIT to rows that are still current, and the
      clause it writes into FILTER_TEXT says so. The restriction is applied here
      so the rows and the text that describes them stay in step - otherwise the
      grid would show expired rows while telling the user they were excluded.
      A row with no end date is open-ended, as the clause implies.
    */
    const validityVar = tableReg && tableReg.var_txto ? String(tableReg.var_txto).trim() : ''
    if (validityVar) {
        const nowSeconds = Math.floor(Date.now() / 1000)
        visibleRows = visibleRows.filter(row => {
            const key = Object.keys(row).find(
                k => k.toLowerCase() === validityVar.toLowerCase()
            )
            const raw = key ? row[key] : undefined
            if (raw === undefined || raw === null || String(raw).trim() === '') return true
            const numeric = Number(raw)
            const seconds = isFinite(numeric) ? numeric : Date.parse(String(raw)) / 1000
            return isNaN(seconds) ? true : seconds > nowSeconds
        })
    }
    // ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
    // Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
    // been applied and the rows sorted, with the data in work.OUT, and may
@@ -336,6 +445,14 @@ if (tableData && tableData.columns && tableData.rows) {
    for (const col of visibleColumns) {
        colLookup[col.name.toLowerCase()] = { name: col.name, ddtype: getDdType(col) }
    }
    /*
      A SAS character variable is padded to its length, so a blank one reaches
      the client as spaces (' ') rather than as an absent key. The mock's rows
      are sparse by nature, so the padding is applied here - without it a blank
      value is missing from the JSON and every accepted-difference list fills up
      with paths that only exist because the key was absent.
    */
    const charColumns = visibleColumns.filter(col => getDdType(col) === 'C')
    const augmentedRows = visibleRows.map(r => {
        const normalised = {}
        for (const key of Object.keys(r)) {
@@ -343,9 +460,50 @@ if (tableData && tableData.columns && tableData.rows) {
            const targetKey = info ? info.name : key
            normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
        }
        for (const col of charColumns) {
            const value = normalised[col.name]
            if (value === undefined || value === null) {
                normalised[col.name] = ' '
            } else {
                // the JSON writer strips a character value, and an all-blank
                // one arrives as a single space rather than as an empty string
                const text = String(value).trim()
                normalised[col.name] = text === '' ? ' ' : text
            }
        }
        return { ...normalised }
    })
    /*
      getdata.sas sorts the selection by the primary key before it writes it out
      (proc sort by the pk fields), and the grid's duplicate-key detection and
      the golden both depend on that order. A table with no registered key keeps
      the order it was read in.
    */
    if (pkColumns.length) {
        // the row keys keep the DDL's casing, so the sort keys are resolved to
        // the column's own name rather than uppercased: a lowercase DDL would
        // otherwise compare undefined against undefined and lose the order
        const keys = pkColumns.map(name => {
            const match = visibleColumns.find(
                col => col.name.toUpperCase() === name.toUpperCase()
            )
            return match ? match.name : name
        })
        augmentedRows.sort((a, b) => {
            for (const key of keys) {
                const left = a[key]
                const right = b[key]
                if (left === right) continue
                if (left === undefined) return -1
                if (right === undefined) return 1
                if (typeof left === 'number' && typeof right === 'number') return left - right
                return String(left) < String(right) ? -1 : 1
            }
            return 0
        })
    }
    // $sasdata vars: temporal columns are char ($200.), others keep their SAS type
    const vars = {}
    for (const col of visibleColumns) {
@@ -369,24 +527,49 @@ if (tableData && tableData.columns && tableData.rows) {
    // is the JS counterpart of %mp_getcols.sas - the macro getdata.sas itself
    // calls to type these columns - so the editor sees the same DDTYPE the
    // service would give it.
    /*
      The data dictionary is where a column's LABEL and LONGDESC come from when
      someone has written one: getdata.sas augments mp_getcols with the
      MPE_DATADICTIONARY rows for this lib.ds.column, so a table registered there
      shows the business description instead of the bare column name.
    */
    const dictionaryData = mpeLoadTableData('MPE_DATADICTIONARY')
    const dictionary = {}
    if (dictionaryData && dictionaryData.rows) {
        const prefix = (requestedLibref + '.' + requestedTable + '.').toUpperCase()
        for (const row of dictionaryData.rows) {
            // the seeded rows carry UPPERCASE keys (DD_SOURCE), a row that has
            // been through the edit flow comes back lower-cased - read both
            const source = String(field(row, 'dd_source') || '').toUpperCase()
            if (String(field(row, 'dd_type') || '').toUpperCase() !== 'COLUMN') continue
            if (source.indexOf(prefix) !== 0) continue
            dictionary[source.slice(prefix.length)] = {
                short: field(row, 'dd_shortdesc'),
                long: field(row, 'dd_longdesc')
            }
        }
    }
    const cols = mpGetcols(tableData.columns).map((c, i) => {
        const raw = tableData.columns[i]
        const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
        let coltype
-        if (ddtype === 'DATE') coltype = `{"data":"${raw.name}","type":"date"}`
+        // a temporal column carries no LYTE spec on the estate - getdata.sas
-        else if (ddtype === 'DATETIME') coltype = `{"data":"${raw.name}","type":"datetime"}`
+        // leaves COLTYPE blank for DATE/DATETIME/TIME and the client types those
-        else if (ddtype === 'TIME') coltype = `{"data":"${raw.name}","type":"time"}`
+        // from DDTYPE
        if (ddtype === 'DATE' || ddtype === 'DATETIME' || ddtype === 'TIME') coltype = ' '
        else if (ddtype === 'N') coltype = `{"data":"${raw.name}","type":"numeric","format":"0"}`
        else coltype = `{"data":"${raw.name}"}`
        const entry = dictionary[String(c.NAME).toUpperCase()]
        return {
            NAME: c.NAME,
-            LABEL: c.LABEL,
+            // mp_getcols falls back to the column name when a column has no label
            LABEL: (entry && entry.short) || c.LABEL || c.NAME,
            FMTNAME: c.FMTNAME || ' ',
            DDTYPE: ddtype,
            CLS_RULE: 'READ',
            MEMLABEL: ' ',
-            DESC: ' ',
+            LONGDESC: (entry && entry.long) || ' ',
            LONGDESC: ' ',
            COLTYPE: coltype
        }
    }).sort((a, b) => a.NAME.localeCompare(b.NAME))
@@ -400,16 +583,83 @@ if (tableData && tableData.columns && tableData.rows) {
                let val = row[col.name]
                if (val === undefined) val = row[lcName]
                if (val !== undefined && val !== null) {
-                    const len = String(val).length
+                    // SAS measures the value as stored, so a blank character
                    // value is length 0 even though it is padded to the
                    // variable's length
                    const len = String(val).trim().length
                    if (len > maxLen) maxLen = len
                }
            }
-            return { NAME: col.name.toLowerCase(), MAXLEN: maxLen }
+            // the selection renames the key columns to upper case and leaves
            // the others as the DDL spells them
            const isKey = pkColumns.some(name => name.toUpperCase() === col.name.toUpperCase())
            return { NAME: isKey ? col.name.toUpperCase() : col.name.toLowerCase(), MAXLEN: maxLen }
        })
    ]
    /*
      getdata.sas builds this dropdown from the groups with APPROVE access to
      the table, falling back to the admin group when no row grants it, and
      expands each group to its members. The members come from MPE_GROUPS on the
      mock (the estate asks its identity provider), so the list is the estate's
      own metadata rather than a fixed name.
    */
    function approversForTable() {
        const now = Math.floor(Date.now() / 1000)
        const security = mpeLoadTableData('MPE_SECURITY')
        const groupNames = []
        if (security && security.rows) {
            const seen = new Set()
            for (const row of security.rows) {
                // the seeded rows carry uppercase keys and an edited one comes
                // back lower-cased, so every read goes through field()
                if (String(field(row, 'libref')).toUpperCase() !== requestedLibref) continue
                if (String(field(row, 'dsn')).toUpperCase() !== requestedTable) continue
                if (String(field(row, 'access_level')).toUpperCase() !== 'APPROVE') continue
                if (!rowInForce(row, now)) continue
                const group = field(row, 'sas_group')
                if (group && !seen.has(group)) {
                    seen.add(group)
                    groupNames.push(group)
                }
            }
        }
        if (!groupNames.length) groupNames.push(dcAdminGroup)
        const groupsData = mpeLoadTableData('MPE_GROUPS')
        const approvers = []
        const seenUsers = new Set()
        for (const group of groupNames) {
            for (const row of groupsData && groupsData.rows ? groupsData.rows : []) {
                if (String(field(row, 'group_name')).toUpperCase() !== String(group).toUpperCase()) continue
                const user = field(row, 'user_name')
                if (!user || seenUsers.has(user)) continue
                seenUsers.add(user)
                approvers.push({ PERSONNAME: user, EMAIL: user, USERID: user })
            }
        }
        return approvers
    }
    /*
      mpe_filtermaster appends the current-rows clause for an EDIT whenever the
      table has a validity variable and the query does not already mention it -
      the client shows FILTER_TEXT to explain why a historical row is absent.
    */
    let effectiveFilterText = filterText || ''
    if (
        validityVar &&
        !new RegExp('\\b' + validityVar + '\\b', 'i').test(effectiveFilterText)
    ) {
        const clause =
            '("%sysfunc(datetime(), E8601DT26.6 )"dt < ' +
            validityVar.toUpperCase() + ' )'
        effectiveFilterText = effectiveFilterText.trim() === ''
            ? clause
            : effectiveFilterText + ' AND ' + clause
    }
    response = {
-        approvers: [{ PERSONNAME: 'sasdemo', EMAIL: 'sasdemo', USERID: 'sasdemo' }],
+        approvers: approversForTable(),
        cols,
        dqdata, dqrules,
        dsmeta: [
@@ -425,22 +675,23 @@ if (tableData && tableData.columns && tableData.rows) {
        $sasdata: { vars },
        sasparams: [{
            COLHEADERS: colHeaders.join(','),
-            FILTER_TEXT: filterText === '' ? ' ' : filterText,
+            FILTER_TEXT: effectiveFilterText.trim() === '' ? ' ' : effectiveFilterText,
            PKCNT: pkColumns.length,
            PK: pkFields,
-            DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
+            // a blank SAS character value reaches the client as one space
-            DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
+            DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : ' ',
-            TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',
+            DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : ' ',
            TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : ' ',
            LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
            RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
            CLS_FLAG: 0,
            ISMAP: 0,
            TRUNCATED: truncated,
-            TRUNC_MSG: truncMsg,
+            TRUNC_MSG: truncMsg === '' ? ' ' : truncMsg,
            // names the derived columns for the grid, as getdata.sas does - the
            // READONLY rule above is why they cannot be edited, this is how the
            // EDIT screen knows they are computed rather than locked
-            DERIVEDVARS: derived.map(d => d.name).join(' ')
+            DERIVEDVARS: derived.length ? derived.map(d => d.name).join(' ') : ' '
        }],
        versions: [],
        xl_rules: []
@@ -183,7 +183,18 @@ if (tableData && tableData.columns) {
                    }
                }
            }
-            row[col.name] = String(val)
+            /*
              A value the row does not carry is SAS's missing, which the viewer
              renders as an empty cell: a space for a character column and '.' for
              a number. Building the cell with String() would put the text
              "undefined" on screen instead, which is what a missing property
              stringifies to.
            */
            if (val === undefined || val === null || val === '') {
                row[col.name] = col.type === 'N' ? '.' : ' '
            } else {
                row[col.name] = String(val)
            }
        }
        return row
    })
@@ -196,7 +207,9 @@ if (tableData && tableData.columns) {
    // its "no data found" panel instead of an empty grid).
    if (viewdata.length === 0 && !searchType && physicalColumns.length > 0) {
        const emptyRow = {}
-        for (const col of physicalColumns) emptyRow[col.name] = col.type === 'N' ? '.' : ''
+        // a blank character cell is a space everywhere else in this service,
        // so the empty row says the same thing
        for (const col of physicalColumns) emptyRow[col.name] = col.type === 'N' ? '.' : ' '
        viewdata = [emptyRow]
    }
@@ -10,21 +10,32 @@ eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'd
 // DC_JSLIB is the control library; other librefs (e.g. TESTDATA) hold
 // user/demo data tables. Each gets a synthetic LIBRARYID/engine entry the
 // same way a real SAS site lists assigned libraries.
-const dcLibref = 'DC_JSLIB'
+// dcInit() is the mock's %mpe_init(): the control library comes from the
 // settings program makedata writes, not from a constant in this file
 // the fallback matters: a deploy that has not run makedata yet has no
 // settings.js, and this service must still list the control library
 const dcLibref = dcInit(driveRoot, appLoc).dcLibref || 'DC_JSLIB'
 /*
  The shape SAS answers with, not a nicer one: %mp_getlibs puts the engine in
  LIBRARYID for the control library, leaves ENGINE blank and uses the libref as
  LIBRARYNAME. The client reads LIBRARYREF, so the fields are reproduced as they
  arrive rather than corrected - a mock that improves on the payload is a mock
  the Cypress suite cannot be trusted against.
 */
 let saslibs = allLibrefs().map((lib) => ({
-    LIBRARYID: lib,
+    LIBRARYID: 'V9',
-    LIBRARYNAME: 'Data Controller(' + lib + ')',
+    LIBRARYNAME: lib,
    LIBRARYREF: lib,
-    ENGINE: 'V9'
+    ENGINE: ' '
 }))
 // DC_JSLIB must always be present even if the folder is missing (fresh deploy)
 if (!saslibs.find(l => l.LIBRARYREF === dcLibref)) {
    saslibs.unshift({
-        LIBRARYID: dcLibref,
+        LIBRARYID: 'V9',
-        LIBRARYNAME: 'Data Controller(' + dcLibref + ')',
+        LIBRARYNAME: dcLibref,
        LIBRARYREF: dcLibref,
-        ENGINE: 'V9'
+        ENGINE: ' '
    })
 }
@@ -0,0 +1,80 @@
 /**
 * Two reads in getdata go through the control library's own rows, and both have
 * a case that looks like an empty answer rather than a bug:
 *
 *   - the security and group rows are seeded with UPPERCASE keys (`SAS_GROUP`)
 *     while a row that has been through the edit flow comes back lower-cased, so
 *     a direct property read compares against undefined and finds no approver;
 *   - a `tx_to` that is blank or absent is an open-ended row, not an expired
 *     one, and reading it as NaN drops the rule or the grant with no signal.
 */
 const test = require('node:test')
 const assert = require('node:assert')
 const { seededEstate, runService, readTable, writeTable } = require('./harness')
 function getdata(estate, libds) {
  return JSON.parse(
    runService(estate, 'editors/getdata', {
      tableName: 'SASControlTable',
      rows: [{ LIBDS: libds, FILTER_RK: -1 }]
    })
  )
 }
 test('an APPROVE grant whose keys are upper case still yields its group', () => {
  const estate = seededEstate({ demo: false })
  const security = readTable(estate, 'MPE_SECURITY')
  writeTable(
    estate,
    'MPE_SECURITY',
    security.concat([
      {
        TX_FROM: 0,
        TX_TO: 127490111999,
        LIBREF: 'DC_JSLIB',
        DSN: 'MPE_TABLES',
        ACCESS_LEVEL: 'APPROVE',
        SAS_GROUP: 'dc-admin'
      }
    ])
  )
  const body = getdata(estate, 'DC_JSLIB.MPE_TABLES')
  assert.equal(body.sasjsAbort, undefined)
  const users = body.approvers.map((row) => row.USERID).sort()
  assert.deepEqual(
    users,
    ['allbow', 'dctestuser1', 'mihmed'],
    'the grant is found and its group expanded, rather than falling back'
  )
 })
 test('a rule with no end date is in force, not dropped', () => {
  const estate = seededEstate({ demo: false })
  const validations = readTable(estate, 'MPE_VALIDATIONS')
  writeTable(
    estate,
    'MPE_VALIDATIONS',
    validations.concat([
      {
        tx_from: 0,
        TX_TO: ' ',
        BASE_LIB: 'DC_JSLIB',
        BASE_DS: 'MPE_TABLES',
        BASE_COL: 'LOADTYPE',
        RULE_TYPE: 'NOTNULL',
        RULE_VALUE: ' ',
        RULE_ACTIVE: 1
      }
    ])
  )
  const body = getdata(estate, 'DC_JSLIB.MPE_TABLES')
  const rule = body.dqrules.find(
    (row) => row.BASE_COL === 'LOADTYPE' && row.RULE_TYPE === 'NOTNULL'
  )
  assert.ok(rule, 'a blank end date means open-ended, so the rule is returned')
 })
@@ -0,0 +1,56 @@
 /**
 * The harness stands in for the runtime, so the parts of `webout()` the mocks
 * rely on have to be there: the `$<member>` metadata the client reads for
 * column formats and labels, and a body that exists while the program is still
 * running - `requireMemberMetadata()` reads it before CLOSE on purpose.
 */
 const test = require('node:test')
 const assert = require('node:assert')
 const fs = require('fs')
 const path = require('path')
 const { makeEstate, runService } = require('./harness')
 function probe(estate, name, source) {
  const dir = path.join(estate.servicesDir, 'public')
  fs.mkdirSync(dir, { recursive: true })
  fs.writeFileSync(path.join(dir, name + '.js'), source)
 }
 test('showmeta is written beside the rows, before CLOSE', () => {
  const estate = makeEstate()
  probe(
    estate,
    'showmeta_probe',
    [
      "webout('OPEN')",
      "webout('OBJ', 'sasdata', [{ A: 1 }], { showmeta: { vars: { A: { type: 'num' } } } })",
      '// the mocks call this before CLOSE, exactly as getdata does',
      "if (fs.readFileSync(weboutPath, 'utf8').indexOf('\"$sasdata\"') < 0) {",
      "  throw new Error('the metadata was not in the body before CLOSE')",
      '}',
      "webout('CLOSE')"
    ].join('\n')
  )
  const body = JSON.parse(runService(estate, 'public/showmeta_probe', {}))
  assert.deepEqual(body.sasdata, [{ A: 1 }])
  assert.deepEqual(body['$sasdata'], { vars: { A: { type: 'num' } } })
 })
 test('a member written without showmeta has no $member beside it', () => {
  const estate = makeEstate()
  probe(
    estate,
    'no_showmeta_probe',
    ["webout('OPEN')", "webout('OBJ', 'rows', [{ A: 1 }])", "webout('CLOSE')"].join(
      '\n'
    )
  )
  const body = JSON.parse(runService(estate, 'public/no_showmeta_probe', {}))
  assert.deepEqual(body.rows, [{ A: 1 }])
  assert.equal(body['$rows'], undefined)
 })
@@ -117,7 +117,7 @@ function runService(
    // the three calls the mocks make, with the same validation and the same
    // shape, so a service that misuses OBJ/ARR fails here too.
    'let __wb = null;',
-    'function webout(action, name, rows) {',
+    'function webout(action, name, rows, options) {',
    "  const cmd = String(action === undefined ? '' : action).toUpperCase();",
    "  if (cmd === 'OPEN') { __wb = {}; return; }",
    "  if (cmd === 'CLOSE') {",
@@ -137,6 +137,15 @@ function runService(
    "  if (cmd === 'ARR' && rows.some(r => !Array.isArray(r))) throw new Error(\"webout('ARR') takes positional rows\");",
    "  if (cmd === 'OBJ' && rows.some(r => !r || typeof r !== 'object' || Array.isArray(r))) throw new Error(\"webout('OBJ') takes rows keyed by column name\");",
    "  __wb[name] = rows;",
    // @sasjs/utils 4.6.0 writes the column metadata as $<member> when the
    // caller passes { showmeta }. A harness that dropped it would make the
    // mock's own requireMemberMetadata guard throw, and a service that stopped
    // emitting formats and labels would look fine here.
    "  if (options && options.showmeta) __wb['$' + name] = options.showmeta;",
    // the real webout() streams to the response file as members are added, and
    // the mocks' own guard reads that file before CLOSE to prove the metadata
    // arrived - so a stub that only wrote at CLOSE makes the guard throw
    "  fs.writeFileSync(weboutPath, JSON.stringify(__wb));",
    '}',
    Object.entries(params)
      .map(
@@ -0,0 +1,66 @@
 /**
 * The settings program is the mock's %mpe_init(): makedata writes
 * services/settings.js with the control library, the admin group and dcpath,
 * and every service reads them from there rather than from a constant - so a
 * deployment that puts the library elsewhere needs no service change.
 *
 * These pin the two services that read it today. The rest of the estate still
 * carries `const dcLibref = 'DC_JSLIB'`; each computes paths from it at file
 * scope, so converting them is a per-service change rather than a sweep.
 */
 const test = require('node:test')
 const assert = require('node:assert')
 const fs = require('fs')
 const path = require('path')
 const { seededEstate, runService } = require('./harness')
 function rewriteSettings(estate, libref, adminGroup) {
  const file = path.join(estate.appDir, 'services', 'settings.js')
  const settings = {
    dc_libref: libref,
    dc_admin_group: adminGroup,
    dc_staging_area: path.join(estate.appDir, 'data', libref, 'dc_staging'),
    dc_macros: path.join(estate.appDir, 'data', libref, 'dc_macros'),
    mpelib: libref,
    dcpath: path.join(estate.appDir, 'data', libref)
  }
  fs.writeFileSync(file, 'var dcSettings = ' + JSON.stringify(settings) + '\n')
 }
 test('viewlibs lists the library the settings file names', () => {
  const estate = seededEstate({ demo: false })
  const before = JSON.parse(runService(estate, 'public/viewlibs', {})).saslibs
  assert.ok(
    before.some((row) => row.LIBRARYREF === 'DC_JSLIB'),
    'the seeded estate names its control library'
  )
  rewriteSettings(estate, 'DC7QATEST', 'dc-admin')
  const after = JSON.parse(runService(estate, 'public/viewlibs', {})).saslibs
  assert.ok(
    after.some((row) => row.LIBRARYREF === 'DC7QATEST'),
    'viewlibs follows the settings file rather than a constant'
  )
 })
 test('getdata reads its table from the library the settings file names', () => {
  const estate = seededEstate({ demo: false })
  // move the control library's data files to the new libref, as a deployment
  // with a different DCPATH would
  const from = path.join(estate.appDir, 'data', 'DC_JSLIB')
  const to = path.join(estate.appDir, 'data', 'DC7QATEST')
  fs.cpSync(from, to, { recursive: true })
  rewriteSettings(estate, 'DC7QATEST', 'dc-admin')
  const body = JSON.parse(
    runService(estate, 'editors/getdata', {
      tableName: 'SASControlTable',
      rows: [{ LIBDS: 'DC7QATEST.MPE_TABLES', FILTER_RK: -1 }]
    })
  )
  assert.equal(body.sasjsAbort, undefined)
  assert.ok(body.sasdata.length > 0, 'the table was found in the new library')
 })
@@ -31,6 +31,19 @@ test('a golden is replayed for every scenario', () => {
  const estate = seededEstate({ demo: false })
  for (const result of replay(estate, runService, scenarios)) {
    const id = result.scenario.service + '/' + result.scenario.id
    /*
      `gated: false` marks a scenario that is recorded but whose mock has not
      been aligned yet: the difference count is printed so it cannot rot out of
      sight, and it does not fail the build. The flag is a to-do - remove it
      when the mock matches, at which point the assertions below take over.
    */
    if (result.scenario.gated === false) {
      console.log(
        'advisory ' + id + ': ' + result.unexpected.length +
          ' difference(s), not gated'
      )
      continue
    }
    assert.ok(
      result.golden,
      id + ' has no golden - record one, or remove the scenario'
@@ -23,8 +23,10 @@ node sas/mocks/test/parity/record.js --target server --ca /path/to/estate.pem
 # throwaway host only - warns, and _meta records tls: unverified
 node sas/mocks/test/parity/record.js --target server --insecure
-# record one
+# record one (an id on its own matches every service that uses it)
 node sas/mocks/test/parity/record.js --target server --scenario default
 node sas/mocks/test/parity/record.js --target server \
  --scenario editors/getdata/mpe_tables --libref DC634301
 # replay against a mock estate and diff (no server, no credentials)
 node sas/mocks/test/parity/check.js
@@ -39,6 +41,19 @@ Raw responses land in `.captures/` (gitignored) so a refusal can be diagnosed wi
 `scenarios.json` names, per scenario: the service, the URL parameters, the members to keep, and why the scenario exists. Everything not named in `members` is dropped by the projection, so a member added upstream is excluded until someone names it - the default is to publish less.
 A service that reads a **table input** rather than URL parameters takes an `input` block:
 ```json
 "input":     { "table": "SASControlTable", "rows": [ { "LIBDS": "{{LIBREF}}.MPE_TABLES", "FILTER_RK": -1 } ] },
 "mockInput": { "table": "SASControlTable", "rows": [ { "LIBDS": "DC_JSLIB.MPE_TABLES",   "FILTER_RK": -1 } ] }
 ```
 `input` is posted to the estate and `mockInput` is handed to the mock at replay, because the rows name the control library and the two sides do not share one - only the values differ, and the shape is the point of the comparison. `{{LIBREF}}` is filled from `--libref`, so the estate's library name stays out of the repository.
 The uploaded file is not a plain CSV: the server reads its first line as the SAS `input` statement and the rows from line two, so `record.js` writes informats (`LIBDS :$32. FILTER_RK :best32.`) and the runtime rewrites `:$N.` to `:$char`. Get that wrong and the control table arrives with zero observations, which the service reports as `syscc=1012 prior to dc_getsettings` rather than as a bad upload.
 A scenario recorded before the mock has been aligned carries `"gated": false`. The gate then prints its difference count and does not fail, and `check.js` labels it `ADVISORY`. The flag is a to-do with a number attached - remove it when the mock matches, at which point the scenario is enforced like the others.
 ## Refreshing the goldens
 The build pipeline cannot reach a SAS estate, so re-recording is an **ad-hoc job run by an agent or a maintainer** holding the estate credentials - not a scheduled CI job. That splits the harness in two:
@@ -60,4 +75,6 @@ Recording is always against the synthetic estate, and that is enforced rather th
 ## Status
-Phase 0: one scenario (`public/startupservice`), the projection, the publish gate, and the replay gate in CI. The corpus, the fault-path scenarios and the refresh runbook are next; see the parity section of the `data-controller` skill.
+Every scenario in `scenarios.json` is gated: `parity-check.test.js` replays it against the mock estate and fails on any difference that is not declared in `accepted` with a reason. A scenario recorded before the mock has been aligned carries `"gated": false` instead, which makes it advisory - the gate prints its difference count and does not fail - and the flag comes off once the mock matches. The scenario's own `note` says what it pins; the per-scenario diff counts live in the gate's output, not here, because a count changes with any mock edit.
 Next: fault-path scenarios (an aborted service, a table the caller may not see) and a refresh runbook; see the parity section of the `data-controller` skill.
@@ -53,10 +53,12 @@ try {
      console.log('SKIP ' + id + ' - no golden')
      continue
    }
    const advisory = result.scenario.gated === false
    if (result.unexpected.length || result.stale.length) {
      console.log(
-        'DIFF ' + id + ' - ' + result.unexpected.length +
+        (advisory ? 'ADVISORY ' : 'DIFF ') + id + ' - ' +
-          ' unaccepted, ' + result.stale.length + ' stale acceptance(s)'
+          result.unexpected.length + ' unaccepted, ' +
          result.stale.length + ' stale acceptance(s)'
      )
      for (const difference of result.unexpected.slice(0, 40)) {
        console.log('     ' + difference)
@@ -64,7 +66,8 @@ try {
      for (const stale of result.stale) {
        console.log('     ' + stale + ': accepted, but no longer differs')
      }
-      failed++
+      // a scenario marked gated: false is reported, never counted
      if (!advisory) failed++
      continue
    }
    console.log(
@@ -0,0 +1,33 @@
 {
  "_meta": {
    "scenario": "default",
    "service": "public/viewlibs",
    "estate": "synthetic",
    "targetType": "SASJS",
    "sas": "9.4M9",
    "dc": "7.22.0",
    "captured": "2026-11-06",
    "tls": "verified",
    "members": [
      "saslibs"
    ]
  },
  "body": {
    "saslibs": [
      {
        "LIBRARYREF": "<LIBREF>",
        "LIBRARYNAME": "<LIBREF>",
        "ENGINE": " ",
        "LIBRARYID": "V9"
      },
      {
        "LIBRARYREF": "SASHELP",
        "LIBRARYNAME": "SASHELP",
        "ENGINE": "V9",
        "LIBRARYID": " "
      }
    ],
    "SYSCC": "0",
    "SYSENCODING": "utf-8"
  }
 }
@@ -4,6 +4,12 @@
 *
 *   node sas/mocks/test/parity/record.js --target server
 *   node sas/mocks/test/parity/record.js --target server --scenario default
 *   node sas/mocks/test/parity/record.js --target server --scenario mpe_tables \
 *     --libref DC634301
 *
 * A scenario with an `input` block posts a table to the service, the way the
 * client does; `--libref` fills the {{LIBREF}} placeholder in its rows, because
 * the control library's name is estate state and is not committed here.
 *
 * What it writes:
 *
@@ -106,6 +112,98 @@ function request(url, token, tls) {
  })
 }
 /**
 * The server reads the first line of an uploaded table as the SAS `input`
 * statement, then CSV rows from line two - so the header is informats, not
 * column names. Strings take :$N. (which the init rewrites to :$char), numbers
 * :best32.
 */
 function inputStatement(rows) {
  return Object.keys(rows[0])
    .map(function (name) {
      const values = rows.map(function (row) {
        return row[name]
      })
      const allNumbers = values.every(function (value) {
        return typeof value === 'number'
      })
      if (allNumbers) return name + ' :best32.'
      const width = Math.max.apply(
        null,
        [8].concat(
          values.map(function (value) {
            return String(value).length
          })
        )
      )
      return name + ' :$' + width + '.'
    })
    .join(' ')
 }
 function inputCsv(input) {
  const names = Object.keys(input.rows[0])
  const lines = [inputStatement(input.rows)]
  for (const row of input.rows) {
    lines.push(
      names
        .map(function (name) {
          const value = row[name]
          const text = value === null || value === undefined ? '' : String(value)
          return /[",\r\n]/.test(text)
            ? '"' + text.replace(/"/g, '""') + '"'
            : text
        })
        .join(',')
    )
  }
  return lines.join('\r\n') + '\r\n'
 }
 /** POST one table as multipart/form-data, as the client's request does. */
 function postTable(url, token, tls, tableName, csv) {
  const boundary = '----dcparity' + Date.now().toString(16)
  const body = Buffer.concat([
    Buffer.from(
      '--' + boundary + '\r\n' +
        'Content-Disposition: form-data; name="' + tableName +
        '"; filename="' + tableName + '.csv"\r\n' +
        'Content-Type: text/csv\r\n\r\n'
    ),
    Buffer.from(csv),
    Buffer.from('\r\n--' + boundary + '--\r\n')
  ])
  const options = {
    method: 'POST',
    headers: {
      Authorization: 'Bearer ' + token,
      'Content-Type': 'multipart/form-data; boundary=' + boundary,
      'Content-Length': body.length
    }
  }
  if (tls.insecure) options.rejectUnauthorized = false
  else if (tls.ca) options.ca = tls.ca
  return new Promise(function (resolve, reject) {
    const req = https.request(url, options, function (response) {
      let text = ''
      response.setEncoding('utf8')
      response.on('data', function (chunk) {
        text += chunk
      })
      response.on('end', function () {
        if (response.statusCode !== 200) {
          reject(new Error('HTTP ' + response.statusCode + ' from ' + url))
          return
        }
        resolve(text)
      })
    })
    req.on('error', reject)
    req.write(body)
    req.end()
  })
 }
 /**
 * The SAS version family only - "9.04.01M9P060425" is a version, a hostname is
 * not, and the version is the field that tells a reader when a golden went
@@ -121,6 +219,7 @@ async function main() {
  const only = arg('scenario')
  const insecure = process.argv.indexOf('--insecure') >= 0
  const caPath = arg('ca', null)
  const libref = arg('libref', null)
  const target = readTarget(targetName)
  /*
    A golden may only come from the organisation's own estate - the one makedata
@@ -156,7 +255,10 @@ async function main() {
  const scenarios = JSON.parse(
    fs.readFileSync(path.join(__dirname, 'scenarios.json'), 'utf8')
  ).scenarios.filter(function (scenario) {
-    return !only || scenario.id === only
+    // ids repeat across services (both a startup and a library list are the
    // 'default' call for their screen), so --scenario also takes service/id
    if (!only) return true
    return scenario.id === only || scenario.service + '/' + scenario.id === only
  })
  if (!scenarios.length) throw new Error('no scenario matched --scenario ' + only)
@@ -178,7 +280,27 @@ async function main() {
      scenario.service +
      (query ? '&' + query : '')
-    const body = await request(url, token, tls)
+    let body
    if (scenario.input) {
      if (!libref) {
        throw new Error(
          'scenario ' + scenario.id + ' posts a table - pass --libref <the ' +
            'estate control library> so {{LIBREF}} can be filled in'
        )
      }
      const input = JSON.parse(
        JSON.stringify(scenario.input).split('{{LIBREF}}').join(libref)
      )
      body = await postTable(
        url,
        token,
        tls,
        input.table || 'SASControlTable',
        inputCsv(input)
      )
    } else {
      body = await request(url, token, tls)
    }
    const raw = JSON.parse(body)
    const captureFile = path.join(
@@ -81,9 +81,20 @@ function replay(estate, runService, scenarios) {
      continue
    }
    const golden = JSON.parse(fs.readFileSync(file, 'utf8'))
-    const actual = JSON.parse(
+    /*
-      runService(estate, scenario.service, { params: scenario.params || {} })
+      A scenario recorded with a table input replays with the mock's own copy of
-    )
+      it (`mockInput`): the rows name a libref, and the estate's control library
      is not the mock's DC_JSLIB. Only values differ - the shape is the point of
      the comparison, so a scenario that forgets `mockInput` falls back to the
      recorded rows and fails loudly on the libref rather than silently passing.
    */
    const call = { params: scenario.params || {} }
    const input = scenario.mockInput || scenario.input
    if (input) {
      call.tableName = input.table || 'SASControlTable'
      call.rows = input.rows
    }
    const actual = JSON.parse(runService(estate, scenario.service, call))
    // project with the scenario's list - the one a reviewer reads and edits -
    // and report a golden whose recorded list has drifted from it, instead of
    // silently honouring the older copy inside the file
@@ -45,6 +45,64 @@
          "reason": "blank XLMAP_DESCRIPTION is ' ' on SAS (char padding) and '' in the mock"
        }
      ]
    },
    {
      "id": "mpe_tables",
      "service": "editors/getdata",
      "params": {},
      "input": {
        "table": "SASControlTable",
        "rows": [
          {
            "LIBDS": "{{LIBREF}}.MPE_TABLES",
            "FILTER_RK": -1
          }
        ]
      },
      "mockInput": {
        "table": "SASControlTable",
        "rows": [
          {
            "LIBDS": "DC_JSLIB.MPE_TABLES",
            "FILTER_RK": -1
          }
        ]
      },
      "members": [
        "approvers",
        "cols",
        "dqdata",
        "dqrules",
        "maxvarlengths",
        "query",
        "sasdata",
        "$sasdata",
        "sasparams",
        "versions",
        "xl_rules"
      ],
      "note": "the edit screen's load: one table's rows, column metadata, the data-quality rules and their lookup values, approvals outstanding and the Excel rules for the library. dsmeta is excluded on purpose - it is the engine's report on the dataset (owner, host, encoding, paths, created dates), which is estate state rather than product behaviour",
      "accepted": [
        {
          "path": ".maxvarlengths[2].MAXLEN",
          "reason": "the only numeric column: mp_getmaxvarlengths sizes a numeric by the bytes it occupies (3), the mock measures the length of its text (1). MAXLEN is used to size a grid column, not compared"
        }
      ]
    },
    {
      "id": "default",
      "service": "public/viewlibs",
      "params": {},
      "members": [
        "saslibs"
      ],
      "note": "the library picker's list. SAS also lists SASHELP, the library it installs for itself, which the mock estate has no equivalent of - that is the one declared difference",
      "accepted": [
        {
          "path": ".saslibs.length",
          "reason": "SAS lists SASHELP, the library it installs for itself; the mock estate has only the control library, so it has one row where the estate has two"
        }
      ]
    }
  ]
 }
@@ -0,0 +1,52 @@
 /**
 * public/viewdata is the viewer's payload. Its rows are the values *as SAS
 * prints them*, so a cell the row does not carry has to arrive as SAS's missing
 * - a space for a character column, '.' for a number - and never as the word
 * "undefined", which is what the viewer would otherwise render.
 */
 const test = require('node:test')
 const assert = require('node:assert')
 const { seededEstate, runService } = require('./harness')
 test('a blank character cell reaches the viewer as a space, not "undefined"', () => {
  const estate = seededEstate({ demo: false })
  const body = JSON.parse(
    runService(estate, 'public/viewdata', {
      tableName: 'SASControlTable',
      rows: [{ LIBDS: 'DC_JSLIB.MPE_TABLES' }]
    })
  )
  assert.equal(body.sasjsAbort, undefined, 'the service did not abort')
  const rows = body.viewdata
  assert.ok(rows.length > 0, 'the table has rows')
  // MPE_TABLES rows carry many blank character columns (RK_UNDERLYING, the
  // hook columns), so this is the ordinary case rather than a contrived one
  const text = JSON.stringify(rows)
  assert.ok(
    text.indexOf('undefined') < 0,
    'no cell is the literal string "undefined"'
  )
  assert.ok(
    rows.some((row) => row.RK_UNDERLYING === ' '),
    'a blank character column is a space'
  )
 })
 test('an empty table sends one row of SAS missings, one per column type', () => {
  const estate = seededEstate({ demo: false })
  const body = JSON.parse(
    runService(estate, 'public/viewdata', {
      tableName: 'SASControlTable',
      rows: [{ LIBDS: 'DC_JSLIB.MPE_AUDIT' }]
    })
  )
  assert.equal(body.sasjsAbort, undefined, 'the service did not abort')
  assert.equal(body.viewdata.length, 1, 'the viewer still gets the column set')
  const row = body.viewdata[0]
  assert.equal(row.LIBREF, ' ', 'a character column is a space')
  assert.equal(row.PROCESSED_DTTM, '.', 'a numeric column is a dot')
 })