refactor(mocks): build the control library makedata.sas builds

makedata seeded two things at once: the control library, and the tables the
Cypress suite needs. The estate it produced was therefore not the estate a
SAS deployment produces - and it had drifted: MPE_MAXKEYVALUES and
MPE_SIGNOFFS were never created, and MPE_CONFIG, MPE_VALIDATIONS, MPE_TABLES,
MPE_SELECTBOX and MPE_XLMAP_RULES were missing seed rows that mpe_makedata.sas
inserts.

makedata now transcribes mpe_makedatamodel.sas and mpe_makedata.sas: every
model table with its columns, and every seed row the macro writes. The
Cypress fixtures move to services/admin/createdemodata.js, which the test
estate runs after makedata; it strips and re-adds its rows so a reseed does
not accumulate.

makedata-parity.test.js parses both macros and compares them with the seeded
estate, so the next drift fails a test instead of shipping. The CI workflows,
the Sites demo seed and CONTRIBUTING call createdemodata after makedata.

.gitea/scripts/tests/cypress-changes.test.sh

@@ -26,7 +26,8 @@ check() { # <expected> <path>...
 # deploys, the CLI pin it installs, the scripts that decide what it runs, and
 # the dependency/workflow files that change the first two.
 check true client/src/app/app.component.ts
-check true sas/mocks/sasjs/services/makedata.js
+check true sas/mocks/sasjs/services/admin/makedata.js
+check true sas/mocks/sasjs/services/admin/createdemodata.js
 check true sas/package.json
 check true .gitea/scripts/cypress-specs.sh
 check true package-lock.json

.gitea/workflows/build.yaml

@@ -470,6 +470,10 @@ jobs:
           # NOTE: -d paths resolve from the sasjs project root (sas/mocks),
           # not the cwd (sas/mocks/sasjs).
           sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true
+          # Then the Cypress fixtures: TESTDATA, the search/wide viewers and the
+          # multi-load library.  makedata builds only what a real deployment
+          # builds, so the specs need this second service.
+          sasjs request services/admin/createdemodata -t server-ci -o ./createdemodata_out.json
 
       - name: Prepare and run frontend and cypress
         if: steps.changes.outputs.client != 'false'

.gitea/workflows/release.yaml

@@ -215,6 +215,10 @@ jobs:
           # NOTE: -d paths resolve from the sasjs project root (sas/mocks),
           # not the cwd (sas/mocks/sasjs).
           sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true
+          # Then the Cypress fixtures: TESTDATA, the search/wide viewers and the
+          # multi-load library.  makedata builds only what a real deployment
+          # builds, so the specs need this second service.
+          sasjs request services/admin/createdemodata -t server-ci -o ./createdemodata_out.json
 
       - name: Prepare and run frontend and cypress
         env:

.gitea/workflows/sites.yaml

@@ -137,6 +137,9 @@ jobs:
           # expected. A real failure must not be published as an empty site, so
           # the drive is checked before the snapshot is taken.
           ../../node_modules/.bin/sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true
+          # the sample tables the demo site opens with are a separate service
+          # now, so that makedata matches what a SAS deployment builds
+          ../../node_modules/.bin/sasjs request services/admin/createdemodata -t server-ci -o ./createdemodata_out.json
 
           DATA_DIR="../../../sasjs_root/drive/files$APP_LOC/data"
           SEEDED=$(find "$DATA_DIR" -name '*.json' 2>/dev/null | wc -l)

CONTRIBUTING.md

@@ -131,6 +131,13 @@ sasjs cbd -t server-ci
 
 This compiles, builds, and deploys the JS mock services plus the Angular frontend (as a streaming app) to the SASjs Server. The `server-ci` target in `sas/mocks/sasjs/sasjsconfig.json` points to `http://localhost:5000` with appLoc `/Public/app/dc` and stream service name `clickme`.
 
+The drive still needs seeding - the control library first, then the Cypress fixtures:
+
+```
+sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json
+sasjs request services/admin/createdemodata -t server-ci
+```
+
 After deploy, the streamed app is available at: `http://localhost:5000/AppStream/clickme/`
 
 ### 3. Run the Angular dev server (optional, for hot reload)
@@ -147,9 +154,11 @@ The app will be at `http://localhost:4200`. It calls the JS mock services on por
 
 ### How the mocks work
 
-Each `.js` file in `sas/mocks/sasjs/services/` corresponds to a real SAS service (e.g. `public/startupservice.js` mimics `services/public/startupservice.sas`). The JS runtime predeclares `fs`, `_program` (the program's deployed path), `weboutPath` (the response body it writes) and `_webout`, and the services emit their members through the runtime's `webout()` - `webout('OPEN')`, one `webout('OBJ', name, rows)` (or `webout('ARR', name, rows)` for positional rows) per member, `webout('CLOSE')` - so the response envelope comes from the host and the file carries only what the SAS service returns. A few services still write the body themselves through `_webout`: the ones that emit plain text, CSV, DDL, HTML or a file download and set their own content headers (`admin/exportconfig`, `admin/exportdb`, `admin/makedata`, `auditors/getauditfile`, `public/getddl`, `public/getrawdata`). `sas/mocks/test/sasjs-webout.test.js` holds that list, so a service that goes back to writing its own body fails the suite. Some mocks are stateful: they persist data to a `mock-storage/` folder on the SASjs Drive (e.g. `licence.json`, `users.json`, `filter.txt`) using the predeclared `fs` module.
+Each `.js` file in `sas/mocks/sasjs/services/` corresponds to a real SAS service (e.g. `public/startupservice.js` mimics `services/public/startupservice.sas`). `admin/createdemodata.js` is the exception: it has no SAS counterpart, because it exists only to give the test estate tables a deployed site would not have. The JS runtime predeclares `fs`, `_program` (the program's deployed path), `weboutPath` (the response body it writes) and `_webout`, and the services emit their members through the runtime's `webout()` - `webout('OPEN')`, one `webout('OBJ', name, rows)` (or `webout('ARR', name, rows)` for positional rows) per member, `webout('CLOSE')` - so the response envelope comes from the host and the file carries only what the SAS service returns. A few services still write the body themselves through `_webout`: the ones that emit plain text, CSV, DDL, HTML or a file download and set their own content headers (`admin/exportconfig`, `admin/exportdb`, `admin/makedata`, `auditors/getauditfile`, `public/getddl`, `public/getrawdata`). `sas/mocks/test/sasjs-webout.test.js` holds that list, so a service that goes back to writing its own body fails the suite. Some mocks are stateful: they persist data to a `mock-storage/` folder on the SASjs Drive (e.g. `licence.json`, `users.json`, `filter.txt`) using the predeclared `fs` module.
 
-The mock data includes sample tables (`DC996664.MPE_X_TEST`, `MPE_X_NEW`, `MPE_X_FORMULA_TEST`, etc.) with realistic columns, validation rules (HARDREGEX, SOFTREGEX, HARDSELECT, READONLY, HIDDEN, ROUND, NUMBER_FORMAT), and dropdown values. This is sufficient to exercise the full editor grid, approval workflow, viewer, and admin screens.
+`admin/makedata` builds only what the SAS service builds: the control library defined by `mpe_makedatamodel.sas`, seeded with the rows `mpe_makedata.sas` inserts. `sas/mocks/test/makedata-parity.test.js` reads both macros and fails when the mock estate drifts from them, so the mock and a deployed site open with the same tables and the same configuration.
+
+The fixtures the Cypress suite needs but a DC install does not have live in `admin/createdemodata`, which runs after makedata: the `TESTDATA` demo library (`MPE_X_NEW`, `MPE_X_FORMULA_TEST`, `DERIVED_DEMO`, etc.) with its validation rules (HARDREGEX, SOFTREGEX, HARDSELECT, READONLY, HIDDEN, ROUND, NUMBER_FORMAT) and dropdown values, the `MPE_X_SEARCH` and `MPE_X_WIDE` viewer fixtures, and the `DC996664` multi-load library. It is safe to run twice, and refuses to run before makedata.
 
 There is also a SAS 9 mock variant under `sas/mocks/sas9/` (target `sas9-mocks`) that mimics SAS 9 stored processes with the same JS approach but a different appLoc (`/User Folders/sasdemo/`). Its services are fixed recordings of the SAS 9 responses, emitted through `webout()` in the same way; the payloads are the SAS 9 ones (metadata URIs, `BUILTIN\Administrators` owners, `D:\...` paths, with their separators - the front end splits on `\`, so a value that arrives without it exercises the wrong branch). Only the payloads are SAS 9 flavour: the envelope around them (`_PROGRAM`, `SYSUSERID`, `SYSVLONG`, `SYSDATE`/`SYSTIME`) is whatever runtime serves the request, so do not read this variant for SAS 9 envelope values.
 

client/cypress/e2e/full-table-search.cy.ts

@@ -12,7 +12,7 @@ const serverType = Cypress.env('serverType')
 const libraryToOpenIncludes = Cypress.env(`libraryToOpenIncludes_${serverType}`)
 
 // Fixture: MPE_X_SEARCH, a 1000-row demo table built by the mock data builder
-// (sas/mocks/sasjs/services/admin/makedata.js).  It is deterministic, so every
+// (sas/mocks/sasjs/services/admin/createdemodata.js).  It is deterministic, so every
 // count below can be derived by hand:
 //   - 1000 rows, PK 1001-2000, columns in demo order: PRIMARY_KEY_FIELD,
 //     SITE_NAME, VESSEL, SAMPLE_COUNT, DEPTH_M, SPECIES, CRUISE_DATE,

sas/mocks/test/createdemodata.test.js

@@ -0,0 +1,79 @@
+/**
+ * createdemodata carries the estate the Cypress suite needs but a Data
+ * Controller install does not have: the TESTDATA demo library, the deep and
+ * wide viewer fixtures and the multi-load library.
+ *
+ * It runs after makedata, so it must both add those and leave the control
+ * library alone otherwise; and it must be safe to run twice, because a reseed
+ * runs it twice on the same drive.
+ *
+ * Run with: node --test sas/mocks/test/
+ */
+const test = require('node:test')
+const assert = require('node:assert')
+const fs = require('node:fs')
+const path = require('node:path')
+
+const { seededEstate, runService, readTable } = require('./harness')
+
+const FIXTURE_LIBS = ['TESTDATA', 'DC996664']
+
+test('the fixtures are added to a makedata estate', () => {
+  const estate = seededEstate()
+  const data = path.join(estate.appDir, 'data')
+  assert.deepEqual(fs.readdirSync(data).sort(), [
+    'DC996664',
+    'DC_JSLIB',
+    'TESTDATA'
+  ])
+
+  const tables = readTable(estate, 'MPE_TABLES')
+  for (const lib of FIXTURE_LIBS) {
+    assert.ok(
+      tables.some((row) => String(row.libref).toUpperCase() === lib),
+      lib + ' is registered in MPE_TABLES'
+    )
+  }
+  for (const name of ['MPE_X_SEARCH', 'MPE_X_WIDE']) {
+    assert.ok(
+      tables.some((row) => String(row.dsn).toUpperCase() === name),
+      name + ' is registered in MPE_TABLES'
+    )
+  }
+
+  assert.equal(readTable(estate, 'MPE_X_SEARCH').length, 1000)
+  assert.equal(readTable(estate, 'MPE_X_WIDE').length, 200)
+  assert.ok(readTable(estate, 'MPE_DERIVED').length > 0)
+  assert.ok(
+    readTable(estate, 'MPE_VALIDATIONS').some(
+      (row) => String(row.base_lib).toUpperCase() === 'TESTDATA'
+    ),
+    'the TESTDATA rules are present'
+  )
+  assert.ok(
+    fs.existsSync(path.join(data, 'TESTDATA', 'demo_01.json')),
+    'the TESTDATA tables are written'
+  )
+})
+
+test('a second run does not accumulate rows', () => {
+  const estate = seededEstate()
+  const counted = () =>
+    ['MPE_TABLES', 'MPE_VALIDATIONS', 'MPE_SELECTBOX', 'MPE_SECURITY']
+      .map((name) => name + '=' + readTable(estate, name).length)
+      .join(' ')
+
+  const first = counted()
+  runService(estate, 'admin/createdemodata', {})
+  assert.equal(counted(), first, 'createdemodata must be idempotent')
+})
+
+test('it refuses to run before makedata', () => {
+  const { makeEstate } = require('./harness')
+  const estate = makeEstate()
+  assert.throws(
+    () => runService(estate, 'admin/createdemodata', {}),
+    /run services\/admin\/makedata first/,
+    'a missing control library is reported, not half-created'
+  )
+})

sas/mocks/test/harness.js

@@ -168,12 +168,18 @@ function runService(
   return stored || returned
 }
 
-/** Seeds the estate exactly as the deploy screen does, then returns it. */
-function seededEstate() {
+/**
+ * Seeds the estate the way the CI estate is built: makedata for the control
+ * library, then createdemodata for the Cypress fixtures.  Pass
+ * `{ demo: false }` for the control library alone, which is what a real
+ * deployment gets.
+ */
+function seededEstate({ demo = true } = {}) {
   const estate = makeEstate()
   runService(estate, 'admin/makedata', {
     params: { action: 'MAKEDATA', admin: 'AllUsers' }
   })
+  if (demo) runService(estate, 'admin/createdemodata', {})
   return estate
 }
 

sas/mocks/test/makedata-parity.test.js

@@ -0,0 +1,200 @@
+/**
+ * The mock makedata must build the estate the SAS service builds.
+ *
+ * makedata.sas (the `server` target) runs `%mpe_makedatamodel` for the empty
+ * tables and `%mpe_makedata` for the rows.  The JS mock in
+ * sas/mocks/sasjs/services/admin/makedata.js is a second implementation of
+ * those two macros, and the two used to disagree: two model tables
+ * (MPE_MAXKEYVALUES, MPE_SIGNOFFS) were never created, and a handful of seed
+ * rows were missing.  Nothing failed, because the client only asks for the
+ * tables it opens.
+ *
+ * So the macros are read here, and the estate the mock seeds is compared with
+ * what they declare.  A change to either side that is not made on the other
+ * fails this test.
+ *
+ * Run with: node --test sas/mocks/test/
+ */
+const test = require('node:test')
+const assert = require('node:assert')
+const fs = require('node:fs')
+const path = require('node:path')
+
+const { seededEstate, readTable } = require('./harness')
+
+const MACROS = path.join(__dirname, '..', '..', 'sasjs', 'macros')
+const MODEL = fs.readFileSync(
+  path.join(MACROS, 'mpe_makedatamodel.sas'),
+  'utf8'
+)
+const SEED = fs.readFileSync(path.join(MACROS, 'mpe_makedata.sas'), 'utf8')
+
+/** `create table &lib..name(` blocks, with their column definitions. */
+function modelTables(source) {
+  const tables = {}
+  const re = /create table &lib\.\.\s*([A-Za-z0-9_-]+)\s*\(([\s\S]*?)\n\s*\);/gi
+  let match
+  while ((match = re.exec(source)) !== null) {
+    const name = match[1].toUpperCase()
+    const columns = []
+    for (const line of match[2].split('\n')) {
+      const col = /^\s*,?\s*([A-Za-z0-9_]+)\s+(num|char|varchar|float|int)\b/i.exec(
+        line
+      )
+      if (col) columns.push(col[1].toUpperCase())
+    }
+    tables[name] = columns
+  }
+  return tables
+}
+
+/** `insert into &lib..name set` statements, grouped by table. */
+function seedInserts(source) {
+  const body = source.replace(/\/\*[\s\S]*?\*\//g, '')
+  const rows = {}
+  const re = /insert into &lib\.\.\s*([A-Za-z0-9_-]+)\s+set\s+([\s\S]*?);/gi
+  let match
+  while ((match = re.exec(body)) !== null) {
+    const name = match[1].toUpperCase()
+    const columns = []
+    for (const part of match[2].split(/[\n,]/)) {
+      const assignment = /^\s*([A-Za-z0-9_]+)\s*=/.exec(part)
+      if (assignment) columns.push(assignment[1].toUpperCase())
+    }
+    rows[name] = rows[name] || []
+    rows[name].push(columns.sort())
+  }
+  return rows
+}
+
+const model = modelTables(MODEL)
+const seeds = seedInserts(SEED)
+
+/*
+  These four carry rows the mock cannot copy verbatim:
+  - MPE_GROUPS, MPE_ROW_LEVEL_SECURITY and MPE_SECURITY name SAS metadata
+    groups, which a JS-only estate does not have (the mock uses its own).
+  - MPE_X_TEST is seeded with ~490 rows from a ranuni() loop, and the mock
+    keeps the seven deterministic rows only.
+*/
+const ADAPTED = new Set([
+  'MPE_GROUPS',
+  'MPE_ROW_LEVEL_SECURITY',
+  'MPE_SECURITY',
+  'MPE_X_TEST'
+])
+
+test('mpe_makedatamodel declares tables, and the mock creates all of them', () => {
+  const estate = seededEstate({ demo: false })
+  const created = {}
+  const dir = path.join(estate.appDir, 'data', 'DC_JSLIB')
+  for (const file of fs.readdirSync(dir)) {
+    const table = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8'))
+    created[table.name] = table
+  }
+
+  assert.ok(Object.keys(model).length > 30, 'the model was parsed')
+  const missing = Object.keys(model).filter((name) => !created[name])
+  assert.deepEqual(
+    missing,
+    [],
+    'every table the model creates must exist in the mock estate'
+  )
+
+  const known = new Set(Object.keys(model))
+  // the mock writes a marker for the format catalog, which is a SAS catalog
+  // rather than a dataset, so the model does not declare it
+  known.add('MPE_X_CATALOG')
+  const extra = Object.keys(created).filter((name) => !known.has(name))
+  assert.deepEqual(
+    extra,
+    [],
+    'a new table in the mock estate is a table the SAS service does not build'
+  )
+})
+
+test('each table has the model columns, in the model order', () => {
+  const estate = seededEstate({ demo: false })
+  for (const [name, expected] of Object.entries(model)) {
+    const actual = readTable(estate, name)
+    // readTable returns rows; the schema is read from the file instead
+    const file = path.join(
+      estate.appDir,
+      'data',
+      'DC_JSLIB',
+      name.toLowerCase() + '.json'
+    )
+    assert.ok(fs.existsSync(file), name + ' is missing')
+    const table = JSON.parse(fs.readFileSync(file, 'utf8'))
+    assert.deepEqual(
+      (table.columns || []).map((col) => col.name),
+      expected,
+      name + ' columns differ from mpe_makedatamodel.sas'
+    )
+    assert.ok(Array.isArray(actual), name + ' rows')
+  }
+})
+
+test('each seeded table carries the rows mpe_makedata inserts', () => {
+  const estate = seededEstate({ demo: false })
+  for (const [name, inserts] of Object.entries(seeds)) {
+    if (ADAPTED.has(name)) continue
+    const rows = readTable(estate, name)
+    assert.equal(
+      rows.length,
+      inserts.length,
+      name + ' row count differs from mpe_makedata.sas'
+    )
+    // a table whose inserts name optional columns (MPE_XLMAP_RULES sets
+    // XLMAP_FINISH on some ranges only) has a union; every row must sit
+    // inside it, and the union must be the one the macro writes
+    const union = new Set()
+    for (const insert of inserts) for (const field of insert) union.add(field)
+    const expected = Array.from(union).sort()
+    for (const row of rows) {
+      const fields = Object.keys(row)
+        .map((field) => field.toUpperCase())
+        .sort()
+      for (const field of fields) {
+        assert.ok(
+          union.has(field),
+          name + ' row carries ' + field + ', which mpe_makedata never sets'
+        )
+      }
+      assert.deepEqual(
+        fields,
+        expected.filter((field) => fields.includes(field)),
+        name + ' row fields differ from mpe_makedata.sas'
+      )
+    }
+  }
+})
+
+test('the adapted tables are seeded, and document why they differ', () => {
+  const estate = seededEstate({ demo: false })
+  for (const name of ADAPTED) {
+    const rows = readTable(estate, name)
+    assert.ok(rows.length > 0, name + ' is empty')
+  }
+  // the mock's own groups are the ones its services report
+  const groups = readTable(estate, 'MPE_GROUPS')
+  assert.ok(
+    groups.some((row) => row.group_name === 'dc-admin'),
+    'the mock grants its own group, not a SAS metadata one'
+  )
+})
+
+test('makedata leaves no Cypress fixture behind', () => {
+  const estate = seededEstate({ demo: false })
+  const dir = path.join(estate.appDir, 'data')
+  assert.deepEqual(
+    fs.readdirSync(dir).sort(),
+    ['DC_JSLIB'],
+    'the demo libraries belong to createdemodata, not makedata'
+  )
+  const tables = readTable(estate, 'MPE_TABLES')
+  const fixture = tables.filter(
+    (row) => String(row.libref).toUpperCase() !== 'DC_JSLIB'
+  )
+  assert.deepEqual(fixture, [], 'a fixture table is registered by makedata')
+})