feat(mocks): restore a load, and record the diffs it needs

`editors/restore` had no mock, and the reason it could not have had one is that
`postdata` wrote no audit rows: an approval applied the load and left nothing
to rewind.  Both halves are here.

postdata (final approval): writes MPE_AUDIT the way %mp_storediffs does - one
row per column, keyed by the md5 of the business-key values joined by '|', with
move_type A/D/M, is_pk on the key columns, is_diff on a changed one, and the
old and new value on their own typed columns.  An append carries every column
on the new side, a delete every column on the old side (which is what lets a
restore put the row back), and a modification the keys plus only the columns
that actually changed.  A failure writing the audit is logged and does not undo
an approval that has already landed.

restore: mpe_checkrestore's two decidable checks (the table has an audit table;
the load has entries in it) with their messages, then mp_stripdiffs' window -
the load's timestamp and every change to that table from then on, because a
restore reverts this load *and all subsequent ones*.  Per business key the
pre-window state is the earliest event: an append means the row is staged for
deletion (`_____DELETE__THIS__RECORD_____=YES`), a delete means it is staged
with the values the delete recorded, and a modification means each column takes
its earliest old value while untouched columns keep the current one.  The
result goes through the same stageSubmission the editors use, so it arrives as
a new submission with the reason `Restoring <loadref>` and is approved like any
other - restore.sas never writes the base table either.

The access and group/RLS checks are not emulated, consistent with the rest of
the mocks: nothing enforces MPE_SECURITY there.

Tests: 52 pass across the seven mock suites, 7 of them new.  They drive a real
approve and then read the staged rows back: append staged for deletion,
modification staged with the pre-load value, delete staged with the row's
values, and restoring the first of two loads reverting the second as well.
With the base postdata and no restore service, all 7 fail.
This commit is contained in:
dc committed 2026-11-05 10:44:40 +00:00
1 parent 67d9cfd1bd
commit 1a60c689f4
3 files changed
+636

No files matched your search

@@ -713,6 +713,91 @@ if (action === 'SHOW_DIFFS') {
}
saveTableToLib(dsn, updatedTableData, libref)
// ── MPE_AUDIT: record the diffs (%mp_storediffs) ──────────────────────────
// One row per column that changed, so a later restore can rewind the load.
// key_hash is md5 of the business-key values joined by '|' - the same value
// mp_storediffs writes, and what mp_stripdiffs groups a row's history by.
try {
const auditCrypto = require('crypto')
const md5hex = (s) =>
auditCrypto.createHash('md5').update(String(s)).digest('hex').toUpperCase()
const hashOf = (row) =>
md5hex(
buskey
.map((k) => String(colVal(row, k) ?? '').trim())
.join('|')
)
const auditTable = mpeLoadTableData('MPE_AUDIT') || {
name: 'MPE_AUDIT',
columns: [],
rows: []
}
const entry = (row, moveType, column, oldVal, newVal, isPk, isDiff) => {
const type = column.type === 'N' ? 'N' : 'C'
const auditRow = {
LOAD_REF: loadRef,
LIBREF: libref,
DSN: dsn,
KEY_HASH: hashOf(row),
TGTVAR_NM: column.name,
MOVE_TYPE: moveType,
IS_PK: isPk,
IS_DIFF: isDiff,
TGTVAR_TYPE: type,
OLDVAL_NUM: null,
NEWVAL_NUM: null,
OLDVAL_CHAR: null,
NEWVAL_CHAR: null,
PROCESSED_DTTM: sastime
}
const put = (side, value) => {
if (value === null || value === undefined) return
if (type === 'N') auditRow[side + '_NUM'] = Number(value)
else auditRow[side + '_CHAR'] = String(value)
}
put('OLDVAL', oldVal)
put('NEWVAL', newVal)
auditTable.rows.push(auditRow)
}
const isKey = (name) =>
buskey.some((k) => k.toUpperCase() === String(name).toUpperCase())
// appends: the whole row, with the values it was given
for (const row of sortedNew) {
for (const column of diffCols) {
entry(row, 'A', column, null, colVal(row, column.name), isKey(column.name) ? 1 : 0, -1)
}
}
// deletes: the whole row as it was, so the restore can put it back
for (const row of sortedDeleted) {
for (const column of diffCols) {
entry(row, 'D', column, colVal(row, column.name), null, isKey(column.name) ? 1 : 0, -1)
}
}
// modifications: the keys, then each column whose value actually changed
for (let i = 0; i < sortedUpdates.length; i++) {
const row = sortedUpdates[i]
const original = sortedOriginals[i]
for (const column of diffCols) {
if (isKey(column.name)) {
entry(row, 'M', column, colVal(row, column.name), colVal(row, column.name), 1, -1)
} else if (String(colVal(row, column.name)) !== String(colVal(original, column.name))) {
entry(row, 'M', column, colVal(original, column.name), colVal(row, column.name), 0, 1)
}
}
}
if (auditTable.rows.length) saveTableToLib('MPE_AUDIT', auditTable)
} catch (err) {
// an audit failure must not undo an approval that already landed
console.log('Error writing MPE_AUDIT: ' + err.message)
}
// record the approval in MPE_REVIEW - reloaded under the lock, so an
// approval another reviewer appended in the meantime is not dropped by this
// save (the same read-modify-write the duplicate check guards)
+246
View File
@@ -0,0 +1,246 @@
const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
// Load shared DC mock utilities
eval(
fs.readFileSync(
nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'),
'utf8'
)
)
const dcConf = loadDcSettings(driveRoot, appLoc)
const controlTable = makeControlTableLoader(dcConf.dataDir)
const stagingDir = nodePath.resolve(dcConf.dataDir, 'staging')
const DELETE_FLAG = '_____DELETE__THIS__RECORD_____'
/** LIBREF.TABLE split into its two parts, upper-cased like the SAS compares. */
const splitLibds = (libds) => {
const parts = String(libds || '').toUpperCase().split('.')
return { libref: parts[0] || '', dsn: parts[1] || '' }
}
/** The audit table named by MPE_TABLES, read from whichever library holds it. */
const auditRows = (auditLibds) => {
const parts = splitLibds(auditLibds)
const loader = makeTableLoader(libDataDir(parts.libref))
const table = loader(parts.dsn)
return table && table.rows ? table.rows : []
}
/**
* The state a key had before the window: the old value of each column at its
* earliest change, which is what "revert this load and all subsequent" means.
*/
const earliestByColumn = (rows, column) => {
const matching = rows
.filter((row) => upper(colVal(row, 'TGTVAR_NM')) === upper(column))
.sort(
(a, b) =>
Number(colVal(a, 'PROCESSED_DTTM')) - Number(colVal(b, 'PROCESSED_DTTM'))
)
return matching[0]
}
/** The value a diff row carries for its side: char and numeric are separate. */
const valueOf = (row, side) => {
const isChar = upper(colVal(row, 'TGTVAR_TYPE')) === 'C'
const value = colVal(row, side === 'old' ? 'OLDVAL_CHAR' : 'NEWVAL_CHAR')
const num = colVal(row, side === 'old' ? 'OLDVAL_NUM' : 'NEWVAL_NUM')
if (isChar) return value === undefined || value === null ? '' : String(value)
return num === undefined || num === null ? null : Number(num)
}
const upper = (value) => String(value === undefined || value === null ? '' : value).toUpperCase()
try {
// ─── Input ──────────────────────────────────────────────────────────────────
const request = fetchTable('restore_in')[0] || {}
const loadRef = String(colVal(request, 'LOAD_REF') || '').trim()
// ─── mpe_checkrestore ───────────────────────────────────────────────────────
// The access and group checks are not emulated (no mock service enforces
// MPE_SECURITY); the two checks that decide whether a restore is possible at
// all are, because their messages are what the screen shows.
const submitRow = controlTable('MPE_SUBMIT').find(
(row) => String(colVal(row, 'TABLE_ID')) === loadRef
)
const base = submitRow
? splitLibds(
String(colVal(submitRow, 'BASE_LIB')) +
'.' +
String(colVal(submitRow, 'BASE_DS'))
)
: { libref: '', dsn: '' }
const tableReg = controlTable('MPE_TABLES')
.filter(isCurrentRow)
.find(
(row) =>
upper(colVal(row, 'LIBREF')) === base.libref &&
upper(colVal(row, 'DSN')) === base.dsn
)
const auditLibds =
(tableReg && colVal(tableReg, 'AUDIT_LIBDS')) || dcConf.dcLibref + '.MPE_AUDIT'
if (!tableReg) {
mpAbort(
'Cannot restore because: ' +
base.libref +
'.' +
base.dsn +
' has no audit table configured',
'restore'
)
}
const all = auditRows(auditLibds)
const mine = all.filter((row) => String(colVal(row, 'LOAD_REF')) === loadRef)
if (!mine.length) {
mpAbort(
'Cannot restore because: No entry for ' + loadRef + ' in ' + auditLibds,
'restore'
)
}
// ─── mp_stripdiffs ──────────────────────────────────────────────────────────
// The load's timestamp, then every change to that table from then on: the
// change to revert plus all subsequent ones.
const stamp = Math.min(
...mine.map((row) => Number(colVal(row, 'PROCESSED_DTTM')))
)
const window = all.filter(
(row) =>
upper(colVal(row, 'LIBREF')) === base.libref &&
upper(colVal(row, 'DSN')) === base.dsn &&
Number(colVal(row, 'PROCESSED_DTTM')) >= stamp
)
if (!window.length) {
mpAbort('THERE ARE NO DIFFERENCES TO APPLY', 'restore')
}
// ─── Rebuild the rows to stage ──────────────────────────────────────────────
const baseTable =
makeTableLoader(libDataDir(base.libref))(base.dsn) || { columns: [], rows: [] }
const columns = (baseTable.columns || []).map((column) => column.name)
const byHash = {}
for (const row of window) {
const hash = String(colVal(row, 'KEY_HASH'))
if (!byHash[hash]) byHash[hash] = []
byHash[hash].push(row)
}
const stagedRows = []
for (const hash of Object.keys(byHash)) {
const rows = byHash[hash].slice().sort(
(a, b) =>
Number(colVal(a, 'PROCESSED_DTTM')) - Number(colVal(b, 'PROCESSED_DTTM'))
)
const earliest = rows[0]
const moveType = upper(colVal(earliest, 'MOVE_TYPE'))
// the key values travel on the is_pk rows of the earliest event
const pkRows = rows.filter((row) => Number(colVal(row, 'IS_PK')) === 1)
const keyValues = {}
for (const row of pkRows) {
// coalesce(old, new): an append carries its key on the new side, a delete
// on the old one - mp_stripdiffs coalesces them for the same reason
const value = valueOf(row, 'old')
keyValues[upper(colVal(row, 'TGTVAR_NM'))] =
value === null || value === '' ? valueOf(row, 'new') : value
}
const current = baseTable.rows.find((row) => {
// match on the key values that the audit recorded
return Object.keys(keyValues).every(
(name) => String(colVal(row, name)) === String(keyValues[name])
)
})
if (moveType === 'A') {
// the row did not exist before this append, so restoring removes it
const out = { [DELETE_FLAG]: 'YES' }
for (const name of columns) {
out[name] = current ? colVal(current, name) : keyValues[upper(name)]
}
stagedRows.push(out)
continue
}
const out = { [DELETE_FLAG]: 'No' }
for (const name of columns) {
if (moveType === 'D') {
// a deleted row comes back with the values the delete recorded
const del = rows.find(
(row) =>
upper(colVal(row, 'MOVE_TYPE')) === 'D' &&
upper(colVal(row, 'TGTVAR_NM')) === upper(name)
)
out[name] = del
? valueOf(del, 'old')
: current
? colVal(current, name)
: null
continue
}
// a modification: the column's earliest old value is the pre-window one
const change = earliestByColumn(rows, name)
const changed =
change &&
upper(colVal(change, 'MOVE_TYPE')) === 'M' &&
Number(colVal(change, 'IS_PK')) === 0 &&
Number(colVal(change, 'IS_DIFF')) !== 0
out[name] = changed
? valueOf(change, 'old')
: current
? colVal(current, name)
: keyValues[upper(name)]
}
stagedRows.push(out)
}
if (!stagedRows.length) {
mpAbort('THERE ARE NO DIFFERENCES TO APPLY', 'restore')
}
// ─── mpe_loader: stage it as a new submission ───────────────────────────────
const newRef = makeTableId()
const headers = [DELETE_FLAG].concat(columns)
stageSubmission({
dataDir: dcConf.dataDir,
stagingDir,
tableId: newRef,
libds: base.libref + '.' + base.dsn,
libref: base.libref,
dsn: base.dsn,
message: 'Restoring ' + loadRef,
headers,
rows: stagedRows,
auditFiles: [
{
name: base.libref + '.' + base.dsn + '.csv',
content: toCsv(headers, stagedRows)
}
]
})
webout('OPEN')
webout('OBJ', 'restore_out', [{ loadref: newRef }])
webout('CLOSE')
} catch (err) {
if (!(err && err.mockAbort)) throw err
}
+305
View File
@@ -0,0 +1,305 @@
/**
* The restore service, exercised through a real approve.
*
* restore.sas rewinds a load: it reads the diffs the approval recorded in the
* audit table, takes that load's timestamp, and rebuilds the state the table
* had before it - including every change made since (`mp_stripdiffs` reverts
* "this change, plus ALL SUBSEQUENT CHANGES"). The result is staged as a new
* submission for approval, never applied directly.
*
* Two gaps made that impossible on the mock estate: `postdata` wrote no audit
* rows at all, and there was no restore service. These cases cover both.
*
* Run with: node --test sas/mocks/test/
*/
const test = require('node:test')
const assert = require('node:assert')
const fs = require('node:fs')
const path = require('node:path')
const { seededEstate, runService, readTable, writeTable } = require('./harness')
const BASE_LIB = 'TESTDATA'
const BASE_DS = 'DEMO_01'
const COLUMNS = [
'ID',
'AMOUNT',
'SCORE',
'GRADE',
'REF',
'RATING',
'STATUS',
'LAST_REVIEWED'
]
const DELETE_FLAG = '_____DELETE__THIS__RECORD_____'
/** A submission waiting for its single approval. */
function stage(estate, tableId, rows) {
const stageFolder = path.join(
estate.appDir,
'data',
'DC_JSLIB',
'staging',
tableId
)
fs.mkdirSync(stageFolder, { recursive: true })
fs.writeFileSync(
path.join(stageFolder, 'staged.json'),
JSON.stringify({
table_id: tableId,
libds: BASE_LIB + '.' + BASE_DS,
libref: BASE_LIB,
dsn: BASE_DS,
submitted_by_nm: 'DEMO',
submitted_on_dttm: Date.now() / 1000,
headers: COLUMNS,
rows
})
)
// appended, not replaced: a later load on the same table must not erase the
// submission row an earlier one still needs
writeTable(
estate,
'MPE_SUBMIT',
readTable(estate, 'MPE_SUBMIT').concat([
{
TABLE_ID: tableId,
BASE_LIB: BASE_LIB,
BASE_DS: BASE_DS,
SUBMIT_STATUS_CD: 'SUBMITTED',
NUM_OF_APPROVALS_REQUIRED: 1,
NUM_OF_APPROVALS_REMAINING: 1
}
])
)
writeTable(estate, 'MPE_REVIEW', [])
writeTable(estate, 'MPE_LOCKANYTABLE', [])
}
function approve(estate, tableId) {
return JSON.parse(
runService(estate, 'auditors/postdata', {
tableName: 'SASControlTable',
rows: [{ ACTION: 'APPROVE_TABLE', TABLE: tableId }]
})
)
}
const baseRows = (estate) =>
JSON.parse(
fs.readFileSync(path.join(estate.appDir, 'data', BASE_LIB, 'demo_01.json'), 'utf8')
).rows
/** Runs restore and returns the rows it staged for the new submission. */
function restore(estate, loadRef) {
const body = JSON.parse(
runService(estate, 'editors/restore', {
tableName: 'restore_in',
rows: [{ LOAD_REF: loadRef }]
})
)
if (body.sasjsAbort) return body
const newRef = body.restore_out[0].loadref
const staged = JSON.parse(
fs.readFileSync(
path.join(
estate.appDir,
'data',
'DC_JSLIB',
'staging',
newRef,
'staged.json'
),
'utf8'
)
)
return { newRef, rows: staged.rows, message: staged }
}
const rowWithId = (rows, id) =>
rows.filter((row) => Number(row.ID) === Number(id))[0]
// ─── the audit trail ─────────────────────────────────────────────────────────
test('an approval records the diffs in MPE_AUDIT', () => {
const estate = seededEstate()
stage(estate, 'DC-LOAD-1', [
{
ID: 1,
AMOUNT: 999,
SCORE: 82,
GRADE: '.a',
REF: 1001,
RATING: 4,
STATUS: 1,
LAST_REVIEWED: 23390
}
])
approve(estate, 'DC-LOAD-1')
const audit = readTable(estate, 'MPE_AUDIT')
assert.ok(audit.length > 0, 'the approval wrote audit rows')
const amounts = audit.filter(
(row) => row.TGTVAR_NM === 'AMOUNT' && row.MOVE_TYPE === 'M'
)
assert.equal(amounts.length, 1)
assert.equal(amounts[0].OLDVAL_NUM, 120, 'the value before the change')
assert.equal(amounts[0].NEWVAL_NUM, 999)
assert.equal(amounts[0].IS_DIFF, 1)
assert.equal(String(amounts[0].LOAD_REF), 'DC-LOAD-1')
// the business key travels on its own rows, which is how a key is rebuilt
const keys = audit.filter((row) => Number(row.IS_PK) === 1)
assert.ok(keys.some((row) => row.TGTVAR_NM === 'ID' && row.NEWVAL_NUM === 1))
assert.ok(
keys.every((row) => /^[0-9A-F]{32}$/.test(String(row.KEY_HASH))),
'the key hash is md5 hex, as mp_storediffs writes it'
)
})
// ─── restore ────────────────────────────────────────────────────────────────
test('restoring an append stages the row for deletion', () => {
const estate = seededEstate()
const appended = {
ID: 99,
AMOUNT: 100,
SCORE: 50,
GRADE: '.b',
REF: 1002,
RATING: 3,
STATUS: 1,
LAST_REVIEWED: 23390
}
stage(estate, 'DC-LOAD-2', [appended])
approve(estate, 'DC-LOAD-2')
assert.ok(rowWithId(baseRows(estate), 99), 'the append reached the base table')
const restored = restore(estate, 'DC-LOAD-2')
assert.equal(restored.rows.length, 1)
assert.equal(restored.rows[0][DELETE_FLAG], 'YES')
assert.equal(Number(restored.rows[0].ID), 99)
assert.equal(restored.message.submitted_by_nm, 'DEMO')
})
test('restoring a modification stages the values it had before the load', () => {
const estate = seededEstate()
const before = rowWithId(baseRows(estate), 1)
stage(estate, 'DC-LOAD-3', [
{
ID: 1,
AMOUNT: 999,
SCORE: before.SCORE,
GRADE: before.GRADE,
REF: before.REF,
RATING: before.RATING,
STATUS: before.STATUS,
LAST_REVIEWED: before.LAST_REVIEWED
}
])
approve(estate, 'DC-LOAD-3')
assert.equal(Number(rowWithId(baseRows(estate), 1).AMOUNT), 999)
const restored = restore(estate, 'DC-LOAD-3')
assert.equal(restored.rows.length, 1)
assert.equal(restored.rows[0][DELETE_FLAG], 'No')
assert.equal(Number(restored.rows[0].ID), 1)
assert.equal(
Number(restored.rows[0].AMOUNT),
Number(before.AMOUNT),
'the amount is the one the row had before the load'
)
})
test('restoring a delete brings the row back with its values', () => {
const estate = seededEstate()
const before = rowWithId(baseRows(estate), 2)
stage(estate, 'DC-LOAD-4', [
Object.assign({}, before, { [DELETE_FLAG]: 'YES' })
])
approve(estate, 'DC-LOAD-4')
assert.equal(rowWithId(baseRows(estate), 2), undefined, 'the row was deleted')
const restored = restore(estate, 'DC-LOAD-4')
assert.equal(restored.rows.length, 1)
assert.equal(restored.rows[0][DELETE_FLAG], 'No')
assert.equal(Number(restored.rows[0].ID), 2)
assert.equal(
Number(restored.rows[0].AMOUNT),
Number(before.AMOUNT),
'the deleted row comes back as it was'
)
})
test('restoring a load also reverts the changes made after it', () => {
const estate = seededEstate()
const before = rowWithId(baseRows(estate), 1)
// two approvals touching the same row
stage(estate, 'DC-LOAD-5', [
Object.assign({}, before, { AMOUNT: 200 })
])
approve(estate, 'DC-LOAD-5')
stage(estate, 'DC-LOAD-6', [
Object.assign({}, before, { AMOUNT: 300 })
])
approve(estate, 'DC-LOAD-6')
assert.equal(Number(rowWithId(baseRows(estate), 1).AMOUNT), 300)
// restoring the FIRST load has to undo the second one too
const restored = restore(estate, 'DC-LOAD-5')
assert.equal(restored.rows.length, 1)
assert.equal(
Number(restored.rows[0].AMOUNT),
Number(before.AMOUNT),
'the state before the first load, not the value the second one wrote'
)
})
// ─── refusals ───────────────────────────────────────────────────────────────
test('restoring a load that is not in the audit table is refused', () => {
const estate = seededEstate()
// a submission for the table, so the audit table itself is known
stage(estate, 'DC-LOAD-7', [])
const body = restore(estate, 'DC-LOAD-7')
assert.deepEqual(body.sasjsAbort, [
{
MSG:
'Cannot restore because: No entry for DC-LOAD-7 in DC_JSLIB.MPE_AUDIT',
MAC: 'restore'
}
])
})
test('restoring a table with no audit table configured is refused', () => {
const estate = seededEstate()
stage(estate, 'DC-LOAD-8', [])
writeTable(
estate,
'MPE_TABLES',
readTable(estate, 'MPE_TABLES').filter(
(row) => !(row.libref === BASE_LIB && row.dsn === BASE_DS)
)
)
const body = restore(estate, 'DC-LOAD-8')
assert.equal(body.sasjsAbort.length, 1)
assert.equal(
body.sasjsAbort[0].MSG,
'Cannot restore because: TESTDATA.DEMO_01 has no audit table configured'
)
})