dc
a4a0f9cb2f
@sasjs/core 6.1.3 moved the derived variable field rules out of mp_derivedvars into mp_derivedvarcheck, so the same scan can run before a definition is used. DC carried its own copy of that list in mpe_derived_check.sas, which is now deleted: the post edit hook calls the core macro instead, and the hook keeps only the rules that belong to the control table rather than the expression - LIBREF/DSN required, and ACTIVE within the picker pair. The scan is stricter than the copy it replaces, in three ways: - VAR_NM may not be _N_, _ERROR_ or _IORC_ (automatic variables of the data step) - VAR_LOGIC may not exceed 4000 characters - VAR_LOGIC may not call a function that submits code or reaches the file system, the environment or the macro symbol table - a function name cannot be assembled at run time, so this is a control the expression cannot evade and it relaxes one: VAR_LABEL may carry a line break, which is flattened to a space when the label is used, so only the remaining control characters are refused. The mock hook mirror is updated to the same rules, and stagedata.test.4 gains the two submit-time cases that pin the new wiring.
411 lines
18 KiB
JavaScript
411 lines
18 KiB
JavaScript
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Mock of services/hooks/mpe_derived_postedit
|
|
//
|
|
// Registered against DC_JSLIB.MPE_DERIVED (MPE_TABLES.post_edit_hook). This
|
|
// file is eval'd by editors/stagedata.js and editors/loadfile.js, mirroring how
|
|
// mpe_loader.sas %includes the hook program via %mpe_runhook, so it shares the
|
|
// caller's scope: jsDataRows (the submitted rows - work.staging_ds on the
|
|
// server), libref, dsn, mpeDataDir and the dcMockUtils helpers.
|
|
//
|
|
// A hook rejects a submission by setting hookErrmsg, which the caller turns
|
|
// into a sasjsAbort - the same outcome as the SAS hook's %mp_abort.
|
|
//
|
|
// The rules mirror services/hooks/mpe_derived_postedit.sas, in its order:
|
|
// normalise the incoming rows, reject a duplicate name and a bad field, then
|
|
// check the dependency graph that the resulting configuration would have (the
|
|
// incoming rows plus the current rows they do not replace). The field rules
|
|
// are core's mp_derivedvarcheck, which the server hook calls and the read-time
|
|
// engine runs too - this file spells them out a second time, so a change there
|
|
// has to be mirrored here. As on the server, the collision between VAR_NM and
|
|
// a physical column is left to the read-time engine, which runs with the target
|
|
// library assigned - the hook does not have it here either.
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
;(function mpeDerivedPostedit() {
|
|
const DELETE_MARK = '_____DELETE__THIS__RECORD_____'
|
|
const NAME_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
|
|
const FMT_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.'
|
|
|
|
const text = (v) => (v === undefined || v === null ? '' : String(v))
|
|
const up = (v) => text(v).trim().toUpperCase()
|
|
const strip = (v) => text(v).trim()
|
|
|
|
/** Reads a field as the hook normalised it. */
|
|
const field = (row, name) => up(colVal(row, name))
|
|
|
|
/** Writes a field back under the key the row already carries. */
|
|
const setField = (row, name, value) => {
|
|
const key = Object.keys(row).find((k) => k.toUpperCase() === name)
|
|
if (key) row[key] = value
|
|
else row[name] = value
|
|
}
|
|
|
|
const rows = Array.isArray(jsDataRows) ? jsDataRows : []
|
|
const isDeleted = (row) => field(row, DELETE_MARK) === 'YES'
|
|
const isLive = (row) =>
|
|
!isDeleted(row) && Number(colVal(row, 'ACTIVE')) === 1
|
|
const keyOf = (row) =>
|
|
[field(row, 'LIBREF'), field(row, 'DSN'), field(row, 'VAR_NM')].join(
|
|
'.'
|
|
)
|
|
|
|
const reject = (msg) => {
|
|
hookErrmsg = msg
|
|
}
|
|
|
|
// ─── Normalise the incoming values ─────────────────────────────────────────
|
|
for (const row of rows) {
|
|
if (isDeleted(row)) continue
|
|
for (const name of [
|
|
'LIBREF',
|
|
'DSN',
|
|
'VAR_NM',
|
|
'VAR_TYPE',
|
|
'VAR_FMT',
|
|
'VAR_DEPS'
|
|
]) {
|
|
setField(row, name, up(colVal(row, name)))
|
|
}
|
|
for (const name of ['VAR_LABEL', 'VAR_LOGIC']) {
|
|
setField(row, name, strip(colVal(row, name)))
|
|
}
|
|
}
|
|
|
|
// ─── Duplicate names after normalisation ───────────────────────────────────
|
|
// mpe_loader compares the raw values, so a case variant reaches this point
|
|
const seen = new Set()
|
|
for (const row of rows) {
|
|
if (isDeleted(row)) continue
|
|
const key = keyOf(row)
|
|
if (seen.has(key)) {
|
|
return reject(
|
|
`MPE_DERIVED: ${field(row, 'VAR_NM')} is declared more than once for ${field(row, 'LIBREF')}.${field(row, 'DSN')}`
|
|
)
|
|
}
|
|
seen.add(key)
|
|
}
|
|
|
|
// ─── Field level checks ────────────────────────────────────────────────────
|
|
// Two passes, in the order the SAS hook runs them. The first is the fields
|
|
// this hook owns - the control table key, which the loader normalises but
|
|
// does not require, and ACTIVE, the picker pair the EDIT screen offers.
|
|
for (const row of rows) {
|
|
if (isDeleted(row)) continue
|
|
const name = field(row, 'VAR_NM')
|
|
let reason = ''
|
|
if (!field(row, 'LIBREF') || !field(row, 'DSN'))
|
|
reason = 'LIBREF and DSN are required'
|
|
// ACTIVE is a picker in the EDIT screen (HARDSELECT over the
|
|
// MPE_SELECTBOX pair) - the same pair the hook enforces
|
|
else if (!['0', '1'].includes(String(colVal(row, 'ACTIVE')).trim()))
|
|
reason = 'ACTIVE must be 1 or 0'
|
|
if (reason) return reject(`MPE_DERIVED: ${name}: ${reason}`)
|
|
}
|
|
|
|
// The second pass is the rules for the derived variable itself, which are
|
|
// core's (mp_derivedvarcheck): the engine runs the same macro at read time,
|
|
// so a definition is measured against one rule list wherever it is checked.
|
|
|
|
// C0 controls and DEL: what a label cannot carry once a line break has been
|
|
// flattened to a space, which is what the engine does with it
|
|
const CONTROL_CHARS = /[\u0000-\u001F\u007F]/
|
|
const flatten = (v) => v.replace(/[\r\n]+/g, ' ')
|
|
const AUTOMATIC_VARS = ['_N_', '_ERROR_', '_IORC_']
|
|
|
|
// the capabilities refused by name - a function name cannot be assembled at
|
|
// run time, so naming them is a control the expression cannot evade
|
|
const DENY_FUNCS = new RegExp(
|
|
'\\b(DOSUBL|JAVAOBJ|JAVAMETA|SYMGET|SYMPUTX?|SYSGET|GETOPTION' +
|
|
'|PATHNAME|FILEEXIST|FOPEN|FREAD|FGET|FCLOSE' +
|
|
'|DOPEN|DREAD|DNUM|MOPEN|SLEEP' +
|
|
'|FILENAME|FDELETE|FCOPY|DCREATE|DLCREATE)\\s*\\('
|
|
)
|
|
|
|
// quoted literals are data, not calls
|
|
const unquoted = (v) =>
|
|
v.replace(/'([^']|'')*'/g, ' ').replace(/"([^"]|"")*"/g, ' ')
|
|
|
|
// true when the value carries a character outside the allowed set
|
|
const hasInvalidChar = (value, chars) =>
|
|
[...value].some((c) => !chars.includes(c))
|
|
|
|
for (const row of rows) {
|
|
if (isDeleted(row)) continue
|
|
const name = field(row, 'VAR_NM')
|
|
const type = field(row, 'VAR_TYPE')
|
|
const len = Number(colVal(row, 'VAR_LEN'))
|
|
const fmt = field(row, 'VAR_FMT')
|
|
const label = strip(colVal(row, 'VAR_LABEL'))
|
|
const logic = strip(colVal(row, 'VAR_LOGIC'))
|
|
|
|
let reason = ''
|
|
if (!name) reason = 'VAR_NM is required'
|
|
else if (hasInvalidChar(name, NAME_CHARS))
|
|
reason = 'VAR_NM must be a valid SAS name'
|
|
else if ('0123456789'.includes(name[0]))
|
|
reason = 'VAR_NM must not start with a digit'
|
|
else if (AUTOMATIC_VARS.includes(name))
|
|
reason = 'VAR_NM is an automatic variable of the data step'
|
|
else if (name === DELETE_MARK) reason = 'VAR_NM is reserved'
|
|
else if (type !== 'C' && type !== 'N')
|
|
reason = 'VAR_TYPE must be C or N'
|
|
else if (!Number.isInteger(len) || len < 1)
|
|
reason = 'VAR_LEN must be a positive integer'
|
|
else if (type === 'C' && len > 32767)
|
|
reason = 'VAR_LEN exceeds 32767 for a character variable'
|
|
else if (type === 'N' && len > 8)
|
|
reason = 'VAR_LEN exceeds 8 for a numeric variable'
|
|
else if (
|
|
fmt.includes(';') ||
|
|
fmt.includes('&') ||
|
|
fmt.includes('%') ||
|
|
fmt.includes('/*') ||
|
|
fmt.includes('*/')
|
|
)
|
|
reason = 'VAR_FMT must be a format name, without ; & % or /* */'
|
|
else if (hasInvalidChar(fmt, FMT_CHARS))
|
|
reason = 'VAR_FMT must be a format name'
|
|
// free text, in any script - what is refused is a control character,
|
|
// and a value the engine would cut to 256
|
|
else if (CONTROL_CHARS.test(flatten(label)))
|
|
reason = 'VAR_LABEL must be plain text - no control characters'
|
|
else if ([...label].length > 256)
|
|
reason = 'VAR_LABEL exceeds 256 characters'
|
|
else if (!logic) reason = 'VAR_LOGIC is required'
|
|
else if (logic.length > 4000)
|
|
reason = 'VAR_LOGIC exceeds 4000 characters'
|
|
else if (logic.includes(';'))
|
|
reason = 'VAR_LOGIC must be a single expression - remove the ;'
|
|
else if (logic.includes('&') || logic.includes('%'))
|
|
reason = 'VAR_LOGIC must not contain macro triggers'
|
|
else if (logic.includes('/*') || logic.includes('*/'))
|
|
reason = 'VAR_LOGIC must not contain comments'
|
|
else if (DENY_FUNCS.test(unquoted(logic).toUpperCase()))
|
|
reason =
|
|
'VAR_LOGIC may not call a function that submits code or reaches the file system, the environment or the macro symbol table'
|
|
|
|
if (reason) return reject(`MPE_DERIVED: ${name}: ${reason}`)
|
|
}
|
|
|
|
// ─── The graph of the resulting configuration ──────────────────────────────
|
|
// The incoming live rows plus the current rows they do not replace. Only
|
|
// active rows are in it, because the engine reads nothing else - an inactive
|
|
// row can neither resolve a dependency nor be resolved by one.
|
|
const controlDir = mpeDataDir
|
|
const loadControl = makeTableLoader(controlDir)
|
|
const current = loadControl('MPE_DERIVED')
|
|
const nowSeconds = Date.now() / 1000
|
|
|
|
// mpeDerived reads a row as current when TX_TO is missing, zero or in the
|
|
// future (dcMockUtils.js, the read-time engine), so the same test is used
|
|
// here - a seeded row with no TX_TO is live for the engine, and a dependency
|
|
// on it has to validate
|
|
const isCurrent = (row) =>
|
|
!Number(colVal(row, 'TX_TO')) || Number(colVal(row, 'TX_TO')) > nowSeconds
|
|
|
|
const stagedKeys = new Set(rows.map(keyOf))
|
|
const stagedLive = rows.filter(isLive)
|
|
|
|
// the control-table arm of the SAS hook is limited to the tables this
|
|
// submission touches, so another table's configuration cannot block it
|
|
const stagedTables = new Set(
|
|
rows.map((row) => [field(row, 'LIBREF'), field(row, 'DSN')].join('.'))
|
|
)
|
|
|
|
/** libref.dsn.name - the key every graph lookup uses. */
|
|
const nodeKey = (n) => [n.libref, n.dsn, n.name].join('.')
|
|
|
|
const graph = new Map()
|
|
for (const row of stagedLive) {
|
|
graph.set(keyOf(row), {
|
|
libref: field(row, 'LIBREF'),
|
|
dsn: field(row, 'DSN'),
|
|
name: field(row, 'VAR_NM'),
|
|
deps: field(row, 'VAR_DEPS').split(/\s+/).filter(Boolean),
|
|
logic: strip(colVal(row, 'VAR_LOGIC')).toUpperCase()
|
|
})
|
|
}
|
|
const currentRows =
|
|
current && Array.isArray(current.rows)
|
|
? current.rows.filter(
|
|
(r) =>
|
|
isCurrent(r) &&
|
|
Number(colVal(r, 'ACTIVE')) === 1 &&
|
|
stagedTables.has(
|
|
[up(colVal(r, 'LIBREF')), up(colVal(r, 'DSN'))].join(
|
|
'.'
|
|
)
|
|
) &&
|
|
!stagedKeys.has(
|
|
[
|
|
up(colVal(r, 'LIBREF')),
|
|
up(colVal(r, 'DSN')),
|
|
up(colVal(r, 'VAR_NM'))
|
|
].join('.')
|
|
)
|
|
)
|
|
: []
|
|
for (const row of currentRows) {
|
|
graph.set(
|
|
[
|
|
up(colVal(row, 'LIBREF')),
|
|
up(colVal(row, 'DSN')),
|
|
up(colVal(row, 'VAR_NM'))
|
|
].join('.'),
|
|
{
|
|
libref: up(colVal(row, 'LIBREF')),
|
|
dsn: up(colVal(row, 'DSN')),
|
|
name: up(colVal(row, 'VAR_NM')),
|
|
deps: up(colVal(row, 'VAR_DEPS')).split(/\s+/).filter(Boolean),
|
|
logic: strip(colVal(row, 'VAR_LOGIC')).toUpperCase()
|
|
}
|
|
)
|
|
}
|
|
|
|
/** Names this load leaves inactive, which therefore cannot resolve a dep. */
|
|
const inactive = new Set()
|
|
const liveKeys = new Set(stagedLive.map(keyOf))
|
|
for (const row of rows) {
|
|
const key = keyOf(row)
|
|
if (!liveKeys.has(key)) inactive.add(key)
|
|
}
|
|
if (current && Array.isArray(current.rows)) {
|
|
for (const row of current.rows) {
|
|
const key = [
|
|
up(colVal(row, 'LIBREF')),
|
|
up(colVal(row, 'DSN')),
|
|
up(colVal(row, 'VAR_NM'))
|
|
].join('.')
|
|
if (
|
|
isCurrent(row) &&
|
|
Number(colVal(row, 'ACTIVE')) !== 1 &&
|
|
!liveKeys.has(key)
|
|
) {
|
|
inactive.add(key)
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const node of graph.values()) {
|
|
for (const dep of node.deps) {
|
|
const key = [node.libref, node.dsn, dep].join('.')
|
|
if (inactive.has(key) && !graph.has(key)) {
|
|
return reject(
|
|
`MPE_DERIVED: ${node.name}: VAR_DEPS lists ${dep}, which is not active after this load - the engine cannot resolve it`
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Self reference, undeclared and stale dependencies ─────────────────────
|
|
/** The identifiers an expression uses, with quoted runs removed. */
|
|
const identifiers = (logic) =>
|
|
logic
|
|
.replace(/'([^']|'')*'/g, ' ')
|
|
.replace(/"([^"]|"")*"/g, ' ')
|
|
.replace(/[^A-Za-z0-9_]/g, ' ')
|
|
.toUpperCase()
|
|
.split(/\s+/)
|
|
.filter(Boolean)
|
|
|
|
// scoped per table, as the server does: a name that is a derived variable
|
|
// on another table says nothing about this one
|
|
const graphKeys = new Set([...graph.values()].map((n) => nodeKey(n)))
|
|
|
|
// one check at a time, in the order the server runs them, so the message a
|
|
// row gets is the one it would get there
|
|
for (const node of graph.values()) {
|
|
if (node.deps.includes(node.name)) {
|
|
return reject(
|
|
`MPE_DERIVED: ${node.name}: VAR_DEPS may not list itself`
|
|
)
|
|
}
|
|
}
|
|
|
|
for (const node of graph.values()) {
|
|
for (const token of identifiers(node.logic)) {
|
|
if (
|
|
graphKeys.has([node.libref, node.dsn, token].join('.')) &&
|
|
token !== node.name &&
|
|
!node.deps.includes(token)
|
|
) {
|
|
return reject(
|
|
`MPE_DERIVED: ${node.name}: VAR_LOGIC references ${token}, which is not in VAR_DEPS`
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const node of graph.values()) {
|
|
const tokens = identifiers(node.logic)
|
|
for (const dep of node.deps) {
|
|
if (!tokens.includes(dep)) {
|
|
return reject(
|
|
`MPE_DERIVED: ${node.name}: VAR_DEPS lists ${dep}, which VAR_LOGIC does not reference`
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── A derived variable may not be named by a row level security rule ──────
|
|
const rls = loadControl('MPE_ROW_LEVEL_SECURITY')
|
|
if (rls && Array.isArray(rls.rows)) {
|
|
for (const node of graph.values()) {
|
|
const match = rls.rows.find(
|
|
(r) =>
|
|
up(colVal(r, 'RLS_LIBREF')) === node.libref &&
|
|
up(colVal(r, 'RLS_TABLE')) === node.dsn &&
|
|
up(colVal(r, 'RLS_VARIABLE_NM')) === node.name &&
|
|
Number(colVal(r, 'RLS_ACTIVE')) === 1 &&
|
|
isCurrent(r)
|
|
)
|
|
if (match) {
|
|
return reject(
|
|
`MPE_DERIVED: ${node.name} is named by a row level security rule on ${node.libref}.${node.dsn} - derived variables cannot be used there`
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Cycles ────────────────────────────────────────────────────────────────
|
|
// Repeatedly place the variables whose derived inputs are already placed; a
|
|
// round that places nothing leaves a cycle. The path is walked so the
|
|
// message names it, as the SAS hook does.
|
|
const nodeKeys = new Map(
|
|
[...graph.values()].map((n) => [n, [n.libref, n.dsn, n.name].join('.')])
|
|
)
|
|
const placed = new Set()
|
|
let remaining = [...graph.values()]
|
|
|
|
while (remaining.length > 0) {
|
|
const ready = remaining.filter((n) =>
|
|
n.deps.every((d) => {
|
|
const key = [n.libref, n.dsn, d].join('.')
|
|
return !graph.has(key) || placed.has(key)
|
|
})
|
|
)
|
|
if (ready.length === 0) {
|
|
// walk the unplaced edges so the message names the cycle
|
|
const start = remaining[0]
|
|
const path = [start.name]
|
|
const visited = new Set([nodeKeys.get(start)])
|
|
let cursor = start
|
|
for (let i = 0; i < remaining.length; i++) {
|
|
const parentKey = cursor.deps
|
|
.map((d) => [cursor.libref, cursor.dsn, d].join('.'))
|
|
.find((key) => graph.has(key) && !placed.has(key))
|
|
if (!parentKey) break
|
|
path.push(graph.get(parentKey).name)
|
|
if (visited.has(parentKey)) break
|
|
visited.add(parentKey)
|
|
cursor = graph.get(parentKey)
|
|
}
|
|
return reject(
|
|
`MPE_DERIVED: VAR_DEPS contains a cycle: ${path.join(' -> ')}`
|
|
)
|
|
}
|
|
for (const node of ready) placed.add(nodeKeys.get(node))
|
|
remaining = remaining.filter((n) => !placed.has(nodeKeys.get(n)))
|
|
}
|
|
})()
|