dc
c4021a42ff
`flags` was scoped to the entry loop and read by the central-directory loop, so mpZipPackage threw a ReferenceError on every call - the audit download was broken outright. The value now travels with each central entry.
1908 lines
75 KiB
JavaScript
1908 lines
75 KiB
JavaScript
/**
|
|
* @file dcMockUtils.js
|
|
* @brief Shared utilities for Data Controller JS mock services on SASjs Server.
|
|
*
|
|
* Provides:
|
|
* - fetchTable / fetchRaw / parseCsv : emulate SAS %webout(FETCH)
|
|
* - webOutOpen / webOutObj / webOutClose : emulate SAS %webout(OPEN/OBJ/CLOSE)
|
|
* - mpeinit : emulate the %mpeinit debug dump
|
|
* - formatSasValue : apply a SAS format to a numeric value
|
|
* - parseFormattedToSas : convert an ISO string back to a SAS numeric
|
|
* - getDdType : derive DATE/DATETIME/TIME/N/C from a column format
|
|
* - loadTableData / loadTableSchema : read a DC data JSON file from the Drive
|
|
* - sasVarsRow : build a $sasdata/$viewdata vars entry for a column
|
|
* - sasVarsRow : build a $sasdata/$viewdata vars entry for a column
|
|
*
|
|
* Usage (from a mock service that is eval'd by the SASjs Server):
|
|
* const _utilPath = nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js')
|
|
* eval(fs.readFileSync(_utilPath, 'utf8'))
|
|
* const rows = fetchTable('SASControlTable')
|
|
* const libds = rows[0]?.LIBDS
|
|
*/
|
|
|
|
// ─── SAS epoch ────────────────────────────────────────────────────────────────
|
|
|
|
const SAS_EPOCH_MS = Date.UTC(1960, 0, 1)
|
|
const MONTHS_ABBR = ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC']
|
|
const MONTHS_NUM = {JAN:'01',FEB:'02',MAR:'03',APR:'04',MAY:'05',JUN:'06',JUL:'07',AUG:'08',SEP:'09',OCT:'10',NOV:'11',DEC:'12'}
|
|
|
|
// ─── %webout(FETCH) emulation ────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Fetches the raw CSV text of a named input table.
|
|
* Uses eval() to access variables in the CALLER's scope.
|
|
*/
|
|
function fetchRaw(tableName) {
|
|
// 1. Direct variable (SASjs Server may set form fields as top-level JS vars)
|
|
try {
|
|
const val = eval(tableName)
|
|
if (val !== undefined && val !== null && val !== '') return val.toString()
|
|
} catch(e) { /* not defined */ }
|
|
|
|
// 2. _WEBIN_NAME / _WEBIN_FILEREF loop (SASjs Server format)
|
|
// Variable names are UPPERCASE: _WEBIN_NAME1, _WEBIN_FILEREF1, etc.
|
|
// _WEBIN_FILEREF may be a Buffer (from fs.readFileSync) or a string.
|
|
for (let i = 1; i <= 10; i++) {
|
|
try {
|
|
const nameVal = eval('_WEBIN_NAME' + i)
|
|
if (nameVal && nameVal.toLowerCase() === tableName.toLowerCase()) {
|
|
const refVal = eval('_WEBIN_FILEREF' + i)
|
|
if (refVal) return refVal.toString()
|
|
}
|
|
} catch(e) {}
|
|
// Also try lowercase variant
|
|
try {
|
|
const nameVal = eval('_webin_name' + i)
|
|
if (nameVal && nameVal.toLowerCase() === tableName.toLowerCase()) {
|
|
const refVal = eval('_webin_fileref' + i)
|
|
if (refVal) return refVal.toString()
|
|
}
|
|
} catch(e) {}
|
|
}
|
|
|
|
// 3. _WEBIN_FILENAME / _WEBIN_FILEREF fallback
|
|
for (let i = 1; i <= 10; i++) {
|
|
try {
|
|
const nameVal = eval('_WEBIN_FILENAME' + i)
|
|
if (nameVal && nameVal.toLowerCase().includes(tableName.toLowerCase())) {
|
|
const refVal = eval('_WEBIN_FILEREF' + i)
|
|
if (refVal) return refVal.toString()
|
|
}
|
|
} catch(e) {}
|
|
try {
|
|
const nameVal = eval('_webin_filename' + i)
|
|
if (nameVal && nameVal.toLowerCase().includes(tableName.toLowerCase())) {
|
|
const refVal = eval('_webin_fileref' + i)
|
|
if (refVal) return refVal.toString()
|
|
}
|
|
} catch(e) {}
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
/**
|
|
* Fetches a named input table as parsed row objects.
|
|
*/
|
|
function fetchTable(tableName) {
|
|
const raw = fetchRaw(tableName)
|
|
if (!raw) return []
|
|
return parseCsv(raw)
|
|
}
|
|
|
|
/**
|
|
* Parses just the header line of a SASjs adapter CSV string
|
|
* ("COL1:$char10. COL2:$char8." -> ['COL1', 'COL2']).
|
|
*/
|
|
function parseCsvHeader(csvText) {
|
|
if (!csvText) return []
|
|
const lines = csvText.split(/\r?\n/)
|
|
if (lines.length < 1) return []
|
|
return lines[0].trim().split(/\s+/).map((h) => h.split(':')[0])
|
|
}
|
|
|
|
/**
|
|
* Parses a SASjs adapter CSV string into an array of row objects.
|
|
* Header line: "COL1:$char10. COL2:$char8." (space-separated, name before colon)
|
|
* Data lines: comma-separated with double-quote escaping.
|
|
*/
|
|
function parseCsv(csvText) {
|
|
if (!csvText) return []
|
|
const lines = csvText.split(/\r?\n/)
|
|
if (lines.length < 2) return []
|
|
|
|
const headers = lines[0].trim().split(/\s+/).map(h => h.split(':')[0])
|
|
if (headers.length === 0) return []
|
|
|
|
const rows = []
|
|
for (let i = 1; i < lines.length; i++) {
|
|
const line = lines[i].trim()
|
|
if (!line) continue
|
|
const values = _parseCsvLine(line)
|
|
const row = {}
|
|
for (let j = 0; j < headers.length && j < values.length; j++) {
|
|
row[headers[j]] = values[j]
|
|
}
|
|
rows.push(row)
|
|
}
|
|
return rows
|
|
}
|
|
|
|
function _parseCsvLine(line) {
|
|
const values = []
|
|
let cur = ''
|
|
let inQuote = false
|
|
for (let j = 0; j < line.length; j++) {
|
|
const ch = line[j]
|
|
if (ch === '"') {
|
|
if (inQuote && line[j + 1] === '"') { cur += '"'; j++ }
|
|
else inQuote = !inQuote
|
|
} else if (ch === ',' && !inQuote) {
|
|
values.push(cur); cur = ''
|
|
} else { cur += ch }
|
|
}
|
|
values.push(cur)
|
|
return values
|
|
}
|
|
|
|
// ─── SAS format <-> ISO conversions ──────────────────────────────────────────
|
|
|
|
/**
|
|
* Column metadata, mirroring %mp_getcols.sas.
|
|
*
|
|
* The SAS macro runs proc contents and derives, per column: NAME (upcased),
|
|
* TYPE (C/N), LENGTH, LABEL (defaulting to the name), VARNUM, FMTNAME, FORMAT
|
|
* and DDTYPE - CHARACTER, NUMERIC, DATETIME, DATE or TIME. The date list is
|
|
* the macro's (YYMMDD, MMDDYY, DDMMYY, MONYY, E8601DA, B8601DA, E8601DT,
|
|
* NLDATM, NLDATE), so a column types the same way here as it does in the
|
|
* service that shares this code.
|
|
*
|
|
* @param {Array} schemaCols the table's columns as the mock data stores them
|
|
* ({name, type, length, format, label}), the equivalent of the proc contents
|
|
* output the macro reads
|
|
* @returns {Array} one entry per column, in the macro's shape and order
|
|
*/
|
|
function mpGetcols(schemaCols) {
|
|
return (schemaCols || []).map((col, index) => {
|
|
const name = String(col.name || '').toUpperCase()
|
|
const isChar = String(col.type || '').toUpperCase() === 'C'
|
|
const fmtname = String(col.format || '')
|
|
.replace(/[\d.]+$/, '')
|
|
.toUpperCase()
|
|
|
|
let ddtype
|
|
if (isChar) ddtype = 'CHARACTER'
|
|
else if (fmtname.startsWith('DATETIME') || fmtname.startsWith('E8601DT')
|
|
|| fmtname.startsWith('NLDATM')) ddtype = 'DATETIME'
|
|
else if (fmtname.startsWith('DATE') || fmtname.startsWith('DDMMYY')
|
|
|| fmtname.startsWith('MMDDYY') || fmtname.startsWith('YYMMDD')
|
|
|| fmtname.startsWith('E8601DA') || fmtname.startsWith('B8601DA')
|
|
|| fmtname.startsWith('MONYY') || fmtname.startsWith('NLDATE')) ddtype = 'DATE'
|
|
else if (fmtname.startsWith('TIME')) ddtype = 'TIME'
|
|
else ddtype = 'NUMERIC'
|
|
|
|
return {
|
|
NAME: name,
|
|
TYPE: isChar ? 'C' : 'N',
|
|
LENGTH: Number(col.length) || 0,
|
|
LABEL: col.label || col.name,
|
|
VARNUM: index + 1,
|
|
FORMAT: String(col.format || ''),
|
|
FMTNAME: fmtname,
|
|
DDTYPE: ddtype
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Derives a DDTYPE (DATE, DATETIME, TIME, N, C) from a SAS column format
|
|
* string, in the shape the client expects. Delegates to mpGetcols so the
|
|
* inference is the macro's, rather than a second format list that can drift
|
|
* away from the service's.
|
|
*/
|
|
function getDdType(col) {
|
|
const ddtype = mpGetcols([col])[0].DDTYPE
|
|
if (ddtype === 'CHARACTER') return 'C'
|
|
if (ddtype === 'NUMERIC') return 'N'
|
|
return ddtype
|
|
}
|
|
|
|
/**
|
|
* Applies a SAS format to a raw numeric value, producing the display string.
|
|
* DATE: 42 -> "12FEB1960"
|
|
* DATETIME: 42 -> "01JAN1960:00:00:42"
|
|
* TIME: 42 -> "0:00:42"
|
|
*/
|
|
function formatSasValue(value, ddtype) {
|
|
if (value === null || value === undefined || value === '') return String(value)
|
|
if (typeof value !== 'number') return String(value)
|
|
|
|
if (ddtype === 'DATE') {
|
|
const d = new Date(SAS_EPOCH_MS + value * 86400000)
|
|
return `${String(d.getUTCDate()).padStart(2,'0')}${MONTHS_ABBR[d.getUTCMonth()]}${d.getUTCFullYear()}`
|
|
}
|
|
if (ddtype === 'DATETIME') {
|
|
const d = new Date(SAS_EPOCH_MS + value * 1000)
|
|
return `${String(d.getUTCDate()).padStart(2,'0')}${MONTHS_ABBR[d.getUTCMonth()]}${d.getUTCFullYear()}:${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`
|
|
}
|
|
if (ddtype === 'TIME') {
|
|
const h = Math.floor(value / 3600)
|
|
const m = Math.floor((value % 3600) / 60)
|
|
const s = value % 60
|
|
return `${h}:${String(m).padStart(2,'0')}:${String(s).padStart(2,'0')}`
|
|
}
|
|
return String(value)
|
|
}
|
|
|
|
/**
|
|
* Converts a numeric SAS value to the ISO string Handsontable expects.
|
|
* DATE: 42 -> "1960-02-12"
|
|
* DATETIME: 42 -> "1960-01-01 00:00:42"
|
|
* TIME: 42 -> "00:00:42"
|
|
*/
|
|
function sasToIso(value, ddtype) {
|
|
if (value === null || value === undefined || value === '') return value
|
|
if (typeof value !== 'number') return value
|
|
|
|
if (ddtype === 'DATE') {
|
|
const d = new Date(SAS_EPOCH_MS + value * 86400000)
|
|
return `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')}`
|
|
}
|
|
if (ddtype === 'DATETIME') {
|
|
const d = new Date(SAS_EPOCH_MS + value * 1000)
|
|
return `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`
|
|
}
|
|
if (ddtype === 'TIME') {
|
|
const h = Math.floor(value / 3600)
|
|
const m = Math.floor((value % 3600) / 60)
|
|
const s = value % 60
|
|
return `${String(h).padStart(2,'0')}:${String(m).padStart(2,'0')}:${String(s).padStart(2,'0')}`
|
|
}
|
|
return value
|
|
}
|
|
|
|
/**
|
|
* Converts an ISO-formatted string (as sent by the editor) back to a SAS numeric.
|
|
* DATE: "1960-02-12" -> 42
|
|
* DATETIME: "1960-01-01 00:00:42" -> 42
|
|
* TIME: "00:00:42" -> 42
|
|
* Non-temporal values pass through unchanged.
|
|
*/
|
|
function parseFormattedToSas(value, ddtype) {
|
|
if (value === null || value === undefined || value === '') return value
|
|
if (ddtype !== 'DATE' && ddtype !== 'DATETIME' && ddtype !== 'TIME') return value
|
|
if (typeof value === 'number') return value
|
|
const str = String(value)
|
|
|
|
if (ddtype === 'DATE') {
|
|
const m = str.match(/^(\d{4})-(\d{2})-(\d{2})$/)
|
|
if (m) return Math.round((Date.UTC(+m[1], +m[2]-1, +m[3]) - SAS_EPOCH_MS) / 86400000)
|
|
return value
|
|
}
|
|
if (ddtype === 'DATETIME') {
|
|
const m = str.match(/^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/)
|
|
if (m) return Math.round((Date.UTC(+m[1], +m[2]-1, +m[3], +m[4], +m[5], +m[6]) - SAS_EPOCH_MS) / 1000)
|
|
return value
|
|
}
|
|
if (ddtype === 'TIME') {
|
|
const parts = str.split(':')
|
|
if (parts.length >= 3) return (+parts[0]) * 3600 + (+parts[1]) * 60 + (+parts[2])
|
|
return value
|
|
}
|
|
return value
|
|
}
|
|
|
|
/**
|
|
* Handles either numeric or string temporal values, converting to ISO for
|
|
* Handsontable. Numeric values use sasToIso; string values use the legacy
|
|
* regex parsing (for backwards compatibility with old data files).
|
|
*/
|
|
function formatCellValue(value, ddtype) {
|
|
if (value === null || value === undefined || value === '') return value
|
|
if (typeof value === 'number') return sasToIso(value, ddtype)
|
|
|
|
const str = String(value)
|
|
if (ddtype === 'DATE') {
|
|
const m = str.match(/^(\d{1,2})([A-Z]{3})(\d{4})$/)
|
|
if (m) return `${m[3]}-${MONTHS_NUM[m[2]]}-${m[1].padStart(2,'0')}`
|
|
return str
|
|
}
|
|
if (ddtype === 'DATETIME') {
|
|
const m = str.match(/^(\d{1,2})([A-Z]{3})(\d{4}):(\d{1,2}):(\d{2}):(\d{2})$/)
|
|
if (m) return `${m[3]}-${MONTHS_NUM[m[2]]}-${m[1].padStart(2,'0')} ${m[4].padStart(2,'0')}:${m[5]}:${m[6]}`
|
|
return str
|
|
}
|
|
if (ddtype === 'TIME') {
|
|
const parts = str.split(':')
|
|
if (parts.length >= 3) return `${parts[0].padStart(2,'0')}:${parts[1].padStart(2,'0')}:${parts[2].padStart(2,'0')}`
|
|
return str
|
|
}
|
|
return value
|
|
}
|
|
|
|
// ─── DC data file helpers ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Resolves the data directory for a given libref.
|
|
* Each libref has its own subfolder under <appLoc>/data/ (e.g. DC_JSLIB,
|
|
* TESTDATA). DC_JSLIB is the default control-library (mpelib) and is used
|
|
* for MPE_* config tables; other librefs hold user/demo data tables.
|
|
* Requires nodePath, driveRoot and appLoc in the caller's scope.
|
|
*/
|
|
function libDataDir(libref) {
|
|
return nodePath.resolve(driveRoot, 'files', appLoc, 'data', libref)
|
|
}
|
|
|
|
/**
|
|
* Lists all librefs that have a data folder on the Drive.
|
|
* Requires nodePath, driveRoot and appLoc in the caller's scope.
|
|
*/
|
|
function allLibrefs() {
|
|
const base = nodePath.resolve(driveRoot, 'files', appLoc, 'data')
|
|
try {
|
|
return fs.readdirSync(base)
|
|
.filter(f => {
|
|
const p = nodePath.resolve(base, f)
|
|
return fs.existsSync(p) && fs.statSync(p).isDirectory()
|
|
})
|
|
.sort()
|
|
} catch (err) { return [] }
|
|
}
|
|
|
|
/**
|
|
* Creates a loadTableData function bound to a specific data directory.
|
|
* const loadTableData = makeTableLoader(dataDir)
|
|
* const mpeXTest = loadTableData('MPE_X_TEST')
|
|
*/
|
|
function makeTableLoader(dataDir) {
|
|
return function loadTableData(tableName) {
|
|
try {
|
|
const file = nodePath.resolve(dataDir, tableName.toLowerCase() + '.json')
|
|
return JSON.parse(fs.readFileSync(file, {encoding:'utf8'}).toString())
|
|
} catch(err) { return null }
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Loads a table from any libref, searching every data folder until a JSON
|
|
* file named <table>.json is found. Returns { data, libref } or null.
|
|
* Requires nodePath, driveRoot and appLoc in the caller's scope.
|
|
*/
|
|
function loadTableAnyLib(tableName) {
|
|
for (const lib of allLibrefs()) {
|
|
const dir = libDataDir(lib)
|
|
const loader = makeTableLoader(dir)
|
|
const data = loader(tableName)
|
|
if (data) return { data, libref: lib }
|
|
}
|
|
return null
|
|
}
|
|
|
|
/**
|
|
* Loads just the column schema (array of column metadata) for a table.
|
|
*/
|
|
function loadTableSchema(dataDir, tableName) {
|
|
const data = makeTableLoader(dataDir)(tableName)
|
|
return (data && data.columns) ? data.columns : []
|
|
}
|
|
|
|
/**
|
|
* Resolves a hook program name to the source of its mock implementation.
|
|
*
|
|
* MPE_TABLES.pre_edit_hook / post_edit_hook (and the SOFTSELECT_HOOK /
|
|
* HARDSELECT_HOOK rule_value in MPE_VALIDATIONS) hold a hook program: either a
|
|
* physical .sas path, or a path relative to the appLoc (e.g.
|
|
* 'services/hooks/mytable_postedit'). The real backend %includes the program
|
|
* into the running service, so the hook shares the service's macro variables
|
|
* and work datasets and can modify them.
|
|
*
|
|
* The mock mirrors that: this returns the source of the hook's .js counterpart,
|
|
* which the calling service eval()s in its OWN scope so the hook can read and
|
|
* modify the service's variables. Returns null when the hook has no mock
|
|
* implementation, so callers can fall back to an inline emulation or ignore it.
|
|
*
|
|
* Requires nodePath, driveRoot and appLoc in the caller's scope.
|
|
*/
|
|
function mockHookSource(hookValue) {
|
|
if (!hookValue) return null
|
|
let hook = String(hookValue).trim()
|
|
if (!hook) return null
|
|
// Reduce to the services-relative form, dropping any drive path or appLoc
|
|
// prefix the config may carry.
|
|
const idx = hook.indexOf('services/')
|
|
if (idx > -1) hook = hook.slice(idx)
|
|
// A physical hook is a .sas program on the server; its mock is a .js file
|
|
// sitting in the same place on the Drive.
|
|
hook = hook.replace(/\.sas$/i, '')
|
|
const hookFile = nodePath.resolve(driveRoot, 'files', appLoc, hook + '.js')
|
|
// The hook value is hand-edited mock config, and a value carrying '..'
|
|
// segments would otherwise resolve outside the Drive and be eval()'d
|
|
// in-service. Keep it inside the Drive's files tree.
|
|
const filesRoot = nodePath.resolve(driveRoot, 'files')
|
|
if (!hookFile.startsWith(filesRoot + nodePath.sep)) return null
|
|
if (!fs.existsSync(hookFile)) return null
|
|
return fs.readFileSync(hookFile, 'utf8')
|
|
}
|
|
|
|
/**
|
|
* Synthesises NOTNULL dqrules from the table schema, mirroring the
|
|
* dictionary.columns union in getdata.sas:
|
|
* select upcase(name) as base_col, 'NOTNULL' as rule_type, '' as rule_value
|
|
* from dictionary.columns where ... and notnull='yes'
|
|
* Returns an array of { BASE_COL, RULE_TYPE, RULE_VALUE } entries for every
|
|
* schema column whose `notnull` flag is true.
|
|
*/
|
|
function notNullRules(schema) {
|
|
if (!schema) return []
|
|
const rules = []
|
|
for (const col of schema) {
|
|
if (col.notnull) {
|
|
rules.push({
|
|
BASE_COL: String(col.name).toUpperCase(),
|
|
RULE_TYPE: 'NOTNULL',
|
|
RULE_VALUE: ''
|
|
})
|
|
}
|
|
}
|
|
return rules
|
|
}
|
|
|
|
/**
|
|
* Builds a $sasdata/$viewdata vars entry for a column.
|
|
* Temporal columns are sent as char ($200.) by the real SAS backend.
|
|
*/
|
|
function sasVarsEntry(col) {
|
|
const ddtype = getDdType(col)
|
|
if (ddtype === 'DATE' || ddtype === 'DATETIME' || ddtype === 'TIME') {
|
|
return { format: '$200.', label: col.label || col.name, length: '200', type: 'char' }
|
|
}
|
|
const fmt = col.type === 'N' ? (col.format || 'best.') : ('$' + col.length + '.')
|
|
return { format: fmt, label: col.label || col.name, length: String(col.length), type: col.type === 'N' ? 'num' : 'char' }
|
|
}
|
|
|
|
/**
|
|
* Assigns the retained key (RK) to staged rows for a table registered with
|
|
* rk_underlying, mirroring %mp_retainedkey as %mpe_loader invokes it: any RK
|
|
* column in the input is dropped, and every row's RK is derived from its
|
|
* business key - the value that key already holds on the base table, or the
|
|
* next key value for a new one. Business keys are matched the way SAS compares
|
|
* them: case-sensitive, trailing blanks ignored.
|
|
*
|
|
* @param {Object} opts - { rows, tableReg, baseData }
|
|
* tableReg - the table's MPE_TABLES row (needs rk_underlying and buskey)
|
|
* baseData - { columns, rows } of the base table, for the key lookup
|
|
* @returns {String} the RK column name when the assignment ran, else ''
|
|
*/
|
|
function mpeAssignRetainedKey(opts) {
|
|
const { rows, tableReg, baseData } = opts
|
|
const rkUnderlying = tableReg && tableReg.rk_underlying
|
|
? String(tableReg.rk_underlying).trim()
|
|
: ''
|
|
const rkCol = tableReg && tableReg.buskey ? String(tableReg.buskey).trim() : ''
|
|
if (!rkUnderlying || !rkCol || !rows || rows.length === 0) return ''
|
|
|
|
const base = baseData || { columns: [], rows: [] }
|
|
const submitted = Object.keys(rows[0] || {}).find(
|
|
(k) => k.toUpperCase() === rkCol.toUpperCase()
|
|
)
|
|
const rkName = submitted ||
|
|
(base.columns.find((c) => c.name.toUpperCase() === rkCol.toUpperCase()) || {}).name ||
|
|
rkCol
|
|
const keys = rkUnderlying.split(/\s+/)
|
|
const keyValue = (value) => {
|
|
if (value === null || value === undefined) return ''
|
|
return typeof value === 'number' ? String(value) : String(value).replace(/\s+$/, '')
|
|
}
|
|
const businessKey = (row) => keys.map((k) => keyValue(colVal(row, k))).join('|')
|
|
|
|
const baseByKey = new Map()
|
|
let nextRk = 0
|
|
for (const row of base.rows) {
|
|
const rk = colVal(row, rkCol)
|
|
baseByKey.set(businessKey(row), rk)
|
|
const num = Number(rk)
|
|
if (isFinite(num) && num > nextRk) nextRk = num
|
|
}
|
|
|
|
for (const row of rows) {
|
|
Object.keys(row).forEach((k) => {
|
|
if (k.toUpperCase() === rkCol.toUpperCase()) delete row[k]
|
|
})
|
|
const existing = baseByKey.get(businessKey(row))
|
|
row[rkName] = existing === undefined || existing === null ? ++nextRk : existing
|
|
}
|
|
return rkName
|
|
}
|
|
|
|
/**
|
|
* Stages a submission: writes staged.json under staging/<tableId>/, copies the
|
|
* session audit items, and inserts the MPE_SUBMIT row. Shared by the
|
|
* editors/stagedata (grid submit) and editors/loadfile (CSV upload) services.
|
|
*
|
|
* @param {Object} opts - { dataDir, stagingDir, tableId, libds, libref, dsn,
|
|
* message, headers, rows, weboutPath }
|
|
* @returns {Object} the stageData object written to staged.json
|
|
*/
|
|
function stageSubmission(opts) {
|
|
const { dataDir, stagingDir, tableId, libds, libref, dsn, message, headers, rows } = opts
|
|
|
|
const stageFolder = nodePath.resolve(stagingDir, tableId)
|
|
fs.mkdirSync(stageFolder, { recursive: true })
|
|
|
|
const stageData = {
|
|
table_id: tableId,
|
|
libds,
|
|
libref,
|
|
dsn,
|
|
submitted_by_nm: getDcUser(),
|
|
submitted_on_dttm: Date.now() / 1000,
|
|
headers,
|
|
rows
|
|
}
|
|
fs.writeFileSync(
|
|
nodePath.resolve(stageFolder, 'staged.json'),
|
|
JSON.stringify(stageData, null, 2)
|
|
)
|
|
|
|
// keep audit items from this Stored Program run alongside the staged data
|
|
copySessionAuditItems(stageFolder, 'submit')
|
|
|
|
// Files the SAS loader leaves in the approval folder, so the audit pack
|
|
// holds the same artefacts: the cleaned CSV and the uploaded file itself.
|
|
for (const file of opts.auditFiles || []) {
|
|
fs.writeFileSync(nodePath.resolve(stageFolder, file.name), file.content)
|
|
}
|
|
|
|
// Insert a record into MPE_SUBMIT (always in the control library /
|
|
// mpelib, regardless of which library the base table lives in)
|
|
const mpeDataDir = libDataDir('DC_JSLIB')
|
|
const loadTableData = makeTableLoader(mpeDataDir)
|
|
const submitTable = loadTableData('MPE_SUBMIT') || {
|
|
name: 'MPE_SUBMIT',
|
|
columns: [],
|
|
rows: []
|
|
}
|
|
submitTable.rows.push({
|
|
TABLE_ID: tableId,
|
|
SUBMIT_STATUS_CD: 'SUBMITTED',
|
|
BASE_LIB: libref,
|
|
BASE_DS: dsn,
|
|
SUBMITTED_BY_NM: stageData.submitted_by_nm,
|
|
SUBMITTED_ON_DTTM: stageData.submitted_on_dttm,
|
|
SUBMITTED_REASON_TXT: message || '',
|
|
INPUT_OBS: rows.length,
|
|
INPUT_VARS: headers.filter((h) => h !== '_____DELETE__THIS__RECORD_____').length,
|
|
NUM_OF_APPROVALS_REQUIRED: 1,
|
|
NUM_OF_APPROVALS_REMAINING: 1,
|
|
REVIEWED_BY_NM: '',
|
|
REVIEWED_ON_DTTM: null
|
|
})
|
|
const submitFile = nodePath.resolve(mpeDataDir, 'mpe_submit.json')
|
|
fs.writeFileSync(submitFile, JSON.stringify(submitTable, null, 2))
|
|
|
|
return stageData
|
|
}
|
|
|
|
/**
|
|
* Copies audit items from the Stored Program's session directory (its
|
|
* current working directory at runtime) into the staging subfolder:
|
|
* - log.log -> <prefix>.log (runtime log of the submission)
|
|
* - code.js -> <prefix>_code.js (program source as executed)
|
|
* - reqHeaders.txt -> <prefix>_reqHeaders.txt
|
|
*/
|
|
function copySessionAuditItems(stageFolder, prefix) {
|
|
try {
|
|
const sessionDir = nodePath.dirname(weboutPath)
|
|
const copies = [
|
|
['log.log', prefix + '.log'],
|
|
['code.js', prefix + '_code.js'],
|
|
['reqHeaders.txt', prefix + '_reqHeaders.txt']
|
|
]
|
|
for (const [src, dest] of copies) {
|
|
const srcPath = nodePath.resolve(sessionDir, src)
|
|
if (fs.existsSync(srcPath)) {
|
|
fs.copyFileSync(srcPath, nodePath.resolve(stageFolder, dest))
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.log('Error copying session audit items: ' + err.message)
|
|
}
|
|
}
|
|
|
|
// ─── %mp_streamfile + ODS package (zip) ───────────────────────────────────────
|
|
|
|
const ZIP_CRC_TABLE = (() => {
|
|
const table = new Array(256)
|
|
for (let i = 0; i < 256; i++) {
|
|
let c = i
|
|
for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1)
|
|
table[i] = c >>> 0
|
|
}
|
|
return table
|
|
})()
|
|
|
|
function zipCrc32(bytes) {
|
|
let crc = -1
|
|
for (let i = 0; i < bytes.length; i++)
|
|
crc = (crc >>> 8) ^ ZIP_CRC_TABLE[(crc ^ bytes[i]) & 0xff]
|
|
return (crc ^ -1) >>> 0
|
|
}
|
|
|
|
function zipUtf8(value) {
|
|
const text = String(value)
|
|
if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(text)
|
|
const out = []
|
|
for (let i = 0; i < text.length; i++) {
|
|
const code = text.charCodeAt(i)
|
|
if (code < 0x80) out.push(code)
|
|
else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 0x3f))
|
|
else out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 0x3f), 0x80 | (code & 0x3f))
|
|
}
|
|
return Uint8Array.from(out)
|
|
}
|
|
|
|
/**
|
|
* Renders rows as CSV, the counterpart of %mp_ds2csv: the header row, then one
|
|
* line per row, with values quoted only when they carry a delimiter, quote or
|
|
* line break.
|
|
*
|
|
* @param {string[]} headers column names, in order
|
|
* @param {Array<Object>} rows objects keyed by column name
|
|
* @returns {string} CSV text with a trailing newline
|
|
*/
|
|
function toCsv(headers, rows) {
|
|
const cell = (value) => {
|
|
const text = value === undefined || value === null ? '' : String(value)
|
|
return /[",\n\r]/.test(text) ? '"' + text.replace(/"/g, '""') + '"' : text
|
|
}
|
|
const lines = [headers.map(cell).join(',')]
|
|
for (const row of rows) {
|
|
lines.push(headers.map((header) => cell(row[header])).join(','))
|
|
}
|
|
return lines.join('\n') + '\n'
|
|
}
|
|
|
|
/**
|
|
* Builds a zip archive in memory, entries stored rather than deflated - the JS
|
|
* counterpart of the ODS package getauditfile.sas publishes. The zip format
|
|
* leaves compression to the writer, so a stored archive opens anywhere.
|
|
*
|
|
* @param {Array<{name: string, content: string}>} entries archive entries
|
|
* @returns {Uint8Array} the archive
|
|
*/
|
|
function mpZipPackage(entries) {
|
|
const u16 = (n) => [n & 0xff, (n >>> 8) & 0xff]
|
|
const u32 = (n) => [n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, (n >>> 24) & 0xff]
|
|
|
|
const local = []
|
|
const central = []
|
|
let offset = 0
|
|
|
|
for (const entry of entries) {
|
|
const name = zipUtf8(entry.name)
|
|
const data = zipUtf8(entry.content)
|
|
const crc = zipCrc32(data)
|
|
// Bit 11 marks the name as UTF-8. Without it an extractor decodes a
|
|
// non-ASCII name as CP437, so the entry cannot be read back by name.
|
|
const flags = name.some((byte) => byte >= 0x80) ? 0x0800 : 0
|
|
|
|
// 1980-01-01 00:00, the earliest DOS timestamp: a stable value keeps
|
|
// the archive byte-identical for identical inputs.
|
|
const header = [].concat(
|
|
u32(0x04034b50), u16(20), u16(flags), u16(0), u16(0), u16(0x21),
|
|
u32(crc), u32(data.length), u32(data.length), u16(name.length), u16(0)
|
|
)
|
|
|
|
local.push(Uint8Array.from(header), name, data)
|
|
central.push({ name, crc, size: data.length, offset, flags })
|
|
offset += header.length + name.length + data.length
|
|
}
|
|
|
|
const directory = []
|
|
let directorySize = 0
|
|
for (const entry of central) {
|
|
const header = [].concat(
|
|
u32(0x02014b50), u16(20), u16(20), u16(entry.flags), u16(0), u16(0), u16(0x21),
|
|
u32(entry.crc), u32(entry.size), u32(entry.size),
|
|
u16(entry.name.length), u16(0), u16(0), u16(0), u16(0), u32(0),
|
|
u32(entry.offset)
|
|
)
|
|
directory.push(Uint8Array.from(header), entry.name)
|
|
directorySize += header.length + entry.name.length
|
|
}
|
|
|
|
const end = Uint8Array.from([].concat(
|
|
u32(0x06054b50), u16(0), u16(0),
|
|
u16(central.length), u16(central.length),
|
|
u32(directorySize), u32(offset), u16(0)
|
|
))
|
|
|
|
const chunks = local.concat(directory, [end])
|
|
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0)
|
|
const archive = new Uint8Array(total)
|
|
let at = 0
|
|
for (const chunk of chunks) {
|
|
archive.set(chunk, at)
|
|
at += chunk.length
|
|
}
|
|
return archive
|
|
}
|
|
|
|
/**
|
|
* Streams a file back as an attachment, the JS counterpart of %mp_streamfile on
|
|
* the SASJS platform: the headers name the content type and the body is the file
|
|
* itself. Returns false where that is not possible - the browser mock host has
|
|
* a text-only webout - so the caller can fall back to something textual.
|
|
*
|
|
* @param {{filename: string, contentType: string, bytes: Uint8Array}} opts
|
|
* @returns {boolean} whether the file was streamed
|
|
*/
|
|
function mpStreamfile(opts) {
|
|
if (typeof Buffer === 'undefined') return false
|
|
// A content-type header is what tells the SASjs Server to send webout.txt as
|
|
// binary rather than a string (api/src/controllers/internal/Execution.ts).
|
|
fs.writeFileSync(
|
|
_SASJS_WEBOUT_HEADERS,
|
|
`Content-type: ${opts.contentType} \n Content-Disposition: attachment;filename="${opts.filename}"`
|
|
)
|
|
fs.writeFileSync(weboutPath, Buffer.from(opts.bytes))
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* Generates a DC table_id in the same format as the real backend
|
|
* (DC<yyyymmdd-hhmmss.sss>_<rand6>_<pid4>).
|
|
*/
|
|
function makeTableId() {
|
|
const now = new Date()
|
|
const pad = (n, w) => String(n).padStart(w, '0')
|
|
const dtStr =
|
|
now.getFullYear() +
|
|
pad(now.getMonth() + 1, 2) +
|
|
pad(now.getDate(), 2) +
|
|
'-' +
|
|
pad(now.getHours(), 2) +
|
|
pad(now.getMinutes(), 2) +
|
|
pad(now.getSeconds(), 2) +
|
|
'.' +
|
|
pad(now.getMilliseconds(), 3)
|
|
const rand6 = pad(Math.floor(Math.random() * 1000000), 6)
|
|
const pid4 = pad(process.pid % 10000, 4)
|
|
return `DC${dtStr}_${rand6}_${pid4}`
|
|
}
|
|
|
|
// ─── mp_filtergenerate.sas ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Parses a SAS literal RAW_VALUE into a JS value or array.
|
|
* 'O''Brien' -> "O'Brien" (char, doubled quotes)
|
|
* 42 -> 42 (numeric)
|
|
* ('A','B') -> ["A","B"] (IN list)
|
|
* (1,2,3) -> [1,2,3] (IN list, numeric)
|
|
* 1 AND 5 -> [1, 5] (BETWEEN)
|
|
* '20DEC2053'd-> 34322 (date / datetime / time literal)
|
|
*/
|
|
function parseFilterRawValue(raw, operator) {
|
|
if (raw === undefined || raw === null || raw === '') return null
|
|
const s = String(raw).trim()
|
|
|
|
if (operator === 'BETWEEN') {
|
|
const parts = s.split(/\s+AND\s+/i)
|
|
return parts.map(p => parseFilterScalar(p))
|
|
}
|
|
|
|
if (operator === 'IN' || operator === 'NOT IN') {
|
|
const inner = s.replace(/^\(/, '').replace(/\)$/, '')
|
|
return inner.split(',').map(p => parseFilterScalar(p.trim()))
|
|
}
|
|
|
|
return parseFilterScalar(s)
|
|
}
|
|
|
|
/**
|
|
* SAS date / datetime / time literal -> the SAS internal value, or null.
|
|
* 34322 <- '20DEC2053'd
|
|
* 42 <- '01JAN1960:00:00:42'dt
|
|
* 42 <- '00:00:42't
|
|
* The client builds these in canonical ddMMMyyyy form (HelperService
|
|
* .sasTemporalLiteral), so only that shape needs parsing here.
|
|
*/
|
|
const TEMPORAL_LITERAL_RE = /^'(\d{2})([A-Z]{3})(\d{4})(?::(\d{2}):(\d{2}):(\d{2})(\.\d+)?)?'(dt|d|t)$/i
|
|
/* A time literal carries no date part, so it is matched separately */
|
|
const TIME_LITERAL_RE = /^'(\d{2}):(\d{2}):(\d{2})(\.\d+)?'t$/i
|
|
|
|
function parseTemporalLiteral(s) {
|
|
const t = TIME_LITERAL_RE.exec(s)
|
|
if (t) {
|
|
return Number(t[1]) * 3600 + Number(t[2]) * 60 + Number(t[3]) +
|
|
(t[4] ? Number(t[4]) : 0)
|
|
}
|
|
const m = TEMPORAL_LITERAL_RE.exec(s)
|
|
if (!m) return null
|
|
const month = MONTHS_ABBR.indexOf(m[2].toUpperCase())
|
|
if (month < 0) return null
|
|
const days = (Date.UTC(Number(m[3]), month, Number(m[1])) - SAS_EPOCH_MS) / 86400000
|
|
const suffix = m[8].toLowerCase()
|
|
if (suffix === 'd') return days
|
|
const secs = Number(m[4]) * 3600 + Number(m[5]) * 60 + Number(m[6]) +
|
|
(m[7] ? Number(m[7]) : 0)
|
|
return suffix === 'dt' ? days * 86400 + secs : secs
|
|
}
|
|
|
|
function parseFilterScalar(s) {
|
|
s = s.trim()
|
|
const temporal = parseTemporalLiteral(s)
|
|
if (temporal !== null) return temporal
|
|
if (s.startsWith("'") && s.endsWith("'")) {
|
|
return s.slice(1, -1).replace(/''/g, "'")
|
|
}
|
|
if (s.startsWith('"') && s.endsWith('"')) {
|
|
return s.slice(1, -1).replace(/""/g, '"')
|
|
}
|
|
const num = Number(s)
|
|
return Number.isFinite(num) && s !== '' ? num : s
|
|
}
|
|
|
|
/**
|
|
* Compares a row value against a SAS-literal RAW_VALUE for the given operator.
|
|
*/
|
|
function compareFilterValue(val, op, rawVal, isNumCol) {
|
|
if (val === undefined || val === null) return false
|
|
|
|
if (op === '=' || op === 'EQ') {
|
|
const target = parseFilterRawValue(rawVal, '=')
|
|
return isNumCol ? Number(val) === target : String(val) === String(target)
|
|
}
|
|
if (op === 'NE') {
|
|
const target = parseFilterRawValue(rawVal, 'NE')
|
|
return isNumCol ? Number(val) !== target : String(val) !== String(target)
|
|
}
|
|
if (op === '<') {
|
|
const target = parseFilterRawValue(rawVal, '<')
|
|
return isNumCol ? Number(val) < target : String(val) < String(target)
|
|
}
|
|
if (op === '>') {
|
|
const target = parseFilterRawValue(rawVal, '>')
|
|
return isNumCol ? Number(val) > target : String(val) > String(target)
|
|
}
|
|
if (op === '<=') {
|
|
const target = parseFilterRawValue(rawVal, '<=')
|
|
return isNumCol ? Number(val) <= target : String(val) <= String(target)
|
|
}
|
|
if (op === '>=') {
|
|
const target = parseFilterRawValue(rawVal, '>=')
|
|
return isNumCol ? Number(val) >= target : String(val) >= String(target)
|
|
}
|
|
if (op === 'BETWEEN') {
|
|
const parts = parseFilterRawValue(rawVal, 'BETWEEN')
|
|
if (Array.isArray(parts) && parts.length === 2) {
|
|
return isNumCol
|
|
? Number(val) >= parts[0] && Number(val) <= parts[1]
|
|
: String(val) >= String(parts[0]) && String(val) <= String(parts[1])
|
|
}
|
|
return true
|
|
}
|
|
if (op === 'IN') {
|
|
const list = parseFilterRawValue(rawVal, 'IN')
|
|
if (Array.isArray(list)) {
|
|
return isNumCol
|
|
? list.includes(Number(val))
|
|
: list.map(String).includes(String(val))
|
|
}
|
|
return false
|
|
}
|
|
if (op === 'NOT IN') {
|
|
const list = parseFilterRawValue(rawVal, 'NOT IN')
|
|
if (Array.isArray(list)) {
|
|
return isNumCol
|
|
? !list.includes(Number(val))
|
|
: !list.map(String).includes(String(val))
|
|
}
|
|
return true
|
|
}
|
|
if (op === 'CONTAINS') {
|
|
const target = parseFilterRawValue(rawVal, 'CONTAINS')
|
|
return String(val).toLowerCase().includes(String(target).toLowerCase())
|
|
}
|
|
// Unknown operator: don't filter
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* Renders the WHERE clause text that mp_filtergenerate.sas writes to the
|
|
* fileref, joined the way getdata.sas/viewdata.sas collapse the lines into
|
|
* FILTER_TEXT (catx(' ')). Single clause: (( A = 1 )). Empty query: 1=1.
|
|
*/
|
|
function mpFilterGenerateText(fqRows) {
|
|
const sorted = [...fqRows].sort((a, b) =>
|
|
Number(colVal(a, 'SUBGROUP_ID')) - Number(colVal(b, 'SUBGROUP_ID')))
|
|
const lines = []
|
|
let prevSg = null
|
|
for (let i = 0; i < sorted.length; i++) {
|
|
const r = sorted[i]
|
|
const sg = colVal(r, 'SUBGROUP_ID')
|
|
if (i === 0) lines.push('((')
|
|
else if (sg !== prevSg) lines.push((colVal(r, 'GROUP_LOGIC') || 'AND') + ' (')
|
|
else lines.push(colVal(r, 'SUBGROUP_LOGIC') || 'AND')
|
|
lines.push(
|
|
colVal(r, 'VARIABLE_NM') + ' ' +
|
|
colVal(r, 'OPERATOR_NM') + ' ' +
|
|
colVal(r, 'RAW_VALUE'))
|
|
const nextSg = i + 1 < sorted.length ? colVal(sorted[i + 1], 'SUBGROUP_ID') : null
|
|
if (nextSg !== sg) lines.push(i === sorted.length - 1 ? '))' : ')')
|
|
prevSg = sg
|
|
}
|
|
return lines.map(l => String(l).trim()).filter(l => l !== '').join(' ')
|
|
}
|
|
|
|
/**
|
|
* Builds a JS filter predicate from FILTERQUERY rows, mirroring the SAS WHERE
|
|
* clause that mp_filtergenerate.sas would produce.
|
|
*
|
|
* Groups rows by SUBGROUP_ID; each subgroup is AND-ed within (SUBGROUP_LOGIC),
|
|
* then the subgroups are combined by GROUP_LOGIC (AND/OR).
|
|
*
|
|
* @param {Array<Object>} fqRows - Filter query rows (GROUP_LOGIC, SUBGROUP_LOGIC,
|
|
* SUBGROUP_ID, VARIABLE_NM, OPERATOR_NM, RAW_VALUE)
|
|
* @param {Array<Object>} columns - Column metadata [{ name, type, format, ... }]
|
|
* used to determine numeric vs char comparison
|
|
* @returns {function(Object): boolean} predicate function
|
|
*/
|
|
function mpFilterGenerate(fqRows, columns) {
|
|
if (!fqRows || fqRows.length === 0) return () => true
|
|
|
|
// Build a column lookup: UPPER(name) -> { type, ddtype }
|
|
const colMap = {}
|
|
if (columns) {
|
|
for (const c of columns) {
|
|
const ddtype = getDdType(c)
|
|
colMap[c.name.toUpperCase()] = { type: c.type === 'N' ? 'num' : 'char', ddtype }
|
|
}
|
|
}
|
|
|
|
// Group rows by SUBGROUP_ID (matching: proc sort; by SUBGROUP_ID)
|
|
const subgroups = {}
|
|
for (const r of fqRows) {
|
|
const sgId = colVal(r, 'SUBGROUP_ID') || 0
|
|
if (!subgroups[sgId]) subgroups[sgId] = []
|
|
subgroups[sgId].push(r)
|
|
}
|
|
|
|
const subgroupIds = Object.keys(subgroups).sort((a, b) => Number(a) - Number(b))
|
|
|
|
return function matches(row) {
|
|
let result = true
|
|
let firstGroup = true
|
|
|
|
for (const sgId of subgroupIds) {
|
|
const rows = subgroups[sgId]
|
|
let sgResult = true
|
|
|
|
for (const r of rows) {
|
|
const varNm = (colVal(r, 'VARIABLE_NM') || '').toString()
|
|
const val = colVal(row, varNm)
|
|
const op = (colVal(r, 'OPERATOR_NM') || '').toString().toUpperCase()
|
|
const rawVal = colVal(r, 'RAW_VALUE')
|
|
const colInfo = colMap[varNm.toUpperCase()]
|
|
const isNumCol = colInfo && colInfo.type === 'num'
|
|
|
|
if (!compareFilterValue(val, op, rawVal, isNumCol)) sgResult = false
|
|
}
|
|
|
|
const groupLogic = (colVal(rows[0], 'GROUP_LOGIC') || 'AND').toString().toUpperCase()
|
|
if (firstGroup) {
|
|
result = sgResult
|
|
firstGroup = false
|
|
} else if (groupLogic === 'OR') {
|
|
result = result || sgResult
|
|
} else {
|
|
result = result && sgResult
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|
|
}
|
|
|
|
// ─── mpe_filtermaster.sas ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Retrieves and applies a stored filter, mirroring mpe_filtermaster.sas.
|
|
*
|
|
* If filter_rk > 0, looks up the FILTER_HASH from mpe_filteranytable, then
|
|
* fetches the filter rows from mpe_filtersource (sorted by filter_line),
|
|
* dropping filter_hash/filter_line/processed_dttm to produce the query dataset.
|
|
* Then calls mpFilterGenerate to build the predicate.
|
|
*
|
|
* Also applies the current-record filter for EDIT/DLOAD mode (tx_from/tx_to)
|
|
* when the table is TXTEMPORAL/BITEMPORAL — matching the SAS macro.
|
|
*
|
|
* @param {Object} opts
|
|
* @param {string} opts.mode - EDIT, VIEW, DLOAD, ULOAD
|
|
* @param {string} opts.libds - Target table (libref.dataset), uppercased
|
|
* @param {number} opts.filterRk - The filter_rk (-1 if none)
|
|
* @param {string} opts.dataDir - Data directory for the DC control library
|
|
* @param {Array<Object>} [opts.columns] - Column metadata for the target table
|
|
* (needed by mpFilterGenerate for numeric/char comparison)
|
|
* @param {Object} [opts.tableReg] - MPE_TABLES row for the target table
|
|
* (needed for temporal current-record filter)
|
|
* @returns {{ query: Array<Object>, predicate: function(Object): boolean }}
|
|
*/
|
|
function mpeFilterMaster(opts) {
|
|
const { mode, libds, filterRk, dataDir, columns, tableReg } = opts
|
|
|
|
const loadTableData = makeTableLoader(dataDir)
|
|
let query = []
|
|
|
|
// ── Deal with FILTER_RK first ──────────────────────────────────────
|
|
// data _null_; set &dclib..mpe_filteranytable(where=(filter_rk=&filter_rk))
|
|
// call symputx('filter_hash',filter_hash,'l')
|
|
if (filterRk && filterRk > 0) {
|
|
const summaryTable = loadTableData('MPE_FILTERANYTABLE')
|
|
if (summaryTable && summaryTable.rows) {
|
|
const summaryRow = summaryTable.rows.find(r =>
|
|
Number(colVal(r, 'FILTER_RK')) === Number(filterRk)
|
|
)
|
|
if (summaryRow) {
|
|
const filterHash = colVal(summaryRow, 'FILTER_HASH')
|
|
|
|
// proc sort data=&dclib..mpe_filtersource(where=(filter_hash=...))
|
|
// out=&outds(drop=filter_hash filter_line processed_dttm); by filter_line
|
|
const detailTable = loadTableData('MPE_FILTERSOURCE')
|
|
if (detailTable && detailTable.rows) {
|
|
query = detailTable.rows
|
|
.filter(r => colVal(r, 'FILTER_HASH') === filterHash)
|
|
.sort((a, b) => Number(colVal(a, 'FILTER_LINE')) - Number(colVal(b, 'FILTER_LINE')))
|
|
.map(r => ({
|
|
GROUP_LOGIC: colVal(r, 'GROUP_LOGIC'),
|
|
SUBGROUP_LOGIC: colVal(r, 'SUBGROUP_LOGIC'),
|
|
SUBGROUP_ID: Number(colVal(r, 'SUBGROUP_ID')),
|
|
VARIABLE_NM: colVal(r, 'VARIABLE_NM'),
|
|
OPERATOR_NM: colVal(r, 'OPERATOR_NM'),
|
|
RAW_VALUE: colVal(r, 'RAW_VALUE')
|
|
}))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Build the predicate from the query (mp_filtergenerate) ────────
|
|
let predicate = mpFilterGenerate(query, columns)
|
|
let filterText = mpFilterGenerateText(query)
|
|
// data _null_ (getdata.sas/viewdata.sas): if cats(filter_text)='1=1'
|
|
// then filter_text='' — an empty filter hides the client info bar
|
|
if (filterText === '1=1') filterText = ''
|
|
|
|
// ── Current-record filter for EDIT/DLOAD mode ──────────────────────
|
|
// Mirrors: if &varfrom ne 0 and (&mode=EDIT or &mode=DLOAD)
|
|
// ("&datetime"dt < varto)
|
|
if (tableReg && (mode === 'EDIT' || mode === 'DLOAD')) {
|
|
const varFrom = tableReg.var_txfrom
|
|
const varTo = tableReg.var_txto
|
|
if (varFrom && varTo) {
|
|
// Check if the validity vars are mentioned in the query
|
|
const hasValidityVar = query.some(r => {
|
|
const vn = (colVal(r, 'VARIABLE_NM') || '').toUpperCase()
|
|
return vn === varFrom.toUpperCase() || vn === varTo.toUpperCase()
|
|
})
|
|
if (!hasValidityVar) {
|
|
const now = Date.now() / 1000
|
|
const inner = predicate
|
|
predicate = (row) => {
|
|
if (!inner(row)) return false
|
|
// (&dc_dttmtfmt. lt varto) — current record has tx_to > now
|
|
const txTo = colVal(row, varTo)
|
|
return txTo === undefined || txTo === null || Number(txTo) > now
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return { query, predicate, filterText }
|
|
}
|
|
|
|
// ─── Column value lookup / table writer ──────────────────────────────────────
|
|
|
|
/**
|
|
* Gets a value from a row object by column name, case-insensitive.
|
|
* Centralises the lookup pattern used throughout the mock services.
|
|
*/
|
|
function colVal(row, name) {
|
|
if (!row || !name) return undefined
|
|
if (row[name] !== undefined) return row[name]
|
|
const lc = name.toLowerCase()
|
|
if (row[lc] !== undefined) return row[lc]
|
|
const key = Object.keys(row).find(k => k.toLowerCase() === lc)
|
|
return key ? row[key] : undefined
|
|
}
|
|
|
|
/**
|
|
* Saves a table's JSON data to the data directory.
|
|
* Complement to makeTableLoader().
|
|
*/
|
|
function saveTableData(dataDir, tableName, tableData) {
|
|
const file = nodePath.resolve(dataDir, tableName.toLowerCase() + '.json')
|
|
fs.writeFileSync(file, JSON.stringify(tableData, null, 2))
|
|
}
|
|
|
|
// ─── mp_hashdataset.sas ───────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Computes an MD5 hash of a query dataset, mirroring mp_hashdataset.sas.
|
|
*
|
|
* The hash is chained row-to-row: each row's hash includes the previous row's
|
|
* hash (prevkey), and the salt seeds the initial prevkey. Char and num
|
|
* variables are hashed separately — matching mf_getvarlist(typefilter=C/N) —
|
|
* and prevkey is appended to the char var list (cvars = charVars & prevkeyvar).
|
|
*
|
|
* Each value is inner-md5'd (put(md5(trim(var)),$hex32.)), the results are
|
|
* concatenated (!! operator), and the outer md5 produces the row hash.
|
|
*
|
|
* @param {Array<Object>} queryds - Array of row objects (e.g. filterquery rows)
|
|
* @param {string} salt - The salt value (typically the libds, uppercased)
|
|
* @param {string[]} charVars - Char variable names in column order
|
|
* @param {string[]} numVars - Numeric variable names in column order
|
|
* @returns {string} 32-char uppercase hex hash ($hex32. representation)
|
|
*/
|
|
function mpHashDataset(queryds, salt, charVars, numVars) {
|
|
const crypto = require('crypto')
|
|
const md5hex = (s) => crypto.createHash('md5').update(String(s)).digest('hex').toUpperCase()
|
|
|
|
// Empty dataset: hash is just md5(salt), matching the NLOBS=0 branch
|
|
if (!queryds || queryds.length === 0) {
|
|
return md5hex(salt)
|
|
}
|
|
|
|
// Chain seed: prevkey = md5(salt)
|
|
let prevkey = md5hex(salt)
|
|
|
|
for (const row of queryds) {
|
|
const parts = []
|
|
|
|
// Char vars (matching cvars=%mf_getvarlist(typefilter=C) &prevkeyvar)
|
|
for (const v of charVars) {
|
|
parts.push(md5hex(String(colVal(row, v) ?? '').trim()))
|
|
}
|
|
// prevkey is included as a char var
|
|
parts.push(md5hex(prevkey.trim()))
|
|
|
|
// Num vars (matching nvars=%mf_getvarlist(typefilter=N))
|
|
// SAS: put(md5(trim(put(ifn(missing(var),var,var*1),binary64.))),$hex32.)
|
|
for (const v of numVars) {
|
|
const val = colVal(row, v)
|
|
const num = Number(val)
|
|
parts.push(md5hex(String(Number.isFinite(num) ? num : '').trim()))
|
|
}
|
|
|
|
// Outer md5 of the concatenated inner hashes (!! operator)
|
|
prevkey = md5hex(parts.join(''))
|
|
}
|
|
|
|
return prevkey
|
|
}
|
|
|
|
// ─── mp_retainedkey.sas ───────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Allocates a retained key value for a new record, mirroring
|
|
* mp_retainedkey.sas.
|
|
*
|
|
* Gets max(retained_key) from the base table (or 0 if empty), increments by 1
|
|
* for the new record, and updates the maxkeytable with the new max.
|
|
*
|
|
* @param {Object} baseTable - The base (target) table JSON { name, columns, rows }
|
|
* @param {string} retainedKey - Name of the RK column (e.g. 'FILTER_RK')
|
|
* @param {Object} maxkeyTable - The maxkeyvalues table JSON (modified in place)
|
|
* @param {string} keyTableLibds - Uppercased libref.dsn of the base table
|
|
* @returns {number} The new retained key value
|
|
*/
|
|
function mpRetainedKey(baseTable, retainedKey, maxkeyTable, keyTableLibds) {
|
|
// select sum(max(&retained_key),0) into: maxkey from &base_libds
|
|
const rows = (baseTable && baseTable.rows) || []
|
|
let maxKey = 0
|
|
for (const row of rows) {
|
|
const val = Number(colVal(row, retainedKey))
|
|
if (Number.isFinite(val) && val > maxKey) maxKey = val
|
|
}
|
|
const newKey = maxKey + 1
|
|
|
|
// Update maxkeytable (matching the maxkeytable update in mp_retainedkey.sas)
|
|
if (maxkeyTable) {
|
|
const existing = maxkeyTable.rows.find(r =>
|
|
colVal(r, 'KEYTABLE') &&
|
|
colVal(r, 'KEYTABLE').toString().toUpperCase() === keyTableLibds.toUpperCase()
|
|
)
|
|
const now = Date.now() / 1000
|
|
if (existing) {
|
|
existing.MAX_KEY = newKey
|
|
existing.PROCESSED_DTTM = now
|
|
} else {
|
|
maxkeyTable.rows.push({
|
|
KEYTABLE: keyTableLibds,
|
|
KEYCOLUMN: retainedKey,
|
|
MAX_KEY: newKey,
|
|
PROCESSED_DTTM: now
|
|
})
|
|
}
|
|
}
|
|
|
|
return newKey
|
|
}
|
|
|
|
// ─── mp_filterstore.sas ───────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Validates and stores a filter query, mirroring mp_filterstore.sas.
|
|
*
|
|
* Hashes the query (mp_hashdataset with salt=libds), checks for an existing
|
|
* entry in the summary table, and if not found:
|
|
* - allocates a FILTER_RK via mp_retainedkey
|
|
* - inserts into filter_summary (mpe_filteranytable)
|
|
* - inserts into filter_detail (mpe_filtersource)
|
|
* - updates the maxkeytable (mpe_maxkeyvalues)
|
|
*
|
|
* @param {Object} opts
|
|
* @param {string} opts.libds - The target table being filtered (libref.dataset)
|
|
* @param {Array<Object>} opts.queryds - The filterquery rows
|
|
* @param {string} opts.dataDir - Data directory for the control library
|
|
* @returns {Object} { FILTER_RK, FILTER_HASH, FILTER_TABLE } (PROCESSED_DTTM
|
|
* dropped, matching: alter table work.result drop PROCESSED_DTTM)
|
|
*/
|
|
function mpFilterStore(opts) {
|
|
const { libds, queryds, dataDir } = opts
|
|
|
|
// Filterquery column types (matching the SAS dataset definition).
|
|
// mf_getvarlist(typefilter=C) returns these in column order.
|
|
const CHAR_VARS = ['GROUP_LOGIC', 'SUBGROUP_LOGIC', 'VARIABLE_NM', 'OPERATOR_NM', 'RAW_VALUE']
|
|
const NUM_VARS = ['SUBGROUP_ID']
|
|
|
|
// ── Hash the query (salt=uppercased libds, matching salt=&orig_libds) ──
|
|
const filterHash = mpHashDataset(queryds, libds.toUpperCase(), CHAR_VARS, NUM_VARS)
|
|
|
|
// ── Load summary table (mpe_filteranytable) ──────────────────────────
|
|
const loadTableData = makeTableLoader(dataDir)
|
|
const summaryTable = loadTableData('MPE_FILTERANYTABLE') || {
|
|
name: 'MPE_FILTERANYTABLE',
|
|
columns: [
|
|
{ name: 'FILTER_RK', type: 'N', length: 8, format: '', label: '', notnull: true },
|
|
{ name: 'FILTER_HASH', type: 'C', length: 32, format: '', label: '', notnull: true },
|
|
{ name: 'FILTER_TABLE', type: 'C', length: 41, format: '', label: '', notnull: true },
|
|
{ name: 'PROCESSED_DTTM', type: 'N', length: 8, format: 'datetime19.', label: '', notnull: true }
|
|
],
|
|
rows: []
|
|
}
|
|
|
|
// ── Check if hash already exists (set &filter_summary; where filter_hash=) ──
|
|
let existing = summaryTable.rows.find(r =>
|
|
colVal(r, 'FILTER_HASH') === filterHash
|
|
)
|
|
|
|
if (!existing) {
|
|
// ── Allocate FILTER_RK via mp_retainedkey ────────────────────────
|
|
const maxkeyTable = loadTableData('MPE_MAXKEYVALUES') || {
|
|
name: 'MPE_MAXKEYVALUES',
|
|
columns: [
|
|
{ name: 'KEYTABLE', type: 'C', length: 41, format: '', label: '', notnull: false },
|
|
{ name: 'KEYCOLUMN', type: 'C', length: 32, format: '$32.', label: '', notnull: false },
|
|
{ name: 'MAX_KEY', type: 'N', length: 8, format: '', label: '', notnull: false },
|
|
{ name: 'PROCESSED_DTTM', type: 'N', length: 8, format: 'E8601DT26.6', label: '', notnull: false }
|
|
],
|
|
rows: []
|
|
}
|
|
|
|
const filterRk = mpRetainedKey(
|
|
summaryTable, 'FILTER_RK', maxkeyTable,
|
|
dataDir.split('/').pop() === 'DC_JSLIB'
|
|
? 'DC_JSLIB.MPE_FILTERANYTABLE'
|
|
: libds.split('.')[0] + '.MPE_FILTERANYTABLE'
|
|
)
|
|
|
|
// ── Insert into summary table (proc append base=&filter_summary) ──
|
|
const now = Date.now() / 1000
|
|
const newSummaryRow = {
|
|
FILTER_RK: filterRk,
|
|
FILTER_HASH: filterHash,
|
|
FILTER_TABLE: libds.toUpperCase(),
|
|
PROCESSED_DTTM: now
|
|
}
|
|
summaryTable.rows.push(newSummaryRow)
|
|
saveTableData(dataDir, 'MPE_FILTERANYTABLE', summaryTable)
|
|
saveTableData(dataDir, 'MPE_MAXKEYVALUES', maxkeyTable)
|
|
|
|
// ── Insert into detail table (proc append base=&filter_detail) ──
|
|
const detailTable = loadTableData('MPE_FILTERSOURCE') || {
|
|
name: 'MPE_FILTERSOURCE',
|
|
columns: [
|
|
{ name: 'FILTER_HASH', type: 'C', length: 32, format: '', label: '', notnull: true },
|
|
{ name: 'FILTER_LINE', type: 'N', length: 8, format: '', label: '', notnull: true },
|
|
{ name: 'GROUP_LOGIC', type: 'C', length: 3, format: '', label: '', notnull: true },
|
|
{ name: 'SUBGROUP_LOGIC', type: 'C', length: 3, format: '', label: '', notnull: true },
|
|
{ name: 'SUBGROUP_ID', type: 'N', length: 8, format: '', label: '', notnull: true },
|
|
{ name: 'VARIABLE_NM', type: 'C', length: 32, format: '', label: '', notnull: true },
|
|
{ name: 'OPERATOR_NM', type: 'C', length: 12, format: '', label: '', notnull: true },
|
|
{ name: 'RAW_VALUE', type: 'C', length: 4000, format: '', label: '', notnull: true },
|
|
{ name: 'PROCESSED_DTTM', type: 'N', length: 8, format: 'datetime19.', label: '', notnull: true }
|
|
],
|
|
rows: []
|
|
}
|
|
|
|
let lineNum = 0
|
|
for (const row of queryds) {
|
|
lineNum++
|
|
detailTable.rows.push({
|
|
FILTER_HASH: filterHash,
|
|
FILTER_LINE: lineNum,
|
|
GROUP_LOGIC: colVal(row, 'GROUP_LOGIC'),
|
|
SUBGROUP_LOGIC: colVal(row, 'SUBGROUP_LOGIC'),
|
|
SUBGROUP_ID: Number(colVal(row, 'SUBGROUP_ID')),
|
|
VARIABLE_NM: colVal(row, 'VARIABLE_NM'),
|
|
OPERATOR_NM: colVal(row, 'OPERATOR_NM'),
|
|
RAW_VALUE: colVal(row, 'RAW_VALUE'),
|
|
PROCESSED_DTTM: now
|
|
})
|
|
}
|
|
saveTableData(dataDir, 'MPE_FILTERSOURCE', detailTable)
|
|
|
|
existing = newSummaryRow
|
|
}
|
|
|
|
// Return result (PROCESSED_DTTM dropped)
|
|
return {
|
|
FILTER_RK: colVal(existing, 'FILTER_RK'),
|
|
FILTER_HASH: colVal(existing, 'FILTER_HASH'),
|
|
FILTER_TABLE: colVal(existing, 'FILTER_TABLE')
|
|
}
|
|
}
|
|
|
|
// ─── mpe_derived.sas ────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Compiles a SAS expression into a JS function - for the mock only.
|
|
*
|
|
* The real evaluation is done by SAS. This covers the subset used by the mock
|
|
* and demo configs, so the EDIT screen can be exercised (display, filtering,
|
|
* submission) without a SAS server:
|
|
*
|
|
* - column names, numbers, quoted strings
|
|
* - + - * / and comparisons (= ne < > <= >=)
|
|
* - AND OR, and `X IN (a, b, c)`
|
|
* - ifc(), cats(), catx(), upcase(), lowcase(), strip(), trim(), length(),
|
|
* substr(), index()
|
|
*
|
|
* Anything outside that subset throws when a row is derived, and the column is
|
|
* left missing for that row - the same way a failing SAS expression would.
|
|
*/
|
|
function compileSasExpression(expr, columnNames) {
|
|
// Tokenise first, so operators are only rewritten outside quoted strings
|
|
// (a literal such as 'OR' must never be read as an operator).
|
|
const tokens = []
|
|
const s = String(expr)
|
|
let i = 0
|
|
while (i < s.length) {
|
|
const c = s[i]
|
|
if (/\s/.test(c)) { i++; continue }
|
|
if (c === "'" || c === '"') {
|
|
const q = c
|
|
let j = i + 1
|
|
let val = ''
|
|
while (j < s.length) {
|
|
if (s[j] === q) {
|
|
if (s[j + 1] === q) { val += q; j += 2; continue }
|
|
break
|
|
}
|
|
val += s[j]
|
|
j++
|
|
}
|
|
tokens.push({ type: 'str', value: val })
|
|
i = j + 1
|
|
continue
|
|
}
|
|
if (/[0-9]/.test(c)) {
|
|
let j = i
|
|
while (j < s.length && /[0-9.]/.test(s[j])) j++
|
|
tokens.push({ type: 'num', value: s.slice(i, j) })
|
|
i = j
|
|
continue
|
|
}
|
|
if (/[A-Za-z_]/.test(c)) {
|
|
let j = i
|
|
while (j < s.length && /[A-Za-z0-9_]/.test(s[j])) j++
|
|
tokens.push({ type: 'word', value: s.slice(i, j) })
|
|
i = j
|
|
continue
|
|
}
|
|
const two = s.slice(i, i + 2)
|
|
if (['<=', '>=', '!=', '<>'].includes(two)) {
|
|
tokens.push({ type: 'punct', value: two })
|
|
i += 2
|
|
continue
|
|
}
|
|
// only the operators in the documented subset may reach the generated
|
|
// function - anything else is rejected rather than executed
|
|
if (!'+-*/(),<>='.includes(c)) {
|
|
throw new Error('Unsupported character in derived expression: ' + c)
|
|
}
|
|
tokens.push({ type: 'punct', value: c })
|
|
i++
|
|
}
|
|
|
|
// X IN (a, b, c) -> __in([a, b, c], X)
|
|
const out = []
|
|
for (let t = 0; t < tokens.length; t++) {
|
|
const tok = tokens[t]
|
|
if (
|
|
tok.type === 'word' &&
|
|
tok.value.toLowerCase() === 'in' &&
|
|
out.length > 0 &&
|
|
tokens[t + 1] &&
|
|
tokens[t + 1].value === '('
|
|
) {
|
|
let depth = 0
|
|
let j = t + 1
|
|
for (; j < tokens.length; j++) {
|
|
if (tokens[j].value === '(') depth++
|
|
else if (tokens[j].value === ')') {
|
|
depth--
|
|
if (depth === 0) break
|
|
}
|
|
}
|
|
if (j < tokens.length) {
|
|
const left = out.pop()
|
|
const inner = tokens.slice(t + 2, j)
|
|
out.push(
|
|
{ type: 'word', value: '__in' },
|
|
{ type: 'punct', value: '(' },
|
|
{ type: 'punct', value: '[' },
|
|
...inner,
|
|
{ type: 'punct', value: ']' },
|
|
{ type: 'punct', value: ',' },
|
|
left,
|
|
{ type: 'punct', value: ')' }
|
|
)
|
|
t = j
|
|
continue
|
|
}
|
|
}
|
|
out.push(tok)
|
|
}
|
|
|
|
const OPS = {
|
|
and: '&&', or: '||',
|
|
eq: '==', ne: '!=', lt: '<', gt: '>', le: '<=', ge: '>='
|
|
}
|
|
const js = out
|
|
.map(tok => {
|
|
if (tok.type === 'str') return JSON.stringify(tok.value)
|
|
if (tok.type === 'num') return tok.value
|
|
if (tok.type === 'word') return OPS[tok.value.toLowerCase()] || tok.value
|
|
if (tok.value === '=') return '=='
|
|
if (tok.value === '<>') return '!='
|
|
return tok.value
|
|
})
|
|
.join(' ')
|
|
|
|
const sasTrue = v => {
|
|
if (typeof v === 'boolean') return v
|
|
if (typeof v === 'number') return v !== 0 && !isNaN(v)
|
|
if (v === null || v === undefined) return false
|
|
const str = String(v).trim()
|
|
return str !== '' && str !== '0'
|
|
}
|
|
const toStr = v => {
|
|
if (v === null || v === undefined) return ''
|
|
if (typeof v === 'number') return isNaN(v) ? '' : String(v)
|
|
return String(v).trim()
|
|
}
|
|
const fns = {
|
|
ifc: (c, a, b) => (sasTrue(c) ? a : b),
|
|
cats: (...args) => args.map(toStr).join(''),
|
|
catx: (sep, ...args) => args.map(toStr).filter(v => v !== '').join(toStr(sep)),
|
|
upcase: v => toStr(v).toUpperCase(),
|
|
lowcase: v => toStr(v).toLowerCase(),
|
|
strip: v => toStr(v),
|
|
trim: v => toStr(v),
|
|
length: v => toStr(v).length,
|
|
substr: (v, p, n) => {
|
|
const str = toStr(v)
|
|
const start = Number(p) - 1
|
|
return n === undefined
|
|
? str.substring(start)
|
|
: str.substring(start, start + Number(n))
|
|
},
|
|
index: (v, sub) => toStr(v).indexOf(toStr(sub)) + 1,
|
|
__in: (list, v) => list.some(item => String(item) === String(v) || Number(item) === Number(v))
|
|
}
|
|
|
|
const names = Object.keys(fns)
|
|
const fn = new Function(...columnNames, ...names, 'return (' + js + ');')
|
|
return row => fn(...columnNames.map(n => row[n]), ...names.map(n => fns[n]))
|
|
}
|
|
|
|
/**
|
|
* Reads MPE_DERIVED for a table, mirroring %mpe_derived. Returns the active
|
|
* variables with a compiled row function, in dependency order (ties by name),
|
|
* the same sequence the real macro emits.
|
|
*
|
|
* @param {Object} opts
|
|
* @param {string} opts.libref - target libref
|
|
* @param {string} opts.table - target dataset
|
|
* @param {string} opts.dataDir - data directory of the DC control library
|
|
* @param {Array<Object>} opts.columns - physical columns of the target table
|
|
*/
|
|
function mpeDerived(opts) {
|
|
const { libref, table, dataDir, columns } = opts
|
|
const loadTableData = makeTableLoader(dataDir)
|
|
const cfg = loadTableData('MPE_DERIVED')
|
|
if (!cfg || !Array.isArray(cfg.rows) || cfg.rows.length === 0) return []
|
|
|
|
const now = Date.now() / 1000
|
|
const lib = String(libref || '').toUpperCase()
|
|
const dsn = String(table || '').toUpperCase()
|
|
const columnNames = (columns || []).map(c => c.name)
|
|
|
|
const vars = cfg.rows
|
|
.filter(r =>
|
|
String(colVal(r, 'LIBREF') || '').toUpperCase() === lib &&
|
|
String(colVal(r, 'DSN') || '').toUpperCase() === dsn &&
|
|
Number(colVal(r, 'ACTIVE')) === 1 &&
|
|
(!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
|
|
)
|
|
.map(r => {
|
|
const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
|
|
const nm = String(colVal(r, 'VAR_NM') || '').toUpperCase()
|
|
return {
|
|
name: nm,
|
|
// every input the expression uses - columns and derived vars
|
|
deps: String(colVal(r, 'VAR_DEPS') || '')
|
|
.toUpperCase()
|
|
.split(/\s+/)
|
|
.filter(Boolean),
|
|
type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
|
|
length: Number(colVal(r, 'VAR_LEN')) || 8,
|
|
format: String(colVal(r, 'VAR_FMT') || ''),
|
|
logic,
|
|
label: String(colVal(r, 'VAR_LABEL') || '').trim() || nm + '=' + logic
|
|
}
|
|
})
|
|
.filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
|
|
|
|
// dependency order: repeatedly take the variables whose derived inputs are
|
|
// already placed, alphabetically within a round
|
|
const byName = new Map(vars.map(v => [v.name, v]))
|
|
const placed = []
|
|
const done = new Set()
|
|
while (placed.length < vars.length) {
|
|
const ready = vars
|
|
.filter(v => !done.has(v.name))
|
|
.filter(v => v.deps.every(d => !byName.has(d) || done.has(d)))
|
|
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
|
|
if (ready.length === 0) {
|
|
console.log('derived variables for ' + lib + '.' + dsn + ' contain a cycle')
|
|
return []
|
|
}
|
|
for (const v of ready) {
|
|
placed.push(v)
|
|
done.add(v.name)
|
|
}
|
|
}
|
|
vars.length = 0
|
|
vars.push.apply(vars, placed)
|
|
|
|
// compile each against the physical columns plus the variables defined
|
|
// earlier, so a derived variable can build on another one
|
|
const compileCols = columnNames.slice()
|
|
for (const d of vars) {
|
|
try {
|
|
// slice(): the compiled function captures the column list it was
|
|
// built with, not the array as it grows
|
|
d.fn = compileSasExpression(d.logic, compileCols.slice())
|
|
} catch (err) {
|
|
console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
|
|
d.fn = () => (d.type === 'N' ? null : '')
|
|
}
|
|
compileCols.push(d.name)
|
|
}
|
|
return vars
|
|
}
|
|
|
|
/**
|
|
* Adds the derived variables to every row and to the column list, so that the
|
|
* filter (which runs afterwards) and the grid both see them.
|
|
*/
|
|
function mpeDeriveRows(tableData, derived) {
|
|
if (!derived || derived.length === 0) return tableData
|
|
const rows = tableData.rows.map(row => {
|
|
const copy = Object.assign({}, row)
|
|
for (const d of derived) {
|
|
try {
|
|
copy[d.name] = d.fn(copy)
|
|
} catch (err) {
|
|
copy[d.name] = d.type === 'N' ? null : ''
|
|
}
|
|
}
|
|
return copy
|
|
})
|
|
const derivedCols = derived.map(d => ({
|
|
name: d.name,
|
|
type: d.type,
|
|
length: d.length,
|
|
format: d.format,
|
|
label: d.label,
|
|
notnull: false
|
|
}))
|
|
return {
|
|
name: tableData.name,
|
|
columns: (tableData.columns || []).concat(derivedCols),
|
|
rows
|
|
}
|
|
}
|
|
|
|
// ─── DC settings (services/settings.js) ──────────────────────────────────────
|
|
|
|
/**
|
|
* Loads the DC settings written by makedata to services/settings.js
|
|
* (the JS equivalent of the real backend's settings.sas).
|
|
* Returns { dcLibref, dataDir, dcAdminGroup, dcpath } - with defaults when
|
|
* makedata has not run yet.
|
|
* Requires nodePath, fs, driveRoot and appLoc in the caller's scope.
|
|
*/
|
|
function loadDcSettings(driveRoot, appLoc) {
|
|
const result = {
|
|
dcLibref: 'DC_JSLIB',
|
|
dataDir: nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
|
|
dcAdminGroup: null,
|
|
dcpath: null
|
|
}
|
|
try {
|
|
const settingsFile = nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'settings.js')
|
|
const settingsRaw = fs.readFileSync(settingsFile, {encoding:'utf8'}).toString()
|
|
eval(settingsRaw)
|
|
if (typeof dcSettings !== 'undefined') {
|
|
if (dcSettings.dc_libref) result.dcLibref = dcSettings.dc_libref
|
|
if (dcSettings.dc_admin_group) result.dcAdminGroup = dcSettings.dc_admin_group
|
|
if (dcSettings.dcpath) result.dcpath = dcSettings.dcpath
|
|
result.dataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', result.dcLibref)
|
|
}
|
|
} catch(err) { /* makedata hasn't run - keep defaults */ }
|
|
return result
|
|
}
|
|
|
|
/**
|
|
* Reads the DC-scope options out of MPE_CONFIG, mirroring the loop in the real
|
|
* backend's generated settings.sas: only rows that are active (var_active=1)
|
|
* and current (now < tx_to) count, and the latest tx_from wins. Returns a map
|
|
* of var_name -> trimmed var_value.
|
|
*
|
|
* An option that is absent, or switched off, is simply missing from the map so
|
|
* the caller's own default applies - which is what the real service's
|
|
* %symexist fallback does (a switched-off option means "use the code default",
|
|
* not "no limit").
|
|
*
|
|
* Requires nodePath, fs and the caller's driveRoot/appLoc/dataDir.
|
|
*/
|
|
function loadDcConfig(driveRoot, appLoc, dataDir) {
|
|
const out = {}
|
|
try {
|
|
const configFile = nodePath.resolve(
|
|
dataDir || nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
|
|
'mpe_config.json'
|
|
)
|
|
const config = JSON.parse(fs.readFileSync(configFile, {encoding:'utf8'}).toString())
|
|
const now = Math.floor(Date.now() / 1000)
|
|
const latest = {}
|
|
for (const row of (config.rows || [])) {
|
|
// colVal, not row.var_name: a row that has been through the app's
|
|
// own edit and approve flow comes back from the adapter with
|
|
// upper-case keys, and a direct property read would skip it - which
|
|
// silently drops the option back to its default.
|
|
if (colVal(row, 'VAR_SCOPE') !== 'DC') continue
|
|
if (Number(colVal(row, 'VAR_ACTIVE')) !== 1) continue
|
|
if (Number(colVal(row, 'TX_TO')) <= now) continue
|
|
const name = String(colVal(row, 'VAR_NAME') || '').trim()
|
|
if (!name) continue
|
|
const from = Number(colVal(row, 'TX_FROM')) || 0
|
|
if (latest[name] === undefined || from > latest[name]) {
|
|
const val = colVal(row, 'VAR_VALUE')
|
|
latest[name] = from
|
|
out[name] = String(val == null ? '' : val).trim()
|
|
}
|
|
}
|
|
} catch(err) { /* no MPE_CONFIG - every option falls back to its default */ }
|
|
return out
|
|
}
|
|
|
|
/**
|
|
* A numeric DC option out of a loadDcConfig() map, or the service's own default
|
|
* when the option is absent, blank, or not a number.
|
|
*/
|
|
function dcConfigNumber(cfg, name, fallback) {
|
|
const raw = cfg ? cfg[name] : undefined
|
|
if (raw === undefined || raw === '') return fallback
|
|
const num = Number(raw)
|
|
return isNaN(num) ? fallback : num
|
|
}
|
|
|
|
// ─── %webout(OPEN/OBJ/CLOSE) emulation ───────────────────────────────────────
|
|
|
|
/**
|
|
* Internal buffer for the webOut stream.
|
|
* webOutOpen() starts it, webOutObj() appends tables, webOutClose()
|
|
* finalises and assigns to _webout.
|
|
*/
|
|
let _webOutBuf = ''
|
|
let _webOutFirst = true
|
|
|
|
/**
|
|
* Opens the JSON stream, writing the opening brace and SYSDATE/SYSTIME.
|
|
* Mirrors %webout(OPEN) / mv_webout(OPEN).
|
|
*
|
|
* Usage:
|
|
* webOutOpen()
|
|
* webOutObj(rows, 'sasdata', { vars })
|
|
* webOutObj(params, 'sasparams')
|
|
* webOutClose()
|
|
*/
|
|
function webOutOpen() {
|
|
const now = new Date()
|
|
const pad = n => String(n).padStart(2, '0')
|
|
const dateStr = now.getDate() + '' +
|
|
['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC'][now.getMonth()] +
|
|
String(now.getFullYear()).slice(2)
|
|
const timeStr = pad(now.getHours()) + ':' + pad(now.getMinutes())
|
|
_webOutBuf = '{"SYSDATE" : "' + dateStr + '"'
|
|
_webOutBuf += ',"SYSTIME" : "' + timeStr + '"'
|
|
_webOutFirst = false
|
|
}
|
|
|
|
/**
|
|
* Appends a table to the JSON stream.
|
|
* Mirrors %webout(OBJ,ds,dslabel=,showmeta=).
|
|
*
|
|
* @param {Array<Object>} rows - Array of row objects to send as a JSON array.
|
|
* @param {string} label - The key name in the JSON (e.g. 'sasdata', 'sasparams').
|
|
* @param {Object} [meta] - Optional metadata object. If provided, it is sent
|
|
* as `$label` (e.g. `$sasdata`), matching the showmeta=YES output from SAS.
|
|
* Typically { vars: { COL1: {format,label,length,type}, ... } }.
|
|
*/
|
|
function webOutObj(rows, label, meta) {
|
|
_webOutBuf += ',"' + label + '":' + JSON.stringify(rows)
|
|
if (meta) {
|
|
_webOutBuf += ',"$' + label + '":' + JSON.stringify(meta)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Closes the JSON stream, appends the standard SAS runtime variables
|
|
* (SYSDATE, SYSCC, _PROGRAM, etc.), and assigns the final JSON string to
|
|
* the `_webout` variable that the SASjs Server reads.
|
|
* Mirrors mv_webout(CLOSE).
|
|
*/
|
|
// Current user for MPE_SUBMIT writes/reads - mock equivalent of %mf_getuser().
|
|
// SASjs Server predeclares _METAUSER for authenticated requests; the desktop
|
|
// mode (no auth) leaves it undefined.
|
|
function getDcUser() {
|
|
return typeof _METAUSER !== 'undefined' && _METAUSER ? _METAUSER : 'sasdemo'
|
|
}
|
|
|
|
function webOutClose() {
|
|
const now = new Date()
|
|
const pad = n => String(n).padStart(2, '0')
|
|
const endDttm = now.getFullYear() + '-' + pad(now.getMonth()+1) + '-' + pad(now.getDate()) +
|
|
'T' + pad(now.getHours()) + ':' + pad(now.getMinutes()) + ':' + pad(now.getSeconds()) +
|
|
'.' + String(now.getMilliseconds()).padStart(3,'0')
|
|
|
|
// Derive system vars from SASjs Server runtime variables where possible
|
|
const mfGetuser = getDcUser()
|
|
const metaperson = (typeof _METAPERSON !== 'undefined') ? _METAPERSON : 'sasdemo'
|
|
const sysprocmode = (typeof SASJSPROCESSMODE !== 'undefined') ? SASJSPROCESSMODE : 'Stored Program'
|
|
|
|
_webOutBuf += ',"_DEBUG" : ""'
|
|
_webOutBuf += ',"_METAUSER": "' + mfGetuser + '@SAS"'
|
|
_webOutBuf += ',"_METAPERSON": "' + metaperson + '"'
|
|
_webOutBuf += ',"_PROGRAM" : "' + (typeof _program !== 'undefined' ? _program : '') + '"'
|
|
_webOutBuf += ',"AUTOEXEC" : ""'
|
|
_webOutBuf += ',"MF_GETUSER" : "' + mfGetuser + '"'
|
|
_webOutBuf += ',"SYS_JES_JOB_URI" : " "'
|
|
_webOutBuf += ',"SYSCC" : "0"'
|
|
_webOutBuf += ',"SYSENCODING" : "utf-8"'
|
|
_webOutBuf += ',"SYSERRORTEXT" : ""'
|
|
_webOutBuf += ',"SYSHOSTNAME" : "sasjs-server"'
|
|
_webOutBuf += ',"SYSPROCESSID" : "0"'
|
|
_webOutBuf += ',"SYSPROCESSMODE" : "' + sysprocmode + '"'
|
|
_webOutBuf += ',"SYSPROCESSNAME" : ""'
|
|
_webOutBuf += ',"SYSJOBID" : "0"'
|
|
_webOutBuf += ',"SYSSCPL" : "LINUX"'
|
|
_webOutBuf += ',"SYSSITE" : "123"'
|
|
_webOutBuf += ',"SYSUSERID" : "sasdemo"'
|
|
_webOutBuf += ',"SYSVLONG" : "9.04.01M7P080520"'
|
|
_webOutBuf += ',"SYSWARNINGTEXT" : ""'
|
|
_webOutBuf += ',"END_DTTM" : "' + endDttm + '"'
|
|
_webOutBuf += ',"MEMSIZE" : "0KB"'
|
|
_webOutBuf += '}'
|
|
_webout = _webOutBuf
|
|
}
|
|
|
|
/**
|
|
* Convenience: builds a complete response in one call.
|
|
* Opens, adds each table, closes.
|
|
*
|
|
* @param {Object} tables - { label1: { rows: [...], meta: {...} }, label2: { rows: [...] } }
|
|
*/
|
|
function webOutSend(tables) {
|
|
webOutOpen()
|
|
for (const [label, entry] of Object.entries(tables)) {
|
|
webOutObj(entry.rows, label, entry.meta)
|
|
}
|
|
webOutClose()
|
|
}
|
|
|
|
// ─── %mpeinit() emulation ────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* The `_debug` values that mean "debug on", mirroring the test in mpeinit.sas.
|
|
*
|
|
* 131 is what the adapter sends - SASjsApiClient.executeJob() puts it on every
|
|
* server-mode request, and on the multipart form when the payload goes out as
|
|
* FormData. 128 is what arrives on the Viya WEB JES path when runAsTask is
|
|
* enabled, and 2477 / 'fields,log,trace' are the SASjs debug values.
|
|
*/
|
|
const MPEINIT_DEBUG_VALUES = ['2477', 'fields,log,trace', '131', '128']
|
|
|
|
/* Mirrors the &mpeinit=1 guard in the SAS macro: dump once per execution. */
|
|
let _mpeinitDone = false
|
|
|
|
/**
|
|
* Dumps the browser_url_vars and browser_info input tables, mirroring the
|
|
* debug block of %mpeinit (sas/sasjs/macros/mpeinit.sas).
|
|
*
|
|
* The SAS macro reads work.browser_url_vars (name/value pairs taken from the
|
|
* query string) and work.browser_info (the browser fingerprint). Both arrive
|
|
* here as input tables - see fetchRaw() - and are dumped with console.log,
|
|
* which SASjs Server returns in the request's log (after its
|
|
* SASJS_LOGS_SEPARATOR marker), so the dump shows up in Data Controller's
|
|
* SAS Log tab exactly as the SAS version does.
|
|
*
|
|
* Called at the bottom of this file, so the services that eval() it dump the
|
|
* tables at startup - the same place %mpeinit() runs in the SAS services.
|
|
* That covers every diagnostics-carrying service (startup, editors, auditors);
|
|
* the mocks that do not load this file (lineage, metanav, usernav, a few
|
|
* public ones) have no equivalent startup step in the SAS app either.
|
|
*
|
|
* Difference from the SAS version: the header line is printed once per table,
|
|
* where SAS repeats it for every row (its PUTLOG sits inside the SET loop).
|
|
*/
|
|
function mpeinit() {
|
|
if (_mpeinitDone) return
|
|
|
|
const debugValue = typeof _debug === 'undefined' ? '' : _debug
|
|
if (MPEINIT_DEBUG_VALUES.indexOf(String(debugValue)) < 0) return
|
|
|
|
_mpeinitDone = true
|
|
|
|
const urlVars = fetchRaw('browser_url_vars')
|
|
if (urlVars === null) {
|
|
console.log('NOTE: mpeinit: no work.browser_url_vars on this request')
|
|
} else {
|
|
console.log('NOTE: mpeinit: work.browser_url_vars:')
|
|
parseCsv(urlVars).forEach(row => {
|
|
console.log(
|
|
'name=' + colVal(row, 'name') + ' value=' + colVal(row, 'value')
|
|
)
|
|
})
|
|
}
|
|
|
|
const browserInfo = fetchRaw('browser_info')
|
|
if (browserInfo === null) {
|
|
console.log('NOTE: mpeinit: no work.browser_info on this request')
|
|
} else {
|
|
console.log('NOTE: mpeinit: work.browser_info:')
|
|
parseCsv(browserInfo).forEach(row => {
|
|
console.log(Object.keys(row).map(col => col + '=' + row[col]).join(' '))
|
|
})
|
|
}
|
|
}
|
|
|
|
/* Every service that eval()s this file runs %mpeinit() at startup, the same
|
|
place the SAS services call it. */
|
|
mpeinit()
|