/**
 * @file dcMockUtils.js
 * @brief Shared utilities for Data Controller JS mock services on SASjs Server.
 *
 * Provides:
 *   - fetchTable / fetchRaw / parseCsv : emulate SAS %webout(FETCH)
 *   - webOutOpen / webOutObj / webOutClose : emulate SAS %webout(OPEN/OBJ/CLOSE)
 *   - mpeinit                         : emulate the %mpeinit debug dump
 *   - formatSasValue                  : apply a SAS format to a numeric value
 *   - parseFormattedToSas             : convert an ISO string back to a SAS numeric
 *   - getDdType                       : derive DATE/DATETIME/TIME/N/C from a column format
 *   - loadTableData / loadTableSchema : read a DC data JSON file from the Drive
 *   - sasVarsRow                      : build a $sasdata/$viewdata vars entry for a column
 *   - sasVarsRow                      : build a $sasdata/$viewdata vars entry for a column
 *
 * Usage (from a mock service that is eval'd by the SASjs Server):
 *   const _utilPath = nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js')
 *   eval(fs.readFileSync(_utilPath, 'utf8'))
 *   const rows = fetchTable('SASControlTable')
 *   const libds = rows[0]?.LIBDS
 */

// ─── SAS epoch ────────────────────────────────────────────────────────────────

const SAS_EPOCH_MS = Date.UTC(1960, 0, 1)
const MONTHS_ABBR = ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC']
const MONTHS_NUM = {JAN:'01',FEB:'02',MAR:'03',APR:'04',MAY:'05',JUN:'06',JUL:'07',AUG:'08',SEP:'09',OCT:'10',NOV:'11',DEC:'12'}

// ─── %webout(FETCH) emulation ────────────────────────────────────────────────

/**
 * Fetches the raw CSV text of a named input table.
 * Uses eval() to access variables in the CALLER's scope.
 */
function fetchRaw(tableName) {
    // 1. Direct variable (SASjs Server may set form fields as top-level JS vars)
    try {
        const val = eval(tableName)
        if (val !== undefined && val !== null && val !== '') return val.toString()
    } catch(e) { /* not defined */ }

    // 2. _WEBIN_NAME / _WEBIN_FILEREF loop (SASjs Server format)
    //    Variable names are UPPERCASE: _WEBIN_NAME1, _WEBIN_FILEREF1, etc.
    //    _WEBIN_FILEREF may be a Buffer (from fs.readFileSync) or a string.
    for (let i = 1; i <= 10; i++) {
        try {
            const nameVal = eval('_WEBIN_NAME' + i)
            if (nameVal && nameVal.toLowerCase() === tableName.toLowerCase()) {
                const refVal = eval('_WEBIN_FILEREF' + i)
                if (refVal) return refVal.toString()
            }
        } catch(e) {}
        // Also try lowercase variant
        try {
            const nameVal = eval('_webin_name' + i)
            if (nameVal && nameVal.toLowerCase() === tableName.toLowerCase()) {
                const refVal = eval('_webin_fileref' + i)
                if (refVal) return refVal.toString()
            }
        } catch(e) {}
    }

    // 3. _WEBIN_FILENAME / _WEBIN_FILEREF fallback
    for (let i = 1; i <= 10; i++) {
        try {
            const nameVal = eval('_WEBIN_FILENAME' + i)
            if (nameVal && nameVal.toLowerCase().includes(tableName.toLowerCase())) {
                const refVal = eval('_WEBIN_FILEREF' + i)
                if (refVal) return refVal.toString()
            }
        } catch(e) {}
        try {
            const nameVal = eval('_webin_filename' + i)
            if (nameVal && nameVal.toLowerCase().includes(tableName.toLowerCase())) {
                const refVal = eval('_webin_fileref' + i)
                if (refVal) return refVal.toString()
            }
        } catch(e) {}
    }

    return null
}

/**
 * Fetches a named input table as parsed row objects.
 */
function fetchTable(tableName) {
    const raw = fetchRaw(tableName)
    if (!raw) return []
    return parseCsv(raw)
}

/**
 * Parses just the header line of a SASjs adapter CSV string
 * ("COL1:$char10. COL2:$char8." -> ['COL1', 'COL2']).
 */
function parseCsvHeader(csvText) {
    if (!csvText) return []
    const lines = csvText.split(/\r?\n/)
    if (lines.length < 1) return []
    return lines[0].trim().split(/\s+/).map((h) => h.split(':')[0])
}

/**
 * Parses a SASjs adapter CSV string into an array of row objects.
 * Header line: "COL1:$char10. COL2:$char8."  (space-separated, name before colon)
 * Data lines: comma-separated with double-quote escaping.
 */
function parseCsv(csvText) {
    if (!csvText) return []
    const lines = csvText.split(/\r?\n/)
    if (lines.length < 2) return []

    const headers = lines[0].trim().split(/\s+/).map(h => h.split(':')[0])
    if (headers.length === 0) return []

    const rows = []
    for (let i = 1; i < lines.length; i++) {
        const line = lines[i].trim()
        if (!line) continue
        const values = _parseCsvLine(line)
        const row = {}
        for (let j = 0; j < headers.length && j < values.length; j++) {
            row[headers[j]] = values[j]
        }
        rows.push(row)
    }
    return rows
}

function _parseCsvLine(line) {
    const values = []
    let cur = ''
    let inQuote = false
    for (let j = 0; j < line.length; j++) {
        const ch = line[j]
        if (ch === '"') {
            if (inQuote && line[j + 1] === '"') { cur += '"'; j++ }
            else inQuote = !inQuote
        } else if (ch === ',' && !inQuote) {
            values.push(cur); cur = ''
        } else { cur += ch }
    }
    values.push(cur)
    return values
}

// ─── SAS format <-> ISO conversions ──────────────────────────────────────────

/**
 * Column metadata, mirroring %mp_getcols.sas.
 *
 * The SAS macro runs proc contents and derives, per column: NAME (upcased),
 * TYPE (C/N), LENGTH, LABEL (defaulting to the name), VARNUM, FMTNAME, FORMAT
 * and DDTYPE - CHARACTER, NUMERIC, DATETIME, DATE or TIME.  The date list is
 * the macro's (YYMMDD, MMDDYY, DDMMYY, MONYY, E8601DA, B8601DA, E8601DT,
 * NLDATM, NLDATE), so a column types the same way here as it does in the
 * service that shares this code.
 *
 * @param {Array} schemaCols the table's columns as the mock data stores them
 *   ({name, type, length, format, label}), the equivalent of the proc contents
 *   output the macro reads
 * @returns {Array} one entry per column, in the macro's shape and order
 */
function mpGetcols(schemaCols) {
    return (schemaCols || []).map((col, index) => {
        const name = String(col.name || '').toUpperCase()
        const isChar = String(col.type || '').toUpperCase() === 'C'
        const fmtname = String(col.format || '')
            .replace(/[\d.]+$/, '')
            .toUpperCase()

        let ddtype
        if (isChar) ddtype = 'CHARACTER'
        else if (fmtname.startsWith('DATETIME') || fmtname.startsWith('E8601DT')
            || fmtname.startsWith('NLDATM')) ddtype = 'DATETIME'
        else if (fmtname.startsWith('DATE') || fmtname.startsWith('DDMMYY')
            || fmtname.startsWith('MMDDYY') || fmtname.startsWith('YYMMDD')
            || fmtname.startsWith('E8601DA') || fmtname.startsWith('B8601DA')
            || fmtname.startsWith('MONYY') || fmtname.startsWith('NLDATE')) ddtype = 'DATE'
        else if (fmtname.startsWith('TIME')) ddtype = 'TIME'
        else ddtype = 'NUMERIC'

        return {
            NAME: name,
            TYPE: isChar ? 'C' : 'N',
            LENGTH: Number(col.length) || 0,
            LABEL: col.label || col.name,
            VARNUM: index + 1,
            FORMAT: String(col.format || ''),
            FMTNAME: fmtname,
            DDTYPE: ddtype
        }
    })
}

/**
 * Derives a DDTYPE (DATE, DATETIME, TIME, N, C) from a SAS column format
 * string, in the shape the client expects.  Delegates to mpGetcols so the
 * inference is the macro's, rather than a second format list that can drift
 * away from the service's.
 */
function getDdType(col) {
    const ddtype = mpGetcols([col])[0].DDTYPE
    if (ddtype === 'CHARACTER') return 'C'
    if (ddtype === 'NUMERIC') return 'N'
    return ddtype
}

/**
 * Applies a SAS format to a raw numeric value, producing the display string.
 *   DATE:     42  -> "12FEB1960"
 *   DATETIME: 42  -> "01JAN1960:00:00:42"
 *   TIME:     42  -> "0:00:42"
 */
function formatSasValue(value, ddtype) {
    if (value === null || value === undefined || value === '') return String(value)
    if (typeof value !== 'number') return String(value)

    if (ddtype === 'DATE') {
        const d = new Date(SAS_EPOCH_MS + value * 86400000)
        return `${String(d.getUTCDate()).padStart(2,'0')}${MONTHS_ABBR[d.getUTCMonth()]}${d.getUTCFullYear()}`
    }
    if (ddtype === 'DATETIME') {
        const d = new Date(SAS_EPOCH_MS + value * 1000)
        return `${String(d.getUTCDate()).padStart(2,'0')}${MONTHS_ABBR[d.getUTCMonth()]}${d.getUTCFullYear()}:${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`
    }
    if (ddtype === 'TIME') {
        const h = Math.floor(value / 3600)
        const m = Math.floor((value % 3600) / 60)
        const s = value % 60
        return `${h}:${String(m).padStart(2,'0')}:${String(s).padStart(2,'0')}`
    }
    return String(value)
}

/**
 * Converts a numeric SAS value to the ISO string Handsontable expects.
 *   DATE:     42  -> "1960-02-12"
 *   DATETIME: 42  -> "1960-01-01 00:00:42"
 *   TIME:     42  -> "00:00:42"
 */
function sasToIso(value, ddtype) {
    if (value === null || value === undefined || value === '') return value
    if (typeof value !== 'number') return value

    if (ddtype === 'DATE') {
        const d = new Date(SAS_EPOCH_MS + value * 86400000)
        return `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')}`
    }
    if (ddtype === 'DATETIME') {
        const d = new Date(SAS_EPOCH_MS + value * 1000)
        return `${d.getUTCFullYear()}-${String(d.getUTCMonth()+1).padStart(2,'0')}-${String(d.getUTCDate()).padStart(2,'0')} ${String(d.getUTCHours()).padStart(2,'0')}:${String(d.getUTCMinutes()).padStart(2,'0')}:${String(d.getUTCSeconds()).padStart(2,'0')}`
    }
    if (ddtype === 'TIME') {
        const h = Math.floor(value / 3600)
        const m = Math.floor((value % 3600) / 60)
        const s = value % 60
        return `${String(h).padStart(2,'0')}:${String(m).padStart(2,'0')}:${String(s).padStart(2,'0')}`
    }
    return value
}

/**
 * Converts an ISO-formatted string (as sent by the editor) back to a SAS numeric.
 *   DATE:     "1960-02-12"          -> 42
 *   DATETIME: "1960-01-01 00:00:42" -> 42
 *   TIME:     "00:00:42"            -> 42
 * Non-temporal values pass through unchanged.
 */
function parseFormattedToSas(value, ddtype) {
    if (value === null || value === undefined || value === '') return value
    if (ddtype !== 'DATE' && ddtype !== 'DATETIME' && ddtype !== 'TIME') return value
    if (typeof value === 'number') return value
    const str = String(value)

    if (ddtype === 'DATE') {
        const m = str.match(/^(\d{4})-(\d{2})-(\d{2})$/)
        if (m) return Math.round((Date.UTC(+m[1], +m[2]-1, +m[3]) - SAS_EPOCH_MS) / 86400000)
        return value
    }
    if (ddtype === 'DATETIME') {
        const m = str.match(/^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})/)
        if (m) return Math.round((Date.UTC(+m[1], +m[2]-1, +m[3], +m[4], +m[5], +m[6]) - SAS_EPOCH_MS) / 1000)
        return value
    }
    if (ddtype === 'TIME') {
        const parts = str.split(':')
        if (parts.length >= 3) return (+parts[0]) * 3600 + (+parts[1]) * 60 + (+parts[2])
        return value
    }
    return value
}

/**
 * Handles either numeric or string temporal values, converting to ISO for
 * Handsontable.  Numeric values use sasToIso; string values use the legacy
 * regex parsing (for backwards compatibility with old data files).
 */
function formatCellValue(value, ddtype) {
    if (value === null || value === undefined || value === '') return value
    if (typeof value === 'number') return sasToIso(value, ddtype)

    const str = String(value)
    if (ddtype === 'DATE') {
        const m = str.match(/^(\d{1,2})([A-Z]{3})(\d{4})$/)
        if (m) return `${m[3]}-${MONTHS_NUM[m[2]]}-${m[1].padStart(2,'0')}`
        return str
    }
    if (ddtype === 'DATETIME') {
        const m = str.match(/^(\d{1,2})([A-Z]{3})(\d{4}):(\d{1,2}):(\d{2}):(\d{2})$/)
        if (m) return `${m[3]}-${MONTHS_NUM[m[2]]}-${m[1].padStart(2,'0')} ${m[4].padStart(2,'0')}:${m[5]}:${m[6]}`
        return str
    }
    if (ddtype === 'TIME') {
        const parts = str.split(':')
        if (parts.length >= 3) return `${parts[0].padStart(2,'0')}:${parts[1].padStart(2,'0')}:${parts[2].padStart(2,'0')}`
        return str
    }
    return value
}

// ─── DC data file helpers ────────────────────────────────────────────────────

/**
 * Resolves the data directory for a given libref.
 * Each libref has its own subfolder under <appLoc>/data/ (e.g. DC_JSLIB,
 * TESTDATA). DC_JSLIB is the default control-library (mpelib) and is used
 * for MPE_* config tables; other librefs hold user/demo data tables.
 * Requires nodePath, driveRoot and appLoc in the caller's scope.
 */
function libDataDir(libref) {
    return nodePath.resolve(driveRoot, 'files', appLoc, 'data', libref)
}

/**
 * Lists all librefs that have a data folder on the Drive.
 * Requires nodePath, driveRoot and appLoc in the caller's scope.
 */
function allLibrefs() {
    const base = nodePath.resolve(driveRoot, 'files', appLoc, 'data')
    try {
        return fs.readdirSync(base)
            .filter(f => {
                const p = nodePath.resolve(base, f)
                return fs.existsSync(p) && fs.statSync(p).isDirectory()
            })
            .sort()
    } catch (err) { return [] }
}

/**
 * Creates a loadTableData function bound to a specific data directory.
 *   const loadTableData = makeTableLoader(dataDir)
 *   const mpeXTest = loadTableData('MPE_X_TEST')
 */
function makeTableLoader(dataDir) {
    return function loadTableData(tableName) {
        try {
            const file = nodePath.resolve(dataDir, tableName.toLowerCase() + '.json')
            return JSON.parse(fs.readFileSync(file, {encoding:'utf8'}).toString())
        } catch(err) { return null }
    }
}

/**
 * Loads a table from any libref, searching every data folder until a JSON
 * file named <table>.json is found. Returns { data, libref } or null.
 * Requires nodePath, driveRoot and appLoc in the caller's scope.
 */
function loadTableAnyLib(tableName) {
    for (const lib of allLibrefs()) {
        const dir = libDataDir(lib)
        const loader = makeTableLoader(dir)
        const data = loader(tableName)
        if (data) return { data, libref: lib }
    }
    return null
}

/**
 * Loads just the column schema (array of column metadata) for a table.
 */
function loadTableSchema(dataDir, tableName) {
    const data = makeTableLoader(dataDir)(tableName)
    return (data && data.columns) ? data.columns : []
}

/**
 * Resolves a hook program name to the source of its mock implementation.
 *
 * MPE_TABLES.pre_edit_hook / post_edit_hook (and the SOFTSELECT_HOOK /
 * HARDSELECT_HOOK rule_value in MPE_VALIDATIONS) hold a hook program: either a
 * physical .sas path, or a path relative to the appLoc (e.g.
 * 'services/hooks/mytable_postedit').  The real backend %includes the program
 * into the running service, so the hook shares the service's macro variables
 * and work datasets and can modify them.
 *
 * The mock mirrors that: this returns the source of the hook's .js counterpart,
 * which the calling service eval()s in its OWN scope so the hook can read and
 * modify the service's variables.  Returns null when the hook has no mock
 * implementation, so callers can fall back to an inline emulation or ignore it.
 *
 * Requires nodePath, driveRoot and appLoc in the caller's scope.
 */
function mockHookSource(hookValue) {
    if (!hookValue) return null
    let hook = String(hookValue).trim()
    if (!hook) return null
    // Reduce to the services-relative form, dropping any drive path or appLoc
    // prefix the config may carry.
    const idx = hook.indexOf('services/')
    if (idx > -1) hook = hook.slice(idx)
    // A physical hook is a .sas program on the server; its mock is a .js file
    // sitting in the same place on the Drive.
    hook = hook.replace(/\.sas$/i, '')
    const hookFile = nodePath.resolve(driveRoot, 'files', appLoc, hook + '.js')
    // The hook value is hand-edited mock config, and a value carrying '..'
    // segments would otherwise resolve outside the Drive and be eval()'d
    // in-service. Keep it inside the Drive's files tree.
    const filesRoot = nodePath.resolve(driveRoot, 'files')
    if (!hookFile.startsWith(filesRoot + nodePath.sep)) return null
    if (!fs.existsSync(hookFile)) return null
    return fs.readFileSync(hookFile, 'utf8')
}

/**
 * Synthesises NOTNULL dqrules from the table schema, mirroring the
 * dictionary.columns union in getdata.sas:
 *   select upcase(name) as base_col, 'NOTNULL' as rule_type, '' as rule_value
 *   from dictionary.columns where ... and notnull='yes'
 * Returns an array of { BASE_COL, RULE_TYPE, RULE_VALUE } entries for every
 * schema column whose `notnull` flag is true.
 */
function notNullRules(schema) {
    if (!schema) return []
    const rules = []
    for (const col of schema) {
        if (col.notnull) {
            rules.push({
                BASE_COL: String(col.name).toUpperCase(),
                RULE_TYPE: 'NOTNULL',
                RULE_VALUE: ''
            })
        }
    }
    return rules
}

/**
 * Builds a $sasdata/$viewdata vars entry for a column.
 * Temporal columns are sent as char ($200.) by the real SAS backend.
 */
function sasVarsEntry(col) {
    const ddtype = getDdType(col)
    if (ddtype === 'DATE' || ddtype === 'DATETIME' || ddtype === 'TIME') {
        return { format: '$200.', label: col.label || col.name, length: '200', type: 'char' }
    }
    const fmt = col.type === 'N' ? (col.format || 'best.') : ('$' + col.length + '.')
    return { format: fmt, label: col.label || col.name, length: String(col.length), type: col.type === 'N' ? 'num' : 'char' }
}

/**
 * Assigns the retained key (RK) to staged rows for a table registered with
 * rk_underlying, mirroring %mp_retainedkey as %mpe_loader invokes it: any RK
 * column in the input is dropped, and every row's RK is derived from its
 * business key - the value that key already holds on the base table, or the
 * next key value for a new one. Business keys are matched the way SAS compares
 * them: case-sensitive, trailing blanks ignored.
 *
 * @param {Object} opts - { rows, tableReg, baseData }
 *   tableReg - the table's MPE_TABLES row (needs rk_underlying and buskey)
 *   baseData - { columns, rows } of the base table, for the key lookup
 * @returns {String} the RK column name when the assignment ran, else ''
 */
function mpeAssignRetainedKey(opts) {
    const { rows, tableReg, baseData } = opts
    const rkUnderlying = tableReg && tableReg.rk_underlying
        ? String(tableReg.rk_underlying).trim()
        : ''
    const rkCol = tableReg && tableReg.buskey ? String(tableReg.buskey).trim() : ''
    if (!rkUnderlying || !rkCol || !rows || rows.length === 0) return ''

    const base = baseData || { columns: [], rows: [] }
    const submitted = Object.keys(rows[0] || {}).find(
        (k) => k.toUpperCase() === rkCol.toUpperCase()
    )
    const rkName = submitted ||
        (base.columns.find((c) => c.name.toUpperCase() === rkCol.toUpperCase()) || {}).name ||
        rkCol
    const keys = rkUnderlying.split(/\s+/)
    const keyValue = (value) => {
        if (value === null || value === undefined) return ''
        return typeof value === 'number' ? String(value) : String(value).replace(/\s+$/, '')
    }
    const businessKey = (row) => keys.map((k) => keyValue(colVal(row, k))).join('|')

    const baseByKey = new Map()
    let nextRk = 0
    for (const row of base.rows) {
        const rk = colVal(row, rkCol)
        baseByKey.set(businessKey(row), rk)
        const num = Number(rk)
        if (isFinite(num) && num > nextRk) nextRk = num
    }

    for (const row of rows) {
        Object.keys(row).forEach((k) => {
            if (k.toUpperCase() === rkCol.toUpperCase()) delete row[k]
        })
        const existing = baseByKey.get(businessKey(row))
        row[rkName] = existing === undefined || existing === null ? ++nextRk : existing
    }
    return rkName
}

/**
 * Stages a submission: writes staged.json under staging/<tableId>/, copies the
 * session audit items, and inserts the MPE_SUBMIT row. Shared by the
 * editors/stagedata (grid submit) and editors/loadfile (CSV upload) services.
 *
 * @param {Object} opts - { dataDir, stagingDir, tableId, libds, libref, dsn,
 *   message, headers, rows, weboutPath }
 * @returns {Object} the stageData object written to staged.json
 */
function stageSubmission(opts) {
    const { dataDir, stagingDir, tableId, libds, libref, dsn, message, headers, rows } = opts

    const stageFolder = nodePath.resolve(stagingDir, tableId)
    fs.mkdirSync(stageFolder, { recursive: true })

    const stageData = {
        table_id: tableId,
        libds,
        libref,
        dsn,
        submitted_by_nm: getDcUser(),
        submitted_on_dttm: Date.now() / 1000,
        headers,
        rows
    }
    fs.writeFileSync(
        nodePath.resolve(stageFolder, 'staged.json'),
        JSON.stringify(stageData, null, 2)
    )

    // keep audit items from this Stored Program run alongside the staged data
    copySessionAuditItems(stageFolder, 'submit')

    // Files the SAS loader leaves in the approval folder, so the audit pack
    // holds the same artefacts: the cleaned CSV and the uploaded file itself.
    for (const file of opts.auditFiles || []) {
        fs.writeFileSync(nodePath.resolve(stageFolder, file.name), file.content)
    }

    // Insert a record into MPE_SUBMIT (always in the control library /
    // mpelib, regardless of which library the base table lives in)
    const mpeDataDir = libDataDir('DC_JSLIB')
    const loadTableData = makeTableLoader(mpeDataDir)
    const submitTable = loadTableData('MPE_SUBMIT') || {
        name: 'MPE_SUBMIT',
        columns: [],
        rows: []
    }
    submitTable.rows.push({
        TABLE_ID: tableId,
        SUBMIT_STATUS_CD: 'SUBMITTED',
        BASE_LIB: libref,
        BASE_DS: dsn,
        SUBMITTED_BY_NM: stageData.submitted_by_nm,
        SUBMITTED_ON_DTTM: stageData.submitted_on_dttm,
        SUBMITTED_REASON_TXT: message || '',
        INPUT_OBS: rows.length,
        INPUT_VARS: headers.filter((h) => h !== '_____DELETE__THIS__RECORD_____').length,
        NUM_OF_APPROVALS_REQUIRED: 1,
        NUM_OF_APPROVALS_REMAINING: 1,
        REVIEWED_BY_NM: '',
        REVIEWED_ON_DTTM: null
    })
    const submitFile = nodePath.resolve(mpeDataDir, 'mpe_submit.json')
    fs.writeFileSync(submitFile, JSON.stringify(submitTable, null, 2))

    return stageData
}

/**
 * Copies audit items from the Stored Program's session directory (its
 * current working directory at runtime) into the staging subfolder:
 *   - log.log        -> <prefix>.log  (runtime log of the submission)
 *   - code.js        -> <prefix>_code.js (program source as executed)
 *   - reqHeaders.txt -> <prefix>_reqHeaders.txt
 */
function copySessionAuditItems(stageFolder, prefix) {
    try {
        const sessionDir = nodePath.dirname(weboutPath)
        const copies = [
            ['log.log', prefix + '.log'],
            ['code.js', prefix + '_code.js'],
            ['reqHeaders.txt', prefix + '_reqHeaders.txt']
        ]
        for (const [src, dest] of copies) {
            const srcPath = nodePath.resolve(sessionDir, src)
            if (fs.existsSync(srcPath)) {
                fs.copyFileSync(srcPath, nodePath.resolve(stageFolder, dest))
            }
        }
    } catch (err) {
        console.log('Error copying session audit items: ' + err.message)
    }
}

// ─── %mp_streamfile + ODS package (zip) ───────────────────────────────────────

const ZIP_CRC_TABLE = (() => {
    const table = new Array(256)
    for (let i = 0; i < 256; i++) {
        let c = i
        for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1)
        table[i] = c >>> 0
    }
    return table
})()

function zipCrc32(bytes) {
    let crc = -1
    for (let i = 0; i < bytes.length; i++)
        crc = (crc >>> 8) ^ ZIP_CRC_TABLE[(crc ^ bytes[i]) & 0xff]
    return (crc ^ -1) >>> 0
}

function zipUtf8(value) {
    const text = String(value)
    if (typeof TextEncoder !== 'undefined') return new TextEncoder().encode(text)
    const out = []
    for (let i = 0; i < text.length; i++) {
        const code = text.charCodeAt(i)
        if (code < 0x80) out.push(code)
        else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 0x3f))
        else out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 0x3f), 0x80 | (code & 0x3f))
    }
    return Uint8Array.from(out)
}

/**
 * Renders rows as CSV, the counterpart of %mp_ds2csv: the header row, then one
 * line per row, with values quoted only when they carry a delimiter, quote or
 * line break.
 *
 * @param {string[]} headers column names, in order
 * @param {Array<Object>} rows objects keyed by column name
 * @returns {string} CSV text with a trailing newline
 */
function toCsv(headers, rows) {
    const cell = (value) => {
        const text = value === undefined || value === null ? '' : String(value)
        return /[",\n\r]/.test(text) ? '"' + text.replace(/"/g, '""') + '"' : text
    }
    const lines = [headers.map(cell).join(',')]
    for (const row of rows) {
        lines.push(headers.map((header) => cell(row[header])).join(','))
    }
    return lines.join('\n') + '\n'
}

/**
 * Builds a zip archive in memory, entries stored rather than deflated - the JS
 * counterpart of the ODS package getauditfile.sas publishes. The zip format
 * leaves compression to the writer, so a stored archive opens anywhere.
 *
 * @param {Array<{name: string, content: string}>} entries archive entries
 * @returns {Uint8Array} the archive
 */
function mpZipPackage(entries) {
    const u16 = (n) => [n & 0xff, (n >>> 8) & 0xff]
    const u32 = (n) => [n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, (n >>> 24) & 0xff]

    const local = []
    const central = []
    let offset = 0

    for (const entry of entries) {
        const name = zipUtf8(entry.name)
        const data = zipUtf8(entry.content)
        const crc = zipCrc32(data)
        // Bit 11 marks the name as UTF-8. Without it an extractor decodes a
        // non-ASCII name as CP437, so the entry cannot be read back by name.
        const flags = name.some((byte) => byte >= 0x80) ? 0x0800 : 0

        // 1980-01-01 00:00, the earliest DOS timestamp: a stable value keeps
        // the archive byte-identical for identical inputs.
        const header = [].concat(
            u32(0x04034b50), u16(20), u16(flags), u16(0), u16(0), u16(0x21),
            u32(crc), u32(data.length), u32(data.length), u16(name.length), u16(0)
        )

        local.push(Uint8Array.from(header), name, data)
        central.push({ name, crc, size: data.length, offset, flags })
        offset += header.length + name.length + data.length
    }

    const directory = []
    let directorySize = 0
    for (const entry of central) {
        const header = [].concat(
            u32(0x02014b50), u16(20), u16(20), u16(entry.flags), u16(0), u16(0), u16(0x21),
            u32(entry.crc), u32(entry.size), u32(entry.size),
            u16(entry.name.length), u16(0), u16(0), u16(0), u16(0), u32(0),
            u32(entry.offset)
        )
        directory.push(Uint8Array.from(header), entry.name)
        directorySize += header.length + entry.name.length
    }

    const end = Uint8Array.from([].concat(
        u32(0x06054b50), u16(0), u16(0),
        u16(central.length), u16(central.length),
        u32(directorySize), u32(offset), u16(0)
    ))

    const chunks = local.concat(directory, [end])
    const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0)
    const archive = new Uint8Array(total)
    let at = 0
    for (const chunk of chunks) {
        archive.set(chunk, at)
        at += chunk.length
    }
    return archive
}

/**
 * Streams a file back as an attachment, the JS counterpart of %mp_streamfile on
 * the SASJS platform: the headers name the content type and the body is the file
 * itself. Returns false where that is not possible - the browser mock host has
 * a text-only webout - so the caller can fall back to something textual.
 *
 * @param {{filename: string, contentType: string, bytes: Uint8Array}} opts
 * @returns {boolean} whether the file was streamed
 */
function mpStreamfile(opts) {
    if (typeof Buffer === 'undefined') return false
    // A content-type header is what tells the SASjs Server to send webout.txt as
    // binary rather than a string (api/src/controllers/internal/Execution.ts).
    fs.writeFileSync(
        _SASJS_WEBOUT_HEADERS,
        `Content-type: ${opts.contentType} \n Content-Disposition: attachment;filename="${opts.filename}"`
    )
    fs.writeFileSync(weboutPath, Buffer.from(opts.bytes))
    return true
}

/**
 * Generates a DC table_id in the same format as the real backend
 * (DC<yyyymmdd-hhmmss.sss>_<rand6>_<pid4>).
 */
function makeTableId() {
    const now = new Date()
    const pad = (n, w) => String(n).padStart(w, '0')
    const dtStr =
        now.getFullYear() +
        pad(now.getMonth() + 1, 2) +
        pad(now.getDate(), 2) +
        '-' +
        pad(now.getHours(), 2) +
        pad(now.getMinutes(), 2) +
        pad(now.getSeconds(), 2) +
        '.' +
        pad(now.getMilliseconds(), 3)
    const rand6 = pad(Math.floor(Math.random() * 1000000), 6)
    const pid4 = pad(process.pid % 10000, 4)
    return `DC${dtStr}_${rand6}_${pid4}`
}

// ─── mp_filtergenerate.sas ────────────────────────────────────────────────────

/**
 * Parses a SAS literal RAW_VALUE into a JS value or array.
 *   'O''Brien'  -> "O'Brien"        (char, doubled quotes)
 *   42          -> 42               (numeric)
 *   ('A','B')   -> ["A","B"]        (IN list)
 *   (1,2,3)     -> [1,2,3]          (IN list, numeric)
 *   1 AND 5     -> [1, 5]           (BETWEEN)
 *   '20DEC2053'd-> 34322            (date / datetime / time literal)
 */
function parseFilterRawValue(raw, operator) {
    if (raw === undefined || raw === null || raw === '') return null
    const s = String(raw).trim()

    if (operator === 'BETWEEN') {
        const parts = s.split(/\s+AND\s+/i)
        return parts.map(p => parseFilterScalar(p))
    }

    if (operator === 'IN' || operator === 'NOT IN') {
        const inner = s.replace(/^\(/, '').replace(/\)$/, '')
        return inner.split(',').map(p => parseFilterScalar(p.trim()))
    }

    return parseFilterScalar(s)
}

/**
 * SAS date / datetime / time literal -> the SAS internal value, or null.
 *   34322 <- '20DEC2053'd
 *   42    <- '01JAN1960:00:00:42'dt
 *   42    <- '00:00:42't
 * The client builds these in canonical ddMMMyyyy form (HelperService
 * .sasTemporalLiteral), so only that shape needs parsing here.
 */
const TEMPORAL_LITERAL_RE = /^'(\d{2})([A-Z]{3})(\d{4})(?::(\d{2}):(\d{2}):(\d{2})(\.\d+)?)?'(dt|d|t)$/i
/* A time literal carries no date part, so it is matched separately */
const TIME_LITERAL_RE = /^'(\d{2}):(\d{2}):(\d{2})(\.\d+)?'t$/i

function parseTemporalLiteral(s) {
    const t = TIME_LITERAL_RE.exec(s)
    if (t) {
        return Number(t[1]) * 3600 + Number(t[2]) * 60 + Number(t[3]) +
            (t[4] ? Number(t[4]) : 0)
    }
    const m = TEMPORAL_LITERAL_RE.exec(s)
    if (!m) return null
    const month = MONTHS_ABBR.indexOf(m[2].toUpperCase())
    if (month < 0) return null
    const days = (Date.UTC(Number(m[3]), month, Number(m[1])) - SAS_EPOCH_MS) / 86400000
    const suffix = m[8].toLowerCase()
    if (suffix === 'd') return days
    const secs = Number(m[4]) * 3600 + Number(m[5]) * 60 + Number(m[6]) +
        (m[7] ? Number(m[7]) : 0)
    return suffix === 'dt' ? days * 86400 + secs : secs
}

function parseFilterScalar(s) {
    s = s.trim()
    const temporal = parseTemporalLiteral(s)
    if (temporal !== null) return temporal
    if (s.startsWith("'") && s.endsWith("'")) {
        return s.slice(1, -1).replace(/''/g, "'")
    }
    if (s.startsWith('"') && s.endsWith('"')) {
        return s.slice(1, -1).replace(/""/g, '"')
    }
    const num = Number(s)
    return Number.isFinite(num) && s !== '' ? num : s
}

/**
 * Compares a row value against a SAS-literal RAW_VALUE for the given operator.
 */
function compareFilterValue(val, op, rawVal, isNumCol) {
    if (val === undefined || val === null) return false

    if (op === '=' || op === 'EQ') {
        const target = parseFilterRawValue(rawVal, '=')
        return isNumCol ? Number(val) === target : String(val) === String(target)
    }
    if (op === 'NE') {
        const target = parseFilterRawValue(rawVal, 'NE')
        return isNumCol ? Number(val) !== target : String(val) !== String(target)
    }
    if (op === '<') {
        const target = parseFilterRawValue(rawVal, '<')
        return isNumCol ? Number(val) < target : String(val) < String(target)
    }
    if (op === '>') {
        const target = parseFilterRawValue(rawVal, '>')
        return isNumCol ? Number(val) > target : String(val) > String(target)
    }
    if (op === '<=') {
        const target = parseFilterRawValue(rawVal, '<=')
        return isNumCol ? Number(val) <= target : String(val) <= String(target)
    }
    if (op === '>=') {
        const target = parseFilterRawValue(rawVal, '>=')
        return isNumCol ? Number(val) >= target : String(val) >= String(target)
    }
    if (op === 'BETWEEN') {
        const parts = parseFilterRawValue(rawVal, 'BETWEEN')
        if (Array.isArray(parts) && parts.length === 2) {
            return isNumCol
                ? Number(val) >= parts[0] && Number(val) <= parts[1]
                : String(val) >= String(parts[0]) && String(val) <= String(parts[1])
        }
        return true
    }
    if (op === 'IN') {
        const list = parseFilterRawValue(rawVal, 'IN')
        if (Array.isArray(list)) {
            return isNumCol
                ? list.includes(Number(val))
                : list.map(String).includes(String(val))
        }
        return false
    }
    if (op === 'NOT IN') {
        const list = parseFilterRawValue(rawVal, 'NOT IN')
        if (Array.isArray(list)) {
            return isNumCol
                ? !list.includes(Number(val))
                : !list.map(String).includes(String(val))
        }
        return true
    }
    if (op === 'CONTAINS') {
        const target = parseFilterRawValue(rawVal, 'CONTAINS')
        return String(val).toLowerCase().includes(String(target).toLowerCase())
    }
    // Unknown operator: don't filter
    return true
}

/**
 * Renders the WHERE clause text that mp_filtergenerate.sas writes to the
 * fileref, joined the way getdata.sas/viewdata.sas collapse the lines into
 * FILTER_TEXT (catx(' ')). Single clause: (( A = 1 )). Empty query: 1=1.
 */
function mpFilterGenerateText(fqRows) {
    const sorted = [...fqRows].sort((a, b) =>
        Number(colVal(a, 'SUBGROUP_ID')) - Number(colVal(b, 'SUBGROUP_ID')))
    const lines = []
    let prevSg = null
    for (let i = 0; i < sorted.length; i++) {
        const r = sorted[i]
        const sg = colVal(r, 'SUBGROUP_ID')
        if (i === 0) lines.push('((')
        else if (sg !== prevSg) lines.push((colVal(r, 'GROUP_LOGIC') || 'AND') + ' (')
        else lines.push(colVal(r, 'SUBGROUP_LOGIC') || 'AND')
        lines.push(
            colVal(r, 'VARIABLE_NM') + ' ' +
            colVal(r, 'OPERATOR_NM') + ' ' +
            colVal(r, 'RAW_VALUE'))
        const nextSg = i + 1 < sorted.length ? colVal(sorted[i + 1], 'SUBGROUP_ID') : null
        if (nextSg !== sg) lines.push(i === sorted.length - 1 ? '))' : ')')
        prevSg = sg
    }
    return lines.map(l => String(l).trim()).filter(l => l !== '').join(' ')
}

/**
 * Builds a JS filter predicate from FILTERQUERY rows, mirroring the SAS WHERE
 * clause that mp_filtergenerate.sas would produce.
 *
 * Groups rows by SUBGROUP_ID; each subgroup is AND-ed within (SUBGROUP_LOGIC),
 * then the subgroups are combined by GROUP_LOGIC (AND/OR).
 *
 * @param {Array<Object>} fqRows - Filter query rows (GROUP_LOGIC, SUBGROUP_LOGIC,
 *   SUBGROUP_ID, VARIABLE_NM, OPERATOR_NM, RAW_VALUE)
 * @param {Array<Object>} columns - Column metadata [{ name, type, format, ... }]
 *   used to determine numeric vs char comparison
 * @returns {function(Object): boolean} predicate function
 */
function mpFilterGenerate(fqRows, columns) {
    if (!fqRows || fqRows.length === 0) return () => true

    // Build a column lookup: UPPER(name) -> { type, ddtype }
    const colMap = {}
    if (columns) {
        for (const c of columns) {
            const ddtype = getDdType(c)
            colMap[c.name.toUpperCase()] = { type: c.type === 'N' ? 'num' : 'char', ddtype }
        }
    }

    // Group rows by SUBGROUP_ID (matching: proc sort; by SUBGROUP_ID)
    const subgroups = {}
    for (const r of fqRows) {
        const sgId = colVal(r, 'SUBGROUP_ID') || 0
        if (!subgroups[sgId]) subgroups[sgId] = []
        subgroups[sgId].push(r)
    }

    const subgroupIds = Object.keys(subgroups).sort((a, b) => Number(a) - Number(b))

    return function matches(row) {
        let result = true
        let firstGroup = true

        for (const sgId of subgroupIds) {
            const rows = subgroups[sgId]
            let sgResult = true

            for (const r of rows) {
                const varNm = (colVal(r, 'VARIABLE_NM') || '').toString()
                const val = colVal(row, varNm)
                const op = (colVal(r, 'OPERATOR_NM') || '').toString().toUpperCase()
                const rawVal = colVal(r, 'RAW_VALUE')
                const colInfo = colMap[varNm.toUpperCase()]
                const isNumCol = colInfo && colInfo.type === 'num'

                if (!compareFilterValue(val, op, rawVal, isNumCol)) sgResult = false
            }

            const groupLogic = (colVal(rows[0], 'GROUP_LOGIC') || 'AND').toString().toUpperCase()
            if (firstGroup) {
                result = sgResult
                firstGroup = false
            } else if (groupLogic === 'OR') {
                result = result || sgResult
            } else {
                result = result && sgResult
            }
        }

        return result
    }
}

// ─── mpe_filtermaster.sas ────────────────────────────────────────────────────

/**
 * Retrieves and applies a stored filter, mirroring mpe_filtermaster.sas.
 *
 * If filter_rk > 0, looks up the FILTER_HASH from mpe_filteranytable, then
 * fetches the filter rows from mpe_filtersource (sorted by filter_line),
 * dropping filter_hash/filter_line/processed_dttm to produce the query dataset.
 * Then calls mpFilterGenerate to build the predicate.
 *
 * Also applies the current-record filter for EDIT/DLOAD mode (tx_from/tx_to)
 * when the table is TXTEMPORAL/BITEMPORAL — matching the SAS macro.
 *
 * @param {Object} opts
 * @param {string} opts.mode - EDIT, VIEW, DLOAD, ULOAD
 * @param {string} opts.libds - Target table (libref.dataset), uppercased
 * @param {number} opts.filterRk - The filter_rk (-1 if none)
 * @param {string} opts.dataDir - Data directory for the DC control library
 * @param {Array<Object>} [opts.columns] - Column metadata for the target table
 *   (needed by mpFilterGenerate for numeric/char comparison)
 * @param {Object} [opts.tableReg] - MPE_TABLES row for the target table
 *   (needed for temporal current-record filter)
 * @returns {{ query: Array<Object>, predicate: function(Object): boolean }}
 */
function mpeFilterMaster(opts) {
    const { mode, libds, filterRk, dataDir, columns, tableReg } = opts

    const loadTableData = makeTableLoader(dataDir)
    let query = []

    // ── Deal with FILTER_RK first ──────────────────────────────────────
    // data _null_; set &dclib..mpe_filteranytable(where=(filter_rk=&filter_rk))
    //   call symputx('filter_hash',filter_hash,'l')
    if (filterRk && filterRk > 0) {
        const summaryTable = loadTableData('MPE_FILTERANYTABLE')
        if (summaryTable && summaryTable.rows) {
            const summaryRow = summaryTable.rows.find(r =>
                Number(colVal(r, 'FILTER_RK')) === Number(filterRk)
            )
            if (summaryRow) {
                const filterHash = colVal(summaryRow, 'FILTER_HASH')

                // proc sort data=&dclib..mpe_filtersource(where=(filter_hash=...))
                //   out=&outds(drop=filter_hash filter_line processed_dttm); by filter_line
                const detailTable = loadTableData('MPE_FILTERSOURCE')
                if (detailTable && detailTable.rows) {
                    query = detailTable.rows
                        .filter(r => colVal(r, 'FILTER_HASH') === filterHash)
                        .sort((a, b) => Number(colVal(a, 'FILTER_LINE')) - Number(colVal(b, 'FILTER_LINE')))
                        .map(r => ({
                            GROUP_LOGIC: colVal(r, 'GROUP_LOGIC'),
                            SUBGROUP_LOGIC: colVal(r, 'SUBGROUP_LOGIC'),
                            SUBGROUP_ID: Number(colVal(r, 'SUBGROUP_ID')),
                            VARIABLE_NM: colVal(r, 'VARIABLE_NM'),
                            OPERATOR_NM: colVal(r, 'OPERATOR_NM'),
                            RAW_VALUE: colVal(r, 'RAW_VALUE')
                        }))
                }
            }
        }
    }

    // ── Build the predicate from the query (mp_filtergenerate) ────────
    let predicate = mpFilterGenerate(query, columns)
    let filterText = mpFilterGenerateText(query)
    // data _null_ (getdata.sas/viewdata.sas): if cats(filter_text)='1=1'
    // then filter_text=''  — an empty filter hides the client info bar
    if (filterText === '1=1') filterText = ''

    // ── Current-record filter for EDIT/DLOAD mode ──────────────────────
    // Mirrors: if &varfrom ne 0 and (&mode=EDIT or &mode=DLOAD)
    //   ("&datetime"dt < varto)
    if (tableReg && (mode === 'EDIT' || mode === 'DLOAD')) {
        const varFrom = tableReg.var_txfrom
        const varTo = tableReg.var_txto
        if (varFrom && varTo) {
            // Check if the validity vars are mentioned in the query
            const hasValidityVar = query.some(r => {
                const vn = (colVal(r, 'VARIABLE_NM') || '').toUpperCase()
                return vn === varFrom.toUpperCase() || vn === varTo.toUpperCase()
            })
            if (!hasValidityVar) {
                const now = Date.now() / 1000
                const inner = predicate
                predicate = (row) => {
                    if (!inner(row)) return false
                    // (&dc_dttmtfmt. lt varto) — current record has tx_to > now
                    const txTo = colVal(row, varTo)
                    return txTo === undefined || txTo === null || Number(txTo) > now
                }
            }
        }
    }

    return { query, predicate, filterText }
}

// ─── Column value lookup / table writer ──────────────────────────────────────

/**
 * Gets a value from a row object by column name, case-insensitive.
 * Centralises the lookup pattern used throughout the mock services.
 */
function colVal(row, name) {
    if (!row || !name) return undefined
    if (row[name] !== undefined) return row[name]
    const lc = name.toLowerCase()
    if (row[lc] !== undefined) return row[lc]
    const key = Object.keys(row).find(k => k.toLowerCase() === lc)
    return key ? row[key] : undefined
}

/**
 * Saves a table's JSON data to the data directory.
 * Complement to makeTableLoader().
 */
function saveTableData(dataDir, tableName, tableData) {
    const file = nodePath.resolve(dataDir, tableName.toLowerCase() + '.json')
    fs.writeFileSync(file, JSON.stringify(tableData, null, 2))
}

// ─── mp_hashdataset.sas ───────────────────────────────────────────────────────

/**
 * Computes an MD5 hash of a query dataset, mirroring mp_hashdataset.sas.
 *
 * The hash is chained row-to-row: each row's hash includes the previous row's
 * hash (prevkey), and the salt seeds the initial prevkey.  Char and num
 * variables are hashed separately — matching mf_getvarlist(typefilter=C/N) —
 * and prevkey is appended to the char var list (cvars = charVars & prevkeyvar).
 *
 * Each value is inner-md5'd (put(md5(trim(var)),$hex32.)), the results are
 * concatenated (!! operator), and the outer md5 produces the row hash.
 *
 * @param {Array<Object>} queryds - Array of row objects (e.g. filterquery rows)
 * @param {string} salt - The salt value (typically the libds, uppercased)
 * @param {string[]} charVars - Char variable names in column order
 * @param {string[]} numVars - Numeric variable names in column order
 * @returns {string} 32-char uppercase hex hash ($hex32. representation)
 */
function mpHashDataset(queryds, salt, charVars, numVars) {
    const crypto = require('crypto')
    const md5hex = (s) => crypto.createHash('md5').update(String(s)).digest('hex').toUpperCase()

    // Empty dataset: hash is just md5(salt), matching the NLOBS=0 branch
    if (!queryds || queryds.length === 0) {
        return md5hex(salt)
    }

    // Chain seed: prevkey = md5(salt)
    let prevkey = md5hex(salt)

    for (const row of queryds) {
        const parts = []

        // Char vars (matching cvars=%mf_getvarlist(typefilter=C) &prevkeyvar)
        for (const v of charVars) {
            parts.push(md5hex(String(colVal(row, v) ?? '').trim()))
        }
        // prevkey is included as a char var
        parts.push(md5hex(prevkey.trim()))

        // Num vars (matching nvars=%mf_getvarlist(typefilter=N))
        // SAS: put(md5(trim(put(ifn(missing(var),var,var*1),binary64.))),$hex32.)
        for (const v of numVars) {
            const val = colVal(row, v)
            const num = Number(val)
            parts.push(md5hex(String(Number.isFinite(num) ? num : '').trim()))
        }

        // Outer md5 of the concatenated inner hashes (!! operator)
        prevkey = md5hex(parts.join(''))
    }

    return prevkey
}

// ─── mp_retainedkey.sas ───────────────────────────────────────────────────────

/**
 * Allocates a retained key value for a new record, mirroring
 * mp_retainedkey.sas.
 *
 * Gets max(retained_key) from the base table (or 0 if empty), increments by 1
 * for the new record, and updates the maxkeytable with the new max.
 *
 * @param {Object} baseTable - The base (target) table JSON { name, columns, rows }
 * @param {string} retainedKey - Name of the RK column (e.g. 'FILTER_RK')
 * @param {Object} maxkeyTable - The maxkeyvalues table JSON (modified in place)
 * @param {string} keyTableLibds - Uppercased libref.dsn of the base table
 * @returns {number} The new retained key value
 */
function mpRetainedKey(baseTable, retainedKey, maxkeyTable, keyTableLibds) {
    // select sum(max(&retained_key),0) into: maxkey from &base_libds
    const rows = (baseTable && baseTable.rows) || []
    let maxKey = 0
    for (const row of rows) {
        const val = Number(colVal(row, retainedKey))
        if (Number.isFinite(val) && val > maxKey) maxKey = val
    }
    const newKey = maxKey + 1

    // Update maxkeytable (matching the maxkeytable update in mp_retainedkey.sas)
    if (maxkeyTable) {
        const existing = maxkeyTable.rows.find(r =>
            colVal(r, 'KEYTABLE') &&
            colVal(r, 'KEYTABLE').toString().toUpperCase() === keyTableLibds.toUpperCase()
        )
        const now = Date.now() / 1000
        if (existing) {
            existing.MAX_KEY = newKey
            existing.PROCESSED_DTTM = now
        } else {
            maxkeyTable.rows.push({
                KEYTABLE: keyTableLibds,
                KEYCOLUMN: retainedKey,
                MAX_KEY: newKey,
                PROCESSED_DTTM: now
            })
        }
    }

    return newKey
}

// ─── mp_filterstore.sas ───────────────────────────────────────────────────────

/**
 * Validates and stores a filter query, mirroring mp_filterstore.sas.
 *
 * Hashes the query (mp_hashdataset with salt=libds), checks for an existing
 * entry in the summary table, and if not found:
 *  - allocates a FILTER_RK via mp_retainedkey
 *  - inserts into filter_summary (mpe_filteranytable)
 *  - inserts into filter_detail (mpe_filtersource)
 *  - updates the maxkeytable (mpe_maxkeyvalues)
 *
 * @param {Object} opts
 * @param {string} opts.libds - The target table being filtered (libref.dataset)
 * @param {Array<Object>} opts.queryds - The filterquery rows
 * @param {string} opts.dataDir - Data directory for the control library
 * @returns {Object} { FILTER_RK, FILTER_HASH, FILTER_TABLE } (PROCESSED_DTTM
 *   dropped, matching: alter table work.result drop PROCESSED_DTTM)
 */
function mpFilterStore(opts) {
    const { libds, queryds, dataDir } = opts

    // Filterquery column types (matching the SAS dataset definition).
    // mf_getvarlist(typefilter=C) returns these in column order.
    const CHAR_VARS = ['GROUP_LOGIC', 'SUBGROUP_LOGIC', 'VARIABLE_NM', 'OPERATOR_NM', 'RAW_VALUE']
    const NUM_VARS = ['SUBGROUP_ID']

    // ── Hash the query (salt=uppercased libds, matching salt=&orig_libds) ──
    const filterHash = mpHashDataset(queryds, libds.toUpperCase(), CHAR_VARS, NUM_VARS)

    // ── Load summary table (mpe_filteranytable) ──────────────────────────
    const loadTableData = makeTableLoader(dataDir)
    const summaryTable = loadTableData('MPE_FILTERANYTABLE') || {
        name: 'MPE_FILTERANYTABLE',
        columns: [
            { name: 'FILTER_RK', type: 'N', length: 8, format: '', label: '', notnull: true },
            { name: 'FILTER_HASH', type: 'C', length: 32, format: '', label: '', notnull: true },
            { name: 'FILTER_TABLE', type: 'C', length: 41, format: '', label: '', notnull: true },
            { name: 'PROCESSED_DTTM', type: 'N', length: 8, format: 'datetime19.', label: '', notnull: true }
        ],
        rows: []
    }

    // ── Check if hash already exists (set &filter_summary; where filter_hash=) ──
    let existing = summaryTable.rows.find(r =>
        colVal(r, 'FILTER_HASH') === filterHash
    )

    if (!existing) {
        // ── Allocate FILTER_RK via mp_retainedkey ────────────────────────
        const maxkeyTable = loadTableData('MPE_MAXKEYVALUES') || {
            name: 'MPE_MAXKEYVALUES',
            columns: [
                { name: 'KEYTABLE', type: 'C', length: 41, format: '', label: '', notnull: false },
                { name: 'KEYCOLUMN', type: 'C', length: 32, format: '$32.', label: '', notnull: false },
                { name: 'MAX_KEY', type: 'N', length: 8, format: '', label: '', notnull: false },
                { name: 'PROCESSED_DTTM', type: 'N', length: 8, format: 'E8601DT26.6', label: '', notnull: false }
            ],
            rows: []
        }

        const filterRk = mpRetainedKey(
            summaryTable, 'FILTER_RK', maxkeyTable,
            dataDir.split('/').pop() === 'DC_JSLIB'
                ? 'DC_JSLIB.MPE_FILTERANYTABLE'
                : libds.split('.')[0] + '.MPE_FILTERANYTABLE'
        )

        // ── Insert into summary table (proc append base=&filter_summary) ──
        const now = Date.now() / 1000
        const newSummaryRow = {
            FILTER_RK: filterRk,
            FILTER_HASH: filterHash,
            FILTER_TABLE: libds.toUpperCase(),
            PROCESSED_DTTM: now
        }
        summaryTable.rows.push(newSummaryRow)
        saveTableData(dataDir, 'MPE_FILTERANYTABLE', summaryTable)
        saveTableData(dataDir, 'MPE_MAXKEYVALUES', maxkeyTable)

        // ── Insert into detail table (proc append base=&filter_detail) ──
        const detailTable = loadTableData('MPE_FILTERSOURCE') || {
            name: 'MPE_FILTERSOURCE',
            columns: [
                { name: 'FILTER_HASH', type: 'C', length: 32, format: '', label: '', notnull: true },
                { name: 'FILTER_LINE', type: 'N', length: 8, format: '', label: '', notnull: true },
                { name: 'GROUP_LOGIC', type: 'C', length: 3, format: '', label: '', notnull: true },
                { name: 'SUBGROUP_LOGIC', type: 'C', length: 3, format: '', label: '', notnull: true },
                { name: 'SUBGROUP_ID', type: 'N', length: 8, format: '', label: '', notnull: true },
                { name: 'VARIABLE_NM', type: 'C', length: 32, format: '', label: '', notnull: true },
                { name: 'OPERATOR_NM', type: 'C', length: 12, format: '', label: '', notnull: true },
                { name: 'RAW_VALUE', type: 'C', length: 4000, format: '', label: '', notnull: true },
                { name: 'PROCESSED_DTTM', type: 'N', length: 8, format: 'datetime19.', label: '', notnull: true }
            ],
            rows: []
        }

        let lineNum = 0
        for (const row of queryds) {
            lineNum++
            detailTable.rows.push({
                FILTER_HASH: filterHash,
                FILTER_LINE: lineNum,
                GROUP_LOGIC: colVal(row, 'GROUP_LOGIC'),
                SUBGROUP_LOGIC: colVal(row, 'SUBGROUP_LOGIC'),
                SUBGROUP_ID: Number(colVal(row, 'SUBGROUP_ID')),
                VARIABLE_NM: colVal(row, 'VARIABLE_NM'),
                OPERATOR_NM: colVal(row, 'OPERATOR_NM'),
                RAW_VALUE: colVal(row, 'RAW_VALUE'),
                PROCESSED_DTTM: now
            })
        }
        saveTableData(dataDir, 'MPE_FILTERSOURCE', detailTable)

        existing = newSummaryRow
    }

    // Return result (PROCESSED_DTTM dropped)
    return {
        FILTER_RK: colVal(existing, 'FILTER_RK'),
        FILTER_HASH: colVal(existing, 'FILTER_HASH'),
        FILTER_TABLE: colVal(existing, 'FILTER_TABLE')
    }
}

// ─── mpe_derived.sas ────────────────────────────────────────────────────────

/**
 * Compiles a SAS expression into a JS function - for the mock only.
 *
 * The real evaluation is done by SAS. This covers the subset used by the mock
 * and demo configs, so the EDIT screen can be exercised (display, filtering,
 * submission) without a SAS server:
 *
 *   - column names, numbers, quoted strings
 *   - + - * / and comparisons  (= ne < > <= >=)
 *   - AND OR, and `X IN (a, b, c)`
 *   - ifc(), cats(), catx(), upcase(), lowcase(), strip(), trim(), length(),
 *     substr(), index()
 *
 * Anything outside that subset throws when a row is derived, and the column is
 * left missing for that row - the same way a failing SAS expression would.
 */
function compileSasExpression(expr, columnNames) {
    // Tokenise first, so operators are only rewritten outside quoted strings
    // (a literal such as 'OR' must never be read as an operator).
    const tokens = []
    const s = String(expr)
    let i = 0
    while (i < s.length) {
        const c = s[i]
        if (/\s/.test(c)) { i++; continue }
        if (c === "'" || c === '"') {
            const q = c
            let j = i + 1
            let val = ''
            while (j < s.length) {
                if (s[j] === q) {
                    if (s[j + 1] === q) { val += q; j += 2; continue }
                    break
                }
                val += s[j]
                j++
            }
            tokens.push({ type: 'str', value: val })
            i = j + 1
            continue
        }
        if (/[0-9]/.test(c)) {
            let j = i
            while (j < s.length && /[0-9.]/.test(s[j])) j++
            tokens.push({ type: 'num', value: s.slice(i, j) })
            i = j
            continue
        }
        if (/[A-Za-z_]/.test(c)) {
            let j = i
            while (j < s.length && /[A-Za-z0-9_]/.test(s[j])) j++
            tokens.push({ type: 'word', value: s.slice(i, j) })
            i = j
            continue
        }
        const two = s.slice(i, i + 2)
        if (['<=', '>=', '!=', '<>'].includes(two)) {
            tokens.push({ type: 'punct', value: two })
            i += 2
            continue
        }
        // only the operators in the documented subset may reach the generated
        // function - anything else is rejected rather than executed
        if (!'+-*/(),<>='.includes(c)) {
            throw new Error('Unsupported character in derived expression: ' + c)
        }
        tokens.push({ type: 'punct', value: c })
        i++
    }

    // X IN (a, b, c)  ->  __in([a, b, c], X)
    const out = []
    for (let t = 0; t < tokens.length; t++) {
        const tok = tokens[t]
        if (
            tok.type === 'word' &&
            tok.value.toLowerCase() === 'in' &&
            out.length > 0 &&
            tokens[t + 1] &&
            tokens[t + 1].value === '('
        ) {
            let depth = 0
            let j = t + 1
            for (; j < tokens.length; j++) {
                if (tokens[j].value === '(') depth++
                else if (tokens[j].value === ')') {
                    depth--
                    if (depth === 0) break
                }
            }
            if (j < tokens.length) {
                const left = out.pop()
                const inner = tokens.slice(t + 2, j)
                out.push(
                    { type: 'word', value: '__in' },
                    { type: 'punct', value: '(' },
                    { type: 'punct', value: '[' },
                    ...inner,
                    { type: 'punct', value: ']' },
                    { type: 'punct', value: ',' },
                    left,
                    { type: 'punct', value: ')' }
                )
                t = j
                continue
            }
        }
        out.push(tok)
    }

    const OPS = {
        and: '&&', or: '||',
        eq: '==', ne: '!=', lt: '<', gt: '>', le: '<=', ge: '>='
    }
    const js = out
        .map(tok => {
            if (tok.type === 'str') return JSON.stringify(tok.value)
            if (tok.type === 'num') return tok.value
            if (tok.type === 'word') return OPS[tok.value.toLowerCase()] || tok.value
            if (tok.value === '=') return '=='
            if (tok.value === '<>') return '!='
            return tok.value
        })
        .join(' ')

    const sasTrue = v => {
        if (typeof v === 'boolean') return v
        if (typeof v === 'number') return v !== 0 && !isNaN(v)
        if (v === null || v === undefined) return false
        const str = String(v).trim()
        return str !== '' && str !== '0'
    }
    const toStr = v => {
        if (v === null || v === undefined) return ''
        if (typeof v === 'number') return isNaN(v) ? '' : String(v)
        return String(v).trim()
    }
    const fns = {
        ifc: (c, a, b) => (sasTrue(c) ? a : b),
        cats: (...args) => args.map(toStr).join(''),
        catx: (sep, ...args) => args.map(toStr).filter(v => v !== '').join(toStr(sep)),
        upcase: v => toStr(v).toUpperCase(),
        lowcase: v => toStr(v).toLowerCase(),
        strip: v => toStr(v),
        trim: v => toStr(v),
        length: v => toStr(v).length,
        substr: (v, p, n) => {
            const str = toStr(v)
            const start = Number(p) - 1
            return n === undefined
                ? str.substring(start)
                : str.substring(start, start + Number(n))
        },
        index: (v, sub) => toStr(v).indexOf(toStr(sub)) + 1,
        __in: (list, v) => list.some(item => String(item) === String(v) || Number(item) === Number(v))
    }

    const names = Object.keys(fns)
    const fn = new Function(...columnNames, ...names, 'return (' + js + ');')
    return row => fn(...columnNames.map(n => row[n]), ...names.map(n => fns[n]))
}

/**
 * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
 * variables with a compiled row function, in dependency order (ties by name),
 * the same sequence the real macro emits.
 *
 * @param {Object} opts
 * @param {string} opts.libref - target libref
 * @param {string} opts.table - target dataset
 * @param {string} opts.dataDir - data directory of the DC control library
 * @param {Array<Object>} opts.columns - physical columns of the target table
 */
function mpeDerived(opts) {
    const { libref, table, dataDir, columns } = opts
    const loadTableData = makeTableLoader(dataDir)
    const cfg = loadTableData('MPE_DERIVED')
    if (!cfg || !Array.isArray(cfg.rows) || cfg.rows.length === 0) return []

    const now = Date.now() / 1000
    const lib = String(libref || '').toUpperCase()
    const dsn = String(table || '').toUpperCase()
    const columnNames = (columns || []).map(c => c.name)

    const vars = cfg.rows
        .filter(r =>
            String(colVal(r, 'LIBREF') || '').toUpperCase() === lib &&
            String(colVal(r, 'DSN') || '').toUpperCase() === dsn &&
            Number(colVal(r, 'ACTIVE')) === 1 &&
            (!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
        )
        .map(r => {
            const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
            const nm = String(colVal(r, 'VAR_NM') || '').toUpperCase()
            return {
                name: nm,
                // every input the expression uses - columns and derived vars
                deps: String(colVal(r, 'VAR_DEPS') || '')
                    .toUpperCase()
                    .split(/\s+/)
                    .filter(Boolean),
                type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
                length: Number(colVal(r, 'VAR_LEN')) || 8,
                format: String(colVal(r, 'VAR_FMT') || ''),
                logic,
                label: String(colVal(r, 'VAR_LABEL') || '').trim() || nm + '=' + logic
            }
        })
        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)

    // dependency order: repeatedly take the variables whose derived inputs are
    // already placed, alphabetically within a round
    const byName = new Map(vars.map(v => [v.name, v]))
    const placed = []
    const done = new Set()
    while (placed.length < vars.length) {
        const ready = vars
            .filter(v => !done.has(v.name))
            .filter(v => v.deps.every(d => !byName.has(d) || done.has(d)))
            .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
        if (ready.length === 0) {
            console.log('derived variables for ' + lib + '.' + dsn + ' contain a cycle')
            return []
        }
        for (const v of ready) {
            placed.push(v)
            done.add(v.name)
        }
    }
    vars.length = 0
    vars.push.apply(vars, placed)

    // compile each against the physical columns plus the variables defined
    // earlier, so a derived variable can build on another one
    const compileCols = columnNames.slice()
    for (const d of vars) {
        try {
            // slice(): the compiled function captures the column list it was
            // built with, not the array as it grows
            d.fn = compileSasExpression(d.logic, compileCols.slice())
        } catch (err) {
            console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
            d.fn = () => (d.type === 'N' ? null : '')
        }
        compileCols.push(d.name)
    }
    return vars
}

/**
 * Adds the derived variables to every row and to the column list, so that the
 * filter (which runs afterwards) and the grid both see them.
 */
function mpeDeriveRows(tableData, derived) {
    if (!derived || derived.length === 0) return tableData
    const rows = tableData.rows.map(row => {
        const copy = Object.assign({}, row)
        for (const d of derived) {
            try {
                copy[d.name] = d.fn(copy)
            } catch (err) {
                copy[d.name] = d.type === 'N' ? null : ''
            }
        }
        return copy
    })
    const derivedCols = derived.map(d => ({
        name: d.name,
        type: d.type,
        length: d.length,
        format: d.format,
        label: d.label,
        notnull: false
    }))
    return {
        name: tableData.name,
        columns: (tableData.columns || []).concat(derivedCols),
        rows
    }
}

// ─── DC settings (services/settings.js) ──────────────────────────────────────

/**
 * Loads the DC settings written by makedata to services/settings.js
 * (the JS equivalent of the real backend's settings.sas).
 * Returns { dcLibref, dataDir, dcAdminGroup, dcpath } - with defaults when
 * makedata has not run yet.
 * Requires nodePath, fs, driveRoot and appLoc in the caller's scope.
 */
function loadDcSettings(driveRoot, appLoc) {
    const result = {
        dcLibref: 'DC_JSLIB',
        dataDir: nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
        dcAdminGroup: null,
        dcpath: null
    }
    try {
        const settingsFile = nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'settings.js')
        const settingsRaw = fs.readFileSync(settingsFile, {encoding:'utf8'}).toString()
        eval(settingsRaw)
        if (typeof dcSettings !== 'undefined') {
            if (dcSettings.dc_libref) result.dcLibref = dcSettings.dc_libref
            if (dcSettings.dc_admin_group) result.dcAdminGroup = dcSettings.dc_admin_group
            if (dcSettings.dcpath) result.dcpath = dcSettings.dcpath
            result.dataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', result.dcLibref)
        }
    } catch(err) { /* makedata hasn't run - keep defaults */ }
    return result
}

/**
 * Reads the DC-scope options out of MPE_CONFIG, mirroring the loop in the real
 * backend's generated settings.sas: only rows that are active (var_active=1)
 * and current (now < tx_to) count, and the latest tx_from wins.  Returns a map
 * of var_name -> trimmed var_value.
 *
 * An option that is absent, or switched off, is simply missing from the map so
 * the caller's own default applies - which is what the real service's
 * %symexist fallback does (a switched-off option means "use the code default",
 * not "no limit").
 *
 * Requires nodePath, fs and the caller's driveRoot/appLoc/dataDir.
 */
function loadDcConfig(driveRoot, appLoc, dataDir) {
    const out = {}
    try {
        const configFile = nodePath.resolve(
            dataDir || nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
            'mpe_config.json'
        )
        const config = JSON.parse(fs.readFileSync(configFile, {encoding:'utf8'}).toString())
        const now = Math.floor(Date.now() / 1000)
        const latest = {}
        for (const row of (config.rows || [])) {
            // colVal, not row.var_name: a row that has been through the app's
            // own edit and approve flow comes back from the adapter with
            // upper-case keys, and a direct property read would skip it - which
            // silently drops the option back to its default.
            if (colVal(row, 'VAR_SCOPE') !== 'DC') continue
            if (Number(colVal(row, 'VAR_ACTIVE')) !== 1) continue
            if (Number(colVal(row, 'TX_TO')) <= now) continue
            const name = String(colVal(row, 'VAR_NAME') || '').trim()
            if (!name) continue
            const from = Number(colVal(row, 'TX_FROM')) || 0
            if (latest[name] === undefined || from > latest[name]) {
                const val = colVal(row, 'VAR_VALUE')
                latest[name] = from
                out[name] = String(val == null ? '' : val).trim()
            }
        }
    } catch(err) { /* no MPE_CONFIG - every option falls back to its default */ }
    return out
}

/**
 * A numeric DC option out of a loadDcConfig() map, or the service's own default
 * when the option is absent, blank, or not a number.
 */
function dcConfigNumber(cfg, name, fallback) {
    const raw = cfg ? cfg[name] : undefined
    if (raw === undefined || raw === '') return fallback
    const num = Number(raw)
    return isNaN(num) ? fallback : num
}

// ─── %webout(OPEN/OBJ/CLOSE) emulation ───────────────────────────────────────

/**
 * Internal buffer for the webOut stream.
 * webOutOpen() starts it, webOutObj() appends tables, webOutClose()
 * finalises and assigns to _webout.
 */
let _webOutBuf = ''
let _webOutFirst = true

/**
 * Opens the JSON stream, writing the opening brace and SYSDATE/SYSTIME.
 * Mirrors %webout(OPEN) / mv_webout(OPEN).
 *
 * Usage:
 *   webOutOpen()
 *   webOutObj(rows, 'sasdata', { vars })
 *   webOutObj(params, 'sasparams')
 *   webOutClose()
 */
function webOutOpen() {
    const now = new Date()
    const pad = n => String(n).padStart(2, '0')
    const dateStr = now.getDate() + '' +
        ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC'][now.getMonth()] +
        String(now.getFullYear()).slice(2)
    const timeStr = pad(now.getHours()) + ':' + pad(now.getMinutes())
    _webOutBuf = '{"SYSDATE" : "' + dateStr + '"'
    _webOutBuf += ',"SYSTIME" : "' + timeStr + '"'
    _webOutFirst = false
}

/**
 * Appends a table to the JSON stream.
 * Mirrors %webout(OBJ,ds,dslabel=,showmeta=).
 *
 * @param {Array<Object>} rows - Array of row objects to send as a JSON array.
 * @param {string} label - The key name in the JSON (e.g. 'sasdata', 'sasparams').
 * @param {Object} [meta] - Optional metadata object.  If provided, it is sent
 *   as `$label` (e.g. `$sasdata`), matching the showmeta=YES output from SAS.
 *   Typically { vars: { COL1: {format,label,length,type}, ... } }.
 */
function webOutObj(rows, label, meta) {
    _webOutBuf += ',"' + label + '":' + JSON.stringify(rows)
    if (meta) {
        _webOutBuf += ',"$' + label + '":' + JSON.stringify(meta)
    }
}

/**
 * Closes the JSON stream, appends the standard SAS runtime variables
 * (SYSDATE, SYSCC, _PROGRAM, etc.), and assigns the final JSON string to
 * the `_webout` variable that the SASjs Server reads.
 * Mirrors mv_webout(CLOSE).
 */
// Current user for MPE_SUBMIT writes/reads - mock equivalent of %mf_getuser().
// SASjs Server predeclares _METAUSER for authenticated requests; the desktop
// mode (no auth) leaves it undefined.
function getDcUser() {
    return typeof _METAUSER !== 'undefined' && _METAUSER ? _METAUSER : 'sasdemo'
}

function webOutClose() {
    const now = new Date()
    const pad = n => String(n).padStart(2, '0')
    const endDttm = now.getFullYear() + '-' + pad(now.getMonth()+1) + '-' + pad(now.getDate()) +
        'T' + pad(now.getHours()) + ':' + pad(now.getMinutes()) + ':' + pad(now.getSeconds()) +
        '.' + String(now.getMilliseconds()).padStart(3,'0')

    // Derive system vars from SASjs Server runtime variables where possible
    const mfGetuser = getDcUser()
    const metaperson = (typeof _METAPERSON !== 'undefined') ? _METAPERSON : 'sasdemo'
    const sysprocmode = (typeof SASJSPROCESSMODE !== 'undefined') ? SASJSPROCESSMODE : 'Stored Program'

    _webOutBuf += ',"_DEBUG" : ""'
    _webOutBuf += ',"_METAUSER": "' + mfGetuser + '@SAS"'
    _webOutBuf += ',"_METAPERSON": "' + metaperson + '"'
    _webOutBuf += ',"_PROGRAM" : "' + (typeof _program !== 'undefined' ? _program : '') + '"'
    _webOutBuf += ',"AUTOEXEC" : ""'
    _webOutBuf += ',"MF_GETUSER" : "' + mfGetuser + '"'
    _webOutBuf += ',"SYS_JES_JOB_URI" : " "'
    _webOutBuf += ',"SYSCC" : "0"'
    _webOutBuf += ',"SYSENCODING" : "utf-8"'
    _webOutBuf += ',"SYSERRORTEXT" : ""'
    _webOutBuf += ',"SYSHOSTNAME" : "sasjs-server"'
    _webOutBuf += ',"SYSPROCESSID" : "0"'
    _webOutBuf += ',"SYSPROCESSMODE" : "' + sysprocmode + '"'
    _webOutBuf += ',"SYSPROCESSNAME" : ""'
    _webOutBuf += ',"SYSJOBID" : "0"'
    _webOutBuf += ',"SYSSCPL" : "LINUX"'
    _webOutBuf += ',"SYSSITE" : "123"'
    _webOutBuf += ',"SYSUSERID" : "sasdemo"'
    _webOutBuf += ',"SYSVLONG" : "9.04.01M7P080520"'
    _webOutBuf += ',"SYSWARNINGTEXT" : ""'
    _webOutBuf += ',"END_DTTM" : "' + endDttm + '"'
    _webOutBuf += ',"MEMSIZE" : "0KB"'
    _webOutBuf += '}'
    _webout = _webOutBuf
}

/**
 * Convenience: builds a complete response in one call.
 * Opens, adds each table, closes.
 *
 * @param {Object} tables - { label1: { rows: [...], meta: {...} }, label2: { rows: [...] } }
 */
function webOutSend(tables) {
    webOutOpen()
    for (const [label, entry] of Object.entries(tables)) {
        webOutObj(entry.rows, label, entry.meta)
    }
    webOutClose()
}

// ─── %mpeinit() emulation ────────────────────────────────────────────────────

/**
 * The `_debug` values that mean "debug on", mirroring the test in mpeinit.sas.
 *
 * 131 is what the adapter sends - SASjsApiClient.executeJob() puts it on every
 * server-mode request, and on the multipart form when the payload goes out as
 * FormData.  128 is what arrives on the Viya WEB JES path when runAsTask is
 * enabled, and 2477 / 'fields,log,trace' are the SASjs debug values.
 */
const MPEINIT_DEBUG_VALUES = ['2477', 'fields,log,trace', '131', '128']

/* Mirrors the &mpeinit=1 guard in the SAS macro: dump once per execution. */
let _mpeinitDone = false

/**
 * Dumps the browser_url_vars and browser_info input tables, mirroring the
 * debug block of %mpeinit (sas/sasjs/macros/mpeinit.sas).
 *
 * The SAS macro reads work.browser_url_vars (name/value pairs taken from the
 * query string) and work.browser_info (the browser fingerprint).  Both arrive
 * here as input tables - see fetchRaw() - and are dumped with console.log,
 * which SASjs Server returns in the request's log (after its
 * SASJS_LOGS_SEPARATOR marker), so the dump shows up in Data Controller's
 * SAS Log tab exactly as the SAS version does.
 *
 * Called at the bottom of this file, so the services that eval() it dump the
 * tables at startup - the same place %mpeinit() runs in the SAS services.
 * That covers every diagnostics-carrying service (startup, editors, auditors);
 * the mocks that do not load this file (lineage, metanav, usernav, a few
 * public ones) have no equivalent startup step in the SAS app either.
 *
 * Difference from the SAS version: the header line is printed once per table,
 * where SAS repeats it for every row (its PUTLOG sits inside the SET loop).
 */
function mpeinit() {
    if (_mpeinitDone) return

    const debugValue = typeof _debug === 'undefined' ? '' : _debug
    if (MPEINIT_DEBUG_VALUES.indexOf(String(debugValue)) < 0) return

    _mpeinitDone = true

    const urlVars = fetchRaw('browser_url_vars')
    if (urlVars === null) {
        console.log('NOTE: mpeinit: no work.browser_url_vars on this request')
    } else {
        console.log('NOTE: mpeinit: work.browser_url_vars:')
        parseCsv(urlVars).forEach(row => {
            console.log(
                'name=' + colVal(row, 'name') + ' value=' + colVal(row, 'value')
            )
        })
    }

    const browserInfo = fetchRaw('browser_info')
    if (browserInfo === null) {
        console.log('NOTE: mpeinit: no work.browser_info on this request')
    } else {
        console.log('NOTE: mpeinit: work.browser_info:')
        parseCsv(browserInfo).forEach(row => {
            console.log(Object.keys(row).map(col => col + '=' + row[col]).join(' '))
        })
    }
}

/* Every service that eval()s this file runs %mpeinit() at startup, the same
   place the SAS services call it. */
mpeinit()