435 lines
19 KiB
JavaScript
435 lines
19 KiB
JavaScript
const nodePath = require('path')
|
|
|
|
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
|
|
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
|
|
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
|
|
const dcLibref = 'DC_JSLIB'
|
|
|
|
// Load shared DC mock utilities
|
|
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
|
|
|
|
// Mirrors SAS cats() for the values a dropdown source can hold: a special
|
|
// missing stored in its period form (".a") becomes the bare uppercase letter
|
|
// ("A") that cats() returns, so the mock's dropdown list matches the real one.
|
|
function cats(value) {
|
|
if (typeof value === 'string') {
|
|
const m = /^\.(_|[a-z])$/i.exec(value.trim())
|
|
if (m) return m[1].toUpperCase()
|
|
return value.trim()
|
|
}
|
|
return String(value)
|
|
}
|
|
|
|
/**
|
|
* SAS sort position of a special missing, or null when the value is ordinary.
|
|
* `._` sorts first, then `.a` to `.z` - all of them below every number.
|
|
*/
|
|
function missingRank(value) {
|
|
const m = /^\.(_|[a-z])$/i.exec(String(value).trim())
|
|
if (!m) return null
|
|
return m[1] === '_' ? 0 : m[1].toUpperCase().charCodeAt(0) - 64
|
|
}
|
|
|
|
// ─── Parse input ──────────────────────────────────────────────────────────────
|
|
|
|
const _sctRow = fetchTable('SASControlTable')[0] || {}
|
|
let requestedLibref = dcLibref
|
|
let requestedTable = ''
|
|
if (_sctRow.LIBDS) {
|
|
const parts = _sctRow.LIBDS.split('.')
|
|
if (parts.length >= 2) { requestedLibref = parts[0]; requestedTable = parts[1] }
|
|
}
|
|
if (!requestedTable) {
|
|
if (_sctRow.DSN) requestedTable = _sctRow.DSN.trim()
|
|
if (_sctRow.LIBREF) requestedLibref = _sctRow.LIBREF.trim()
|
|
}
|
|
|
|
// FILTER_RK (matching getdata.sas: if filter_rk le 0 then filter_rk=-1)
|
|
const filterRk = Number(_sctRow.FILTER_RK) > 0 ? Number(_sctRow.FILTER_RK) : -1
|
|
|
|
// ─── Load table data and schema ──────────────────────────────────────────────
|
|
// A table can live in any libref (DC_JSLIB for control tables, TESTDATA for
|
|
// demo user tables), so resolve the data directory from the libref.
|
|
|
|
const dataDir = libDataDir(requestedLibref)
|
|
const loadTableData = makeTableLoader(dataDir)
|
|
|
|
let tableData = loadTableData(requestedTable)
|
|
if (!tableData) {
|
|
// Fall back to searching every libref if the table isn't in the requested
|
|
// libref (e.g. a LIBDS that doesn't match the actual data folder)
|
|
const found = loadTableAnyLib(requestedTable)
|
|
if (found) {
|
|
tableData = found.data
|
|
requestedLibref = found.libref
|
|
}
|
|
}
|
|
|
|
// Load table registration from MPE_TABLES (needed for PK detection, temporal
|
|
// column exclusion, and sasparams). MPE_TABLES always lives in the control
|
|
// library (DC_JSLIB / mpelib), not in user data libraries.
|
|
const mpeDataDir = libDataDir(dcLibref)
|
|
const mpeLoadTableData = makeTableLoader(mpeDataDir)
|
|
|
|
let tableReg = null
|
|
const mpeTablesData = mpeLoadTableData('MPE_TABLES')
|
|
if (mpeTablesData && mpeTablesData.rows) {
|
|
tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
|
|
}
|
|
|
|
// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
|
|
// Applied before the filter, so a filter can reference them. The derived
|
|
// columns are added to the schema too, which is what puts them in COLS and in
|
|
// the grid (read only, since the mock marks every column READ).
|
|
|
|
const derived = mpeDerived({
|
|
libref: requestedLibref,
|
|
table: requestedTable,
|
|
dataDir: mpeDataDir,
|
|
columns: tableData ? tableData.columns : []
|
|
})
|
|
if (tableData && derived.length > 0) {
|
|
tableData = mpeDeriveRows(tableData, derived)
|
|
}
|
|
|
|
// ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
|
|
// Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
|
|
// NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
|
|
|
|
const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
|
|
let dqrules = []
|
|
let dqdata = []
|
|
if (validationsData && validationsData.rows) {
|
|
const pkCols = new Set()
|
|
if (tableReg && tableReg.buskey) {
|
|
tableReg.buskey.trim().split(/\s+/).forEach(c => pkCols.add(c.toUpperCase()))
|
|
}
|
|
dqrules = validationsData.rows
|
|
.filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
|
|
.map(r => ({
|
|
BASE_COL: r.base_col,
|
|
RULE_TYPE: r.rule_type,
|
|
RULE_VALUE: r.rule_value,
|
|
X: pkCols.has(r.base_col.toUpperCase()) ? 0 : 1
|
|
}))
|
|
}
|
|
|
|
// ─── Apply stored filter (mpe_filtermaster) ──────────────────────────────────
|
|
// Mirrors getdata.sas: %mpe_filtermaster(EDIT,&orig_libds,filter_rk=&filter_rk)
|
|
|
|
const mpeDataDir2 = libDataDir(dcLibref)
|
|
const {
|
|
query: filterQuery,
|
|
predicate: filterPredicate,
|
|
filterText: filterText
|
|
} = mpeFilterMaster({
|
|
mode: 'EDIT',
|
|
libds: (requestedLibref + '.' + requestedTable).toUpperCase(),
|
|
filterRk: filterRk,
|
|
dataDir: mpeDataDir2,
|
|
columns: tableData ? tableData.columns : [],
|
|
tableReg: tableReg
|
|
})
|
|
|
|
// Synthesise NOTNULL rules from the table schema, the same way getdata.sas
|
|
// merges dictionary.columns (notnull='yes') into dqrules.
|
|
if (tableData && tableData.columns) {
|
|
const existingNotNull = new Set(
|
|
dqrules.filter(r => r.RULE_TYPE === 'NOTNULL').map(r => r.BASE_COL.toUpperCase())
|
|
)
|
|
for (const rule of notNullRules(tableData.columns)) {
|
|
if (!existingNotNull.has(rule.BASE_COL)) dqrules.push(rule)
|
|
}
|
|
}
|
|
|
|
// ─── DQ dropdown data (from MPE_SELECTBOX + SOFTSELECT from table data) ──────
|
|
// MPE_SELECTBOX lives in the control library.
|
|
|
|
const selectboxData = mpeLoadTableData('MPE_SELECTBOX')
|
|
if (selectboxData && selectboxData.rows) {
|
|
dqdata = selectboxData.rows
|
|
.filter(r => r.select_lib === requestedLibref && r.select_ds === requestedTable)
|
|
.map(r => ({
|
|
BASE_COL: r.base_column,
|
|
RULE_VALUE: r.base_column,
|
|
RULE_DATA: r.selectbox_value,
|
|
SELECTBOX_ORDER: r.selectbox_order
|
|
}))
|
|
}
|
|
|
|
// For SOFTSELECT/HARDSELECT rules that reference the table's own data,
|
|
// generate dropdown values from the actual table rows
|
|
if (tableData && tableData.rows && tableData.columns) {
|
|
const selectRules = dqrules.filter(r =>
|
|
(r.RULE_TYPE === 'SOFTSELECT' || r.RULE_TYPE === 'HARDSELECT') && r.RULE_VALUE.includes('.'))
|
|
for (const rule of selectRules) {
|
|
const parts = rule.RULE_VALUE.split('.')
|
|
const colName = parts[parts.length - 1]
|
|
const lcName = colName.toLowerCase()
|
|
const seen = new Set()
|
|
const values = []
|
|
for (const row of tableData.rows) {
|
|
let val = row[colName]
|
|
if (val === undefined) val = row[lcName]
|
|
if (val === undefined) {
|
|
const key = Object.keys(row).find(k => k.toLowerCase() === lcName)
|
|
if (key) val = row[key]
|
|
}
|
|
if (val !== undefined && val !== null) {
|
|
const strVal = cats(val)
|
|
if (!seen.has(strVal)) {
|
|
seen.add(strVal)
|
|
values.push({ raw: val, str: strVal })
|
|
}
|
|
}
|
|
}
|
|
// getdata.sas orders the source by the column itself, and a special
|
|
// missing sorts below every number - so the list reads `._`, `.a`-`.z`,
|
|
// then the numbers ascending.
|
|
values.sort((a, b) => {
|
|
const ra = missingRank(a.raw)
|
|
const rb = missingRank(b.raw)
|
|
if (ra !== null && rb !== null) return ra - rb
|
|
if (ra !== null) return -1
|
|
if (rb !== null) return 1
|
|
const na = Number(a.str)
|
|
const nb = Number(b.str)
|
|
if (!isNaN(na) && !isNaN(nb)) return na - nb
|
|
return a.str < b.str ? -1 : a.str > b.str ? 1 : 0
|
|
})
|
|
values.forEach((v, i) => {
|
|
dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: v.str, SELECTBOX_ORDER: i + 1 })
|
|
})
|
|
}
|
|
}
|
|
|
|
// ─── Build response ──────────────────────────────────────────────────────────
|
|
|
|
let response
|
|
// Set when the row or cell limit cut the selection down - mirrors the
|
|
// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.
|
|
|
|
if (tableData && tableData.columns && tableData.rows) {
|
|
// Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
|
|
let excludeCols = new Set()
|
|
if (tableReg && (tableReg.loadtype === 'TXTEMPORAL' || tableReg.loadtype === 'BITEMPORAL')) {
|
|
if (tableReg.var_txfrom) excludeCols.add(tableReg.var_txfrom.toUpperCase())
|
|
if (tableReg.var_txto) excludeCols.add(tableReg.var_txto.toUpperCase())
|
|
}
|
|
if (tableReg && tableReg.loadtype === 'BITEMPORAL') {
|
|
if (tableReg.var_busfrom) excludeCols.add(tableReg.var_busfrom.toUpperCase())
|
|
if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
|
|
}
|
|
|
|
let visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
|
|
let visibleRows = tableData.rows
|
|
.filter(r => filterPredicate(r))
|
|
.map(r => {
|
|
const copy = {}
|
|
for (const key of Object.keys(r)) {
|
|
if (!excludeCols.has(key.toUpperCase())) copy[key] = r[key]
|
|
}
|
|
return copy
|
|
})
|
|
|
|
// ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
|
|
// Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
|
|
// been applied and the rows sorted, with the data in work.OUT, and may
|
|
// replace it. The mock eval()s the hook's .js counterpart in this scope, so
|
|
// the hook may reassign visibleRows / visibleColumns - e.g. to show the live
|
|
// rows of another table behind an empty mirror.
|
|
if (tableReg && tableReg.pre_edit_hook) {
|
|
const hookSrc = mockHookSource(tableReg.pre_edit_hook)
|
|
if (hookSrc) {
|
|
try {
|
|
eval(hookSrc)
|
|
} catch (err) {
|
|
console.log('Error running pre_edit_hook ' + tableReg.pre_edit_hook + ': ' + err.message)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Row and cell limits ─────────────────────────────────────────────────
|
|
// Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
|
|
// cells (rows x columns), whichever is reached first - the selection is
|
|
// cut to min(rows, cells/columns) and flagged in sasparams.
|
|
// Both options are read from MPE_CONFIG, the way the real service picks
|
|
// them up from the settings program it generates - so editing the option
|
|
// in DC changes what the mock serves, and an option that is absent (or
|
|
// switched off, var_active=0) falls back to the shipped default. The
|
|
// check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
|
|
// does - a hook may replace the data with a larger table.
|
|
// MPE_CONFIG lives in the control library, not in the table's own library,
|
|
// so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
|
|
// uses. Passing the requested table's dataDir made the read fail for every
|
|
// non-control libref, and the catch then fell back to the shipped defaults.
|
|
const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
|
|
const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
|
|
const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
|
|
// getdata.sas counts the columns of work.out, i.e. the table's own
|
|
// columns - the delete column is client-owned and not part of it.
|
|
const editColCount = visibleColumns.length
|
|
let maxRows = maxObsWebEdit
|
|
let limitVar = 'DC_MAXOBS_WEBEDIT'
|
|
if (editColCount > 0) {
|
|
const cellRows = Math.floor(maxCellsWebEdit / editColCount)
|
|
if (cellRows < maxRows) {
|
|
maxRows = cellRows
|
|
limitVar = 'DC_MAXCELLS_WEBEDIT'
|
|
}
|
|
}
|
|
if (maxRows < 1) maxRows = 1
|
|
let truncated = 0
|
|
let truncMsg = ''
|
|
if (visibleRows.length > maxRows) {
|
|
truncated = 1
|
|
// Same wording as the macro getdata.sas sets, so the spec can assert
|
|
// on the text the real service would send.
|
|
truncMsg =
|
|
'Only the first ' +
|
|
maxRows +
|
|
' rows are loaded - the ' +
|
|
limitVar +
|
|
' limit has been reached. Filter the table to work with the rest.'
|
|
visibleRows = visibleRows.slice(0, maxRows)
|
|
}
|
|
|
|
// Augment rows: add delete flag, normalise keys, convert temporal values to ISO
|
|
const colLookup = {}
|
|
for (const col of visibleColumns) {
|
|
colLookup[col.name.toLowerCase()] = { name: col.name, ddtype: getDdType(col) }
|
|
}
|
|
const augmentedRows = visibleRows.map(r => {
|
|
const normalised = {}
|
|
for (const key of Object.keys(r)) {
|
|
const info = colLookup[key.toLowerCase()]
|
|
const targetKey = info ? info.name : key
|
|
normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
|
|
}
|
|
return { ...normalised }
|
|
})
|
|
|
|
// $sasdata vars: temporal columns are char ($200.), others keep their SAS type
|
|
const vars = {}
|
|
for (const col of visibleColumns) {
|
|
vars[col.name] = sasVarsEntry(col)
|
|
}
|
|
|
|
const colHeaders = visibleColumns.map(c => c.name)
|
|
|
|
// Determine DTVARS, DTTMVARS, TMVARS from column formats
|
|
let dtVars = [], dttmVars = [], tmVars = []
|
|
for (const col of visibleColumns) {
|
|
const ddtype = getDdType(col)
|
|
if (ddtype === 'DATETIME') dttmVars.push(col.name)
|
|
else if (ddtype === 'DATE') dtVars.push(col.name)
|
|
else if (ddtype === 'TIME') tmVars.push(col.name)
|
|
}
|
|
|
|
// Build cols array. Every column of the table, as getdata.sas emits them:
|
|
// the temporal columns are dropped from the data (the rows below) but not
|
|
// from the cols payload, so a filter can still be built on them. mpGetcols
|
|
// is the JS counterpart of %mp_getcols.sas - the macro getdata.sas itself
|
|
// calls to type these columns - so the editor sees the same DDTYPE the
|
|
// service would give it.
|
|
const cols = mpGetcols(tableData.columns).map((c, i) => {
|
|
const raw = tableData.columns[i]
|
|
const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
|
|
let coltype
|
|
if (ddtype === 'DATE') coltype = `{"data":"${raw.name}","type":"date"}`
|
|
else if (ddtype === 'DATETIME') coltype = `{"data":"${raw.name}","type":"datetime"}`
|
|
else if (ddtype === 'TIME') coltype = `{"data":"${raw.name}","type":"time"}`
|
|
else if (ddtype === 'N') coltype = `{"data":"${raw.name}","type":"numeric","format":"0"}`
|
|
else coltype = `{"data":"${raw.name}"}`
|
|
return {
|
|
NAME: c.NAME,
|
|
LABEL: c.LABEL,
|
|
FMTNAME: c.FMTNAME || ' ',
|
|
DDTYPE: ddtype,
|
|
CLS_RULE: 'READ',
|
|
MEMLABEL: ' ',
|
|
DESC: ' ',
|
|
LONGDESC: ' ',
|
|
COLTYPE: coltype
|
|
}
|
|
}).sort((a, b) => a.NAME.localeCompare(b.NAME))
|
|
|
|
// Build maxvarlengths
|
|
const maxvarlengths = [
|
|
...visibleColumns.map(col => {
|
|
let maxLen = 0
|
|
const lcName = col.name.toLowerCase()
|
|
for (const row of augmentedRows) {
|
|
let val = row[col.name]
|
|
if (val === undefined) val = row[lcName]
|
|
if (val !== undefined && val !== null) {
|
|
const len = String(val).length
|
|
if (len > maxLen) maxLen = len
|
|
}
|
|
}
|
|
return { NAME: col.name.toLowerCase(), MAXLEN: maxLen }
|
|
})
|
|
]
|
|
|
|
response = {
|
|
approvers: [{ PERSONNAME: 'sasdemo', EMAIL: 'sasdemo', USERID: 'sasdemo' }],
|
|
cols,
|
|
dqdata, dqrules,
|
|
dsmeta: [
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: String(tableData.rows.length) },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Variables', VALUE: String(tableData.columns.length) },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Engine', VALUE: 'V9' }
|
|
],
|
|
maxvarlengths,
|
|
query: filterQuery,
|
|
sasdata: augmentedRows,
|
|
$sasdata: { vars },
|
|
sasparams: [{
|
|
COLHEADERS: colHeaders.join(','),
|
|
FILTER_TEXT: filterText === '' ? ' ' : filterText,
|
|
PKCNT: tableReg && tableReg.buskey ? tableReg.buskey.trim().split(/\s+/).length : 0,
|
|
PK: tableReg ? tableReg.buskey || '' : '',
|
|
DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
|
|
DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
|
|
TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',
|
|
LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
|
|
RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
|
|
CLS_FLAG: 0,
|
|
ISMAP: 0,
|
|
TRUNCATED: truncated,
|
|
TRUNC_MSG: truncMsg
|
|
}],
|
|
versions: [],
|
|
xl_rules: []
|
|
}
|
|
} else {
|
|
response = {
|
|
approvers: [], cols: [], dqdata: [], dqrules: [],
|
|
dsmeta: [
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: '0' },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' }
|
|
],
|
|
maxvarlengths: [], query: [], sasdata: [],
|
|
$sasdata: { vars: {} },
|
|
sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
|
|
versions: [], xl_rules: []
|
|
}
|
|
}
|
|
|
|
webOutOpen()
|
|
webOutObj(response.approvers, 'approvers')
|
|
webOutObj(response.cols, 'cols')
|
|
webOutObj(response.dqdata, 'dqdata')
|
|
webOutObj(response.dqrules, 'dqrules')
|
|
webOutObj(response.dsmeta, 'dsmeta')
|
|
webOutObj(response.maxvarlengths, 'maxvarlengths')
|
|
webOutObj(response.query, 'query')
|
|
webOutObj(response.sasdata, 'sasdata', response.$sasdata)
|
|
webOutObj(response.sasparams, 'sasparams')
|
|
webOutObj(response.versions, 'versions')
|
|
webOutObj(response.xl_rules, 'xl_rules')
|
|
webOutClose()
|