Files
dc/sas/mocks/sasjs/services/editors/getdata.js
T
dc 6513d09f05 feat(editor): mark derived variables as computed in the EDIT grid
Derived variables are read-only, but they were rendered exactly like any other
read-only column: dimmed via readonlyCell, with no indication that the column is
computed at read time rather than locked by a rule or by column level security.
The grid could not tell the difference either - getdata holds a derived column
read-only with a READONLY DQ rule, which is also a rule type a table can be
configured with.

getdata now names the derived columns in sasparams.DERIVEDVARS (space separated,
upper case, from MPE_DERIVED's outvars). The EDIT screen holds that list, and
marks those columns with their own classes instead of the read-only dimming:
a tinted header with an 'fx' marker and a tooltip, tinted italic cells, and a
left edge on the first derived column so the computed block reads as a group.
Blue is the info family, leaving the primary key green and the warning ochre to
their existing meanings. Light and dark values are both set.

The SASjs mock getdata.js returns the same field, so the Cypress suite exercises
the styling; the existing derived-variables spec asserted readonlyCell on the
derived header, which is now the fallback for a plain read-only column, so that
assertion moves to dc-derived-header and a physical column stays the control.

Lint: prettier --check clean on the touched client files; sasjs lint reports no
new warnings for getdata.sas.
2026-10-27 14:34:13 +00:00

476 lines
21 KiB
JavaScript

const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
const dcLibref = 'DC_JSLIB'
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
// Mirrors SAS cats() for the values a dropdown source can hold: a special
// missing stored in its period form (".a") becomes the bare uppercase letter
// ("A") that cats() returns, so the mock's dropdown list matches the real one.
function cats(value) {
if (typeof value === 'string') {
const m = /^\.(_|[a-z])$/i.exec(value.trim())
if (m) return m[1].toUpperCase()
return value.trim()
}
return String(value)
}
/**
* SAS sort position of a special missing, or null when the value is ordinary.
* `._` sorts first, then `.a` to `.z` - all of them below every number.
*/
function missingRank(value) {
const m = /^\.(_|[a-z])$/i.exec(String(value).trim())
if (!m) return null
return m[1] === '_' ? 0 : m[1].toUpperCase().charCodeAt(0) - 64
}
// ─── Parse input ──────────────────────────────────────────────────────────────
const _sctRow = fetchTable('SASControlTable')[0] || {}
let requestedLibref = dcLibref
let requestedTable = ''
if (_sctRow.LIBDS) {
const parts = _sctRow.LIBDS.split('.')
if (parts.length >= 2) { requestedLibref = parts[0]; requestedTable = parts[1] }
}
if (!requestedTable) {
if (_sctRow.DSN) requestedTable = _sctRow.DSN.trim()
if (_sctRow.LIBREF) requestedLibref = _sctRow.LIBREF.trim()
}
// FILTER_RK (matching getdata.sas: if filter_rk le 0 then filter_rk=-1)
const filterRk = Number(_sctRow.FILTER_RK) > 0 ? Number(_sctRow.FILTER_RK) : -1
// ─── Load table data and schema ──────────────────────────────────────────────
// A table can live in any libref (DC_JSLIB for control tables, TESTDATA for
// demo user tables), so resolve the data directory from the libref.
const dataDir = libDataDir(requestedLibref)
const loadTableData = makeTableLoader(dataDir)
let tableData = loadTableData(requestedTable)
if (!tableData) {
// Fall back to searching every libref if the table isn't in the requested
// libref (e.g. a LIBDS that doesn't match the actual data folder)
const found = loadTableAnyLib(requestedTable)
if (found) {
tableData = found.data
requestedLibref = found.libref
}
}
// Load table registration from MPE_TABLES (needed for PK detection, temporal
// column exclusion, and sasparams). MPE_TABLES always lives in the control
// library (DC_JSLIB / mpelib), not in user data libraries.
const mpeDataDir = libDataDir(dcLibref)
const mpeLoadTableData = makeTableLoader(mpeDataDir)
let tableReg = null
const mpeTablesData = mpeLoadTableData('MPE_TABLES')
if (mpeTablesData && mpeTablesData.rows) {
tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
}
// The variables exposed to the editor as the primary key, matching getdata.sas:
// pk = coalesce(rk_underlying, buskey). A table registered with rk_underlying
// is keyed on its business key in the editor - the RK itself is assigned by the
// backend loader, so it is neither shown nor sent by the user.
const rkUnderlying = tableReg && tableReg.rk_underlying ? tableReg.rk_underlying.trim() : ''
const buskey = tableReg && tableReg.buskey ? tableReg.buskey.trim() : ''
const pkFields = rkUnderlying || buskey
const pkColumns = pkFields ? pkFields.split(/\s+/) : []
// Order columns with the PK variables first, in PK order, then the remaining
// columns in schema order. getdata.sas does the same ("order delete var and pk
// fields at start of table") and the client reads the first PKCNT columns
// positionally when it detects duplicate keys.
function pkFirst(columns, keys) {
const rank = new Map(keys.map((name, i) => [name.toUpperCase(), i]))
const keyColumns = columns
.filter(col => rank.has(col.name.toUpperCase()))
.sort((a, b) => rank.get(a.name.toUpperCase()) - rank.get(b.name.toUpperCase()))
return keyColumns.concat(columns.filter(col => !rank.has(col.name.toUpperCase())))
}
// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
// Applied before the filter, so a filter can reference them. The derived
// columns are added to the schema too, which is what puts them in COLS and in
// the grid; they are held read-only by a READONLY rule below.
const derived = mpeDerived({
libref: requestedLibref,
table: requestedTable,
dataDir: mpeDataDir,
columns: tableData ? tableData.columns : []
})
if (tableData && derived.length > 0) {
tableData = mpeDeriveRows(tableData, derived)
}
// ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
// Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
// NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
let dqrules = []
let dqdata = []
if (validationsData && validationsData.rows) {
const pkCols = new Set(pkColumns.map(c => c.toUpperCase()))
dqrules = validationsData.rows
.filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
.map(r => ({
BASE_COL: r.base_col,
RULE_TYPE: r.rule_type,
RULE_VALUE: r.rule_value,
X: pkCols.has(r.base_col.toUpperCase()) ? 0 : 1
}))
}
// ─── Apply stored filter (mpe_filtermaster) ──────────────────────────────────
// Mirrors getdata.sas: %mpe_filtermaster(EDIT,&orig_libds,filter_rk=&filter_rk)
const mpeDataDir2 = libDataDir(dcLibref)
const {
query: filterQuery,
predicate: filterPredicate,
filterText: filterText
} = mpeFilterMaster({
mode: 'EDIT',
libds: (requestedLibref + '.' + requestedTable).toUpperCase(),
filterRk: filterRk,
dataDir: mpeDataDir2,
columns: tableData ? tableData.columns : [],
tableReg: tableReg
})
// Synthesise NOTNULL rules from the table schema, the same way getdata.sas
// merges dictionary.columns (notnull='yes') into dqrules.
if (tableData && tableData.columns) {
const existingNotNull = new Set(
dqrules.filter(r => r.RULE_TYPE === 'NOTNULL').map(r => r.BASE_COL.toUpperCase())
)
for (const rule of notNullRules(tableData.columns)) {
if (!existingNotNull.has(rule.BASE_COL)) dqrules.push(rule)
}
}
// A derived variable is computed at read time, so the grid must not accept a
// value for it. COLS carries READ for the same reason (a derived column has no
// CLS rule row), but the client only applies COLS.CLS_RULE when CLS_FLAG is
// set, which column-level security drives - so the rule is declared here too,
// as getdata.sas does. dqrules is the request-local array the filter/map above
// built, so appending to it leaves the loaded MPE_VALIDATIONS rows untouched.
for (const v of derived) {
dqrules.push({ BASE_COL: v.name, RULE_TYPE: 'READONLY', RULE_VALUE: '' })
}
// ─── DQ dropdown data (from MPE_SELECTBOX + SOFTSELECT from table data) ──────
// MPE_SELECTBOX lives in the control library.
const selectboxData = mpeLoadTableData('MPE_SELECTBOX')
if (selectboxData && selectboxData.rows) {
dqdata = selectboxData.rows
.filter(r => r.select_lib === requestedLibref && r.select_ds === requestedTable)
.map(r => ({
BASE_COL: r.base_column,
RULE_VALUE: r.base_column,
RULE_DATA: r.selectbox_value,
SELECTBOX_ORDER: r.selectbox_order
}))
}
// For SOFTSELECT/HARDSELECT rules that reference the table's own data,
// generate dropdown values from the actual table rows
if (tableData && tableData.rows && tableData.columns) {
const selectRules = dqrules.filter(r =>
(r.RULE_TYPE === 'SOFTSELECT' || r.RULE_TYPE === 'HARDSELECT') && r.RULE_VALUE.includes('.'))
for (const rule of selectRules) {
const parts = rule.RULE_VALUE.split('.')
const colName = parts[parts.length - 1]
const lcName = colName.toLowerCase()
const seen = new Set()
const values = []
for (const row of tableData.rows) {
let val = row[colName]
if (val === undefined) val = row[lcName]
if (val === undefined) {
const key = Object.keys(row).find(k => k.toLowerCase() === lcName)
if (key) val = row[key]
}
if (val !== undefined && val !== null) {
const strVal = cats(val)
if (!seen.has(strVal)) {
seen.add(strVal)
values.push({ raw: val, str: strVal })
}
}
}
// getdata.sas orders the source by the column itself, and a special
// missing sorts below every number - so the list reads `._`, `.a`-`.z`,
// then the numbers ascending.
values.sort((a, b) => {
const ra = missingRank(a.raw)
const rb = missingRank(b.raw)
if (ra !== null && rb !== null) return ra - rb
if (ra !== null) return -1
if (rb !== null) return 1
const na = Number(a.str)
const nb = Number(b.str)
if (!isNaN(na) && !isNaN(nb)) return na - nb
return a.str < b.str ? -1 : a.str > b.str ? 1 : 0
})
values.forEach((v, i) => {
dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: v.str, SELECTBOX_ORDER: i + 1 })
})
}
}
// ─── Build response ──────────────────────────────────────────────────────────
let response
// Set when the row or cell limit cut the selection down - mirrors the
// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.
if (tableData && tableData.columns && tableData.rows) {
// Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
let excludeCols = new Set()
if (tableReg && (tableReg.loadtype === 'TXTEMPORAL' || tableReg.loadtype === 'BITEMPORAL')) {
if (tableReg.var_txfrom) excludeCols.add(tableReg.var_txfrom.toUpperCase())
if (tableReg.var_txto) excludeCols.add(tableReg.var_txto.toUpperCase())
}
if (tableReg && tableReg.loadtype === 'BITEMPORAL') {
if (tableReg.var_busfrom) excludeCols.add(tableReg.var_busfrom.toUpperCase())
if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
}
// getdata.sas drops the RK field from the editor view when the table is
// registered with rk_underlying (drop_rk=&buskey), and exposes the business
// key instead.
if (rkUnderlying && buskey) {
buskey.split(/\s+/).forEach(c => excludeCols.add(c.toUpperCase()))
}
let visibleColumns = pkFirst(
tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase())),
pkColumns
)
let visibleRows = tableData.rows
.filter(r => filterPredicate(r))
.map(r => {
const copy = {}
for (const key of Object.keys(r)) {
if (!excludeCols.has(key.toUpperCase())) copy[key] = r[key]
}
return copy
})
// ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
// Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
// been applied and the rows sorted, with the data in work.OUT, and may
// replace it. The mock eval()s the hook's .js counterpart in this scope, so
// the hook may reassign visibleRows / visibleColumns - e.g. to show the live
// rows of another table behind an empty mirror.
if (tableReg && tableReg.pre_edit_hook) {
const hookSrc = mockHookSource(tableReg.pre_edit_hook)
if (hookSrc) {
try {
eval(hookSrc)
} catch (err) {
console.log('Error running pre_edit_hook ' + tableReg.pre_edit_hook + ': ' + err.message)
}
}
}
// ─── Row and cell limits ─────────────────────────────────────────────────
// Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
// cells (rows x columns), whichever is reached first - the selection is
// cut to min(rows, cells/columns) and flagged in sasparams.
// Both options are read from MPE_CONFIG, the way the real service picks
// them up from the settings program it generates - so editing the option
// in DC changes what the mock serves, and an option that is absent (or
// switched off, var_active=0) falls back to the shipped default. The
// check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
// does - a hook may replace the data with a larger table.
// MPE_CONFIG lives in the control library, not in the table's own library,
// so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
// uses. Passing the requested table's dataDir made the read fail for every
// non-control libref, and the catch then fell back to the shipped defaults.
const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
// getdata.sas counts the columns of work.out, i.e. the table's own
// columns - the delete column is client-owned and not part of it.
const editColCount = visibleColumns.length
let maxRows = maxObsWebEdit
let limitVar = 'DC_MAXOBS_WEBEDIT'
if (editColCount > 0) {
const cellRows = Math.floor(maxCellsWebEdit / editColCount)
if (cellRows < maxRows) {
maxRows = cellRows
limitVar = 'DC_MAXCELLS_WEBEDIT'
}
}
if (maxRows < 1) maxRows = 1
let truncated = 0
let truncMsg = ''
if (visibleRows.length > maxRows) {
truncated = 1
// Same wording as the macro getdata.sas sets, so the spec can assert
// on the text the real service would send.
truncMsg =
'Only the first ' +
maxRows +
' rows are loaded - the ' +
limitVar +
' limit has been reached. Filter the table to work with the rest.'
visibleRows = visibleRows.slice(0, maxRows)
}
// Augment rows: add delete flag, normalise keys, convert temporal values to ISO
const colLookup = {}
for (const col of visibleColumns) {
colLookup[col.name.toLowerCase()] = { name: col.name, ddtype: getDdType(col) }
}
const augmentedRows = visibleRows.map(r => {
const normalised = {}
for (const key of Object.keys(r)) {
const info = colLookup[key.toLowerCase()]
const targetKey = info ? info.name : key
normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
}
return { ...normalised }
})
// $sasdata vars: temporal columns are char ($200.), others keep their SAS type
const vars = {}
for (const col of visibleColumns) {
vars[col.name] = sasVarsEntry(col)
}
const colHeaders = visibleColumns.map(c => c.name)
// Determine DTVARS, DTTMVARS, TMVARS from column formats
let dtVars = [], dttmVars = [], tmVars = []
for (const col of visibleColumns) {
const ddtype = getDdType(col)
if (ddtype === 'DATETIME') dttmVars.push(col.name)
else if (ddtype === 'DATE') dtVars.push(col.name)
else if (ddtype === 'TIME') tmVars.push(col.name)
}
// Build cols array. Every column of the table, as getdata.sas emits them:
// the temporal columns are dropped from the data (the rows below) but not
// from the cols payload, so a filter can still be built on them. mpGetcols
// is the JS counterpart of %mp_getcols.sas - the macro getdata.sas itself
// calls to type these columns - so the editor sees the same DDTYPE the
// service would give it.
const cols = mpGetcols(tableData.columns).map((c, i) => {
const raw = tableData.columns[i]
const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
let coltype
if (ddtype === 'DATE') coltype = `{"data":"${raw.name}","type":"date"}`
else if (ddtype === 'DATETIME') coltype = `{"data":"${raw.name}","type":"datetime"}`
else if (ddtype === 'TIME') coltype = `{"data":"${raw.name}","type":"time"}`
else if (ddtype === 'N') coltype = `{"data":"${raw.name}","type":"numeric","format":"0"}`
else coltype = `{"data":"${raw.name}"}`
return {
NAME: c.NAME,
LABEL: c.LABEL,
FMTNAME: c.FMTNAME || ' ',
DDTYPE: ddtype,
CLS_RULE: 'READ',
MEMLABEL: ' ',
DESC: ' ',
LONGDESC: ' ',
COLTYPE: coltype
}
}).sort((a, b) => a.NAME.localeCompare(b.NAME))
// Build maxvarlengths
const maxvarlengths = [
...visibleColumns.map(col => {
let maxLen = 0
const lcName = col.name.toLowerCase()
for (const row of augmentedRows) {
let val = row[col.name]
if (val === undefined) val = row[lcName]
if (val !== undefined && val !== null) {
const len = String(val).length
if (len > maxLen) maxLen = len
}
}
return { NAME: col.name.toLowerCase(), MAXLEN: maxLen }
})
]
response = {
approvers: [{ PERSONNAME: 'sasdemo', EMAIL: 'sasdemo', USERID: 'sasdemo' }],
cols,
dqdata, dqrules,
dsmeta: [
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: String(tableData.rows.length) },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Variables', VALUE: String(tableData.columns.length) },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Engine', VALUE: 'V9' }
],
maxvarlengths,
query: filterQuery,
sasdata: augmentedRows,
$sasdata: { vars },
sasparams: [{
COLHEADERS: colHeaders.join(','),
FILTER_TEXT: filterText === '' ? ' ' : filterText,
PKCNT: pkColumns.length,
PK: pkFields,
DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',
LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
CLS_FLAG: 0,
ISMAP: 0,
TRUNCATED: truncated,
TRUNC_MSG: truncMsg,
// names the derived columns for the grid, as getdata.sas does - the
// READONLY rule above is why they cannot be edited, this is how the
// EDIT screen knows they are computed rather than locked
DERIVEDVARS: derived.map(d => d.name).join(' ')
}],
versions: [],
xl_rules: []
}
} else {
response = {
approvers: [], cols: [], dqdata: [], dqrules: [],
dsmeta: [
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: '0' },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' }
],
maxvarlengths: [], query: [], sasdata: [],
$sasdata: { vars: {} },
sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '', DERIVEDVARS: '' }],
versions: [], xl_rules: []
}
}
webOutOpen()
webOutObj(response.approvers, 'approvers')
webOutObj(response.cols, 'cols')
webOutObj(response.dqdata, 'dqdata')
webOutObj(response.dqrules, 'dqrules')
webOutObj(response.dsmeta, 'dsmeta')
webOutObj(response.maxvarlengths, 'maxvarlengths')
webOutObj(response.query, 'query')
webOutObj(response.sasdata, 'sasdata', response.$sasdata)
webOutObj(response.sasparams, 'sasparams')
webOutObj(response.versions, 'versions')
webOutObj(response.xl_rules, 'xl_rules')
webOutClose()