dc
f08c4c231f
Review follow-ups on the row/cell limits: - getdata.sas checked the limits twice - once before the sort that caps work.out, and again after PRE_EDIT_HOOK. It is now a single check, after both, so the counts cover everything that could reach the browser: the rows the selection matched, plus anything a hook has added. The payload cap moves to where work.outdata is built, which is the only place that needs it. Each mp_abort also names its option once - the limit was repeated in the message text as well as the condition. - The migration closed only var_active=1 rows but inserted active rows unconditionally, so an option a site had switched off was silently switched back on. It now leaves those alone and logs a note. - row-cell-limits.cy.ts test 1 matched a regex against the grid's text, which the fixture's own columns satisfy (DEPTH_M reaches 4293, SAMPLE_COUNT 4210), so it did not pin the raised VIEW cap. It now reads the PK column - the first td of a .ht_master row - and requires values in the PK range 1001-2000 with a maximum above 1500, which the old 500 row cap could never render.
403 lines
17 KiB
JavaScript
403 lines
17 KiB
JavaScript
const nodePath = require('path')
|
|
|
|
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
|
|
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
|
|
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
|
|
const dcLibref = 'DC_JSLIB'
|
|
|
|
// Load shared DC mock utilities
|
|
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
|
|
|
|
// Mirrors SAS cats() for the values a dropdown source can hold: a special
|
|
// missing stored in its period form (".a") becomes the bare uppercase letter
|
|
// ("A") that cats() returns, so the mock's dropdown list matches the real one.
|
|
function cats(value) {
|
|
if (typeof value === 'string') {
|
|
const m = /^\.(_|[a-z])$/i.exec(value.trim())
|
|
if (m) return m[1].toUpperCase()
|
|
return value.trim()
|
|
}
|
|
return String(value)
|
|
}
|
|
|
|
/**
|
|
* SAS sort position of a special missing, or null when the value is ordinary.
|
|
* `._` sorts first, then `.a` to `.z` - all of them below every number.
|
|
*/
|
|
function missingRank(value) {
|
|
const m = /^\.(_|[a-z])$/i.exec(String(value).trim())
|
|
if (!m) return null
|
|
return m[1] === '_' ? 0 : m[1].toUpperCase().charCodeAt(0) - 64
|
|
}
|
|
|
|
// ─── Parse input ──────────────────────────────────────────────────────────────
|
|
|
|
const _sctRow = fetchTable('SASControlTable')[0] || {}
|
|
let requestedLibref = dcLibref
|
|
let requestedTable = ''
|
|
if (_sctRow.LIBDS) {
|
|
const parts = _sctRow.LIBDS.split('.')
|
|
if (parts.length >= 2) { requestedLibref = parts[0]; requestedTable = parts[1] }
|
|
}
|
|
if (!requestedTable) {
|
|
if (_sctRow.DSN) requestedTable = _sctRow.DSN.trim()
|
|
if (_sctRow.LIBREF) requestedLibref = _sctRow.LIBREF.trim()
|
|
}
|
|
|
|
// FILTER_RK (matching getdata.sas: if filter_rk le 0 then filter_rk=-1)
|
|
const filterRk = Number(_sctRow.FILTER_RK) > 0 ? Number(_sctRow.FILTER_RK) : -1
|
|
|
|
// ─── Load table data and schema ──────────────────────────────────────────────
|
|
// A table can live in any libref (DC_JSLIB for control tables, TESTDATA for
|
|
// demo user tables), so resolve the data directory from the libref.
|
|
|
|
const dataDir = libDataDir(requestedLibref)
|
|
const loadTableData = makeTableLoader(dataDir)
|
|
|
|
let tableData = loadTableData(requestedTable)
|
|
if (!tableData) {
|
|
// Fall back to searching every libref if the table isn't in the requested
|
|
// libref (e.g. a LIBDS that doesn't match the actual data folder)
|
|
const found = loadTableAnyLib(requestedTable)
|
|
if (found) {
|
|
tableData = found.data
|
|
requestedLibref = found.libref
|
|
}
|
|
}
|
|
|
|
// Load table registration from MPE_TABLES (needed for PK detection, temporal
|
|
// column exclusion, and sasparams). MPE_TABLES always lives in the control
|
|
// library (DC_JSLIB / mpelib), not in user data libraries.
|
|
const mpeDataDir = libDataDir(dcLibref)
|
|
const mpeLoadTableData = makeTableLoader(mpeDataDir)
|
|
|
|
let tableReg = null
|
|
const mpeTablesData = mpeLoadTableData('MPE_TABLES')
|
|
if (mpeTablesData && mpeTablesData.rows) {
|
|
tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
|
|
}
|
|
|
|
// ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
|
|
// Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
|
|
// NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
|
|
|
|
const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
|
|
let dqrules = []
|
|
let dqdata = []
|
|
if (validationsData && validationsData.rows) {
|
|
const pkCols = new Set()
|
|
if (tableReg && tableReg.buskey) {
|
|
tableReg.buskey.trim().split(/\s+/).forEach(c => pkCols.add(c.toUpperCase()))
|
|
}
|
|
dqrules = validationsData.rows
|
|
.filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
|
|
.map(r => ({
|
|
BASE_COL: r.base_col,
|
|
RULE_TYPE: r.rule_type,
|
|
RULE_VALUE: r.rule_value,
|
|
X: pkCols.has(r.base_col.toUpperCase()) ? 0 : 1
|
|
}))
|
|
}
|
|
|
|
// ─── Apply stored filter (mpe_filtermaster) ──────────────────────────────────
|
|
// Mirrors getdata.sas: %mpe_filtermaster(EDIT,&orig_libds,filter_rk=&filter_rk)
|
|
|
|
const mpeDataDir2 = libDataDir(dcLibref)
|
|
const {
|
|
query: filterQuery,
|
|
predicate: filterPredicate,
|
|
filterText: filterText
|
|
} = mpeFilterMaster({
|
|
mode: 'EDIT',
|
|
libds: (requestedLibref + '.' + requestedTable).toUpperCase(),
|
|
filterRk: filterRk,
|
|
dataDir: mpeDataDir2,
|
|
columns: tableData ? tableData.columns : [],
|
|
tableReg: tableReg
|
|
})
|
|
|
|
// Synthesise NOTNULL rules from the table schema, the same way getdata.sas
|
|
// merges dictionary.columns (notnull='yes') into dqrules.
|
|
if (tableData && tableData.columns) {
|
|
const existingNotNull = new Set(
|
|
dqrules.filter(r => r.RULE_TYPE === 'NOTNULL').map(r => r.BASE_COL.toUpperCase())
|
|
)
|
|
for (const rule of notNullRules(tableData.columns)) {
|
|
if (!existingNotNull.has(rule.BASE_COL)) dqrules.push(rule)
|
|
}
|
|
}
|
|
|
|
// ─── DQ dropdown data (from MPE_SELECTBOX + SOFTSELECT from table data) ──────
|
|
// MPE_SELECTBOX lives in the control library.
|
|
|
|
const selectboxData = mpeLoadTableData('MPE_SELECTBOX')
|
|
if (selectboxData && selectboxData.rows) {
|
|
dqdata = selectboxData.rows
|
|
.filter(r => r.select_lib === requestedLibref && r.select_ds === requestedTable)
|
|
.map(r => ({
|
|
BASE_COL: r.base_column,
|
|
RULE_VALUE: r.base_column,
|
|
RULE_DATA: r.selectbox_value,
|
|
SELECTBOX_ORDER: r.selectbox_order
|
|
}))
|
|
}
|
|
|
|
// For SOFTSELECT/HARDSELECT rules that reference the table's own data,
|
|
// generate dropdown values from the actual table rows
|
|
if (tableData && tableData.rows && tableData.columns) {
|
|
const selectRules = dqrules.filter(r =>
|
|
(r.RULE_TYPE === 'SOFTSELECT' || r.RULE_TYPE === 'HARDSELECT') && r.RULE_VALUE.includes('.'))
|
|
for (const rule of selectRules) {
|
|
const parts = rule.RULE_VALUE.split('.')
|
|
const colName = parts[parts.length - 1]
|
|
const lcName = colName.toLowerCase()
|
|
const seen = new Set()
|
|
const values = []
|
|
for (const row of tableData.rows) {
|
|
let val = row[colName]
|
|
if (val === undefined) val = row[lcName]
|
|
if (val === undefined) {
|
|
const key = Object.keys(row).find(k => k.toLowerCase() === lcName)
|
|
if (key) val = row[key]
|
|
}
|
|
if (val !== undefined && val !== null) {
|
|
const strVal = cats(val)
|
|
if (!seen.has(strVal)) {
|
|
seen.add(strVal)
|
|
values.push({ raw: val, str: strVal })
|
|
}
|
|
}
|
|
}
|
|
// getdata.sas orders the source by the column itself, and a special
|
|
// missing sorts below every number - so the list reads `._`, `.a`-`.z`,
|
|
// then the numbers ascending.
|
|
values.sort((a, b) => {
|
|
const ra = missingRank(a.raw)
|
|
const rb = missingRank(b.raw)
|
|
if (ra !== null && rb !== null) return ra - rb
|
|
if (ra !== null) return -1
|
|
if (rb !== null) return 1
|
|
const na = Number(a.str)
|
|
const nb = Number(b.str)
|
|
if (!isNaN(na) && !isNaN(nb)) return na - nb
|
|
return a.str < b.str ? -1 : a.str > b.str ? 1 : 0
|
|
})
|
|
values.forEach((v, i) => {
|
|
dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: v.str, SELECTBOX_ORDER: i + 1 })
|
|
})
|
|
}
|
|
}
|
|
|
|
// ─── Build response ──────────────────────────────────────────────────────────
|
|
|
|
let response
|
|
// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
|
|
// getdata.sas, and is emitted as sasjsAbort below.
|
|
let abortMsg = ''
|
|
|
|
if (tableData && tableData.columns && tableData.rows) {
|
|
// Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
|
|
let excludeCols = new Set()
|
|
if (tableReg && (tableReg.loadtype === 'TXTEMPORAL' || tableReg.loadtype === 'BITEMPORAL')) {
|
|
if (tableReg.var_txfrom) excludeCols.add(tableReg.var_txfrom.toUpperCase())
|
|
if (tableReg.var_txto) excludeCols.add(tableReg.var_txto.toUpperCase())
|
|
}
|
|
if (tableReg && tableReg.loadtype === 'BITEMPORAL') {
|
|
if (tableReg.var_busfrom) excludeCols.add(tableReg.var_busfrom.toUpperCase())
|
|
if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
|
|
}
|
|
|
|
let visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
|
|
let visibleRows = tableData.rows
|
|
.filter(r => filterPredicate(r))
|
|
.map(r => {
|
|
const copy = {}
|
|
for (const key of Object.keys(r)) {
|
|
if (!excludeCols.has(key.toUpperCase())) copy[key] = r[key]
|
|
}
|
|
return copy
|
|
})
|
|
|
|
// ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
|
|
// Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
|
|
// been applied and the rows sorted, with the data in work.OUT, and may
|
|
// replace it. The mock eval()s the hook's .js counterpart in this scope, so
|
|
// the hook may reassign visibleRows / visibleColumns - e.g. to show the live
|
|
// rows of another table behind an empty mirror.
|
|
if (tableReg && tableReg.pre_edit_hook) {
|
|
const hookSrc = mockHookSource(tableReg.pre_edit_hook)
|
|
if (hookSrc) {
|
|
try {
|
|
eval(hookSrc)
|
|
} catch (err) {
|
|
console.log('Error running pre_edit_hook ' + tableReg.pre_edit_hook + ': ' + err.message)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Row and cell limits ─────────────────────────────────────────────────
|
|
// Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
|
|
// cells (rows x columns), whichever is reached first. The real service
|
|
// reads both from MPE_CONFIG (via the generated settings program); the
|
|
// mock hard-codes the shipped defaults. The check runs after the
|
|
// PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
|
|
// replace the data with a larger table. The cell count includes the
|
|
// housekeeping delete column, which is what getdata.sas counts.
|
|
const maxObsWebEdit = 250
|
|
const maxCellsWebEdit = 200000
|
|
const editColCount = visibleColumns.length + 1
|
|
const cellCount = visibleRows.length * editColCount
|
|
if (visibleRows.length > maxObsWebEdit) {
|
|
abortMsg =
|
|
'Table is too big (' +
|
|
visibleRows.length +
|
|
' rows) - please filter and try again!'
|
|
} else if (cellCount > maxCellsWebEdit) {
|
|
abortMsg =
|
|
'Selection is too wide (' +
|
|
visibleRows.length +
|
|
' rows x ' +
|
|
editColCount +
|
|
' cols) - please filter and try again!'
|
|
}
|
|
|
|
// Augment rows: add delete flag, normalise keys, convert temporal values to ISO
|
|
const colLookup = {}
|
|
for (const col of visibleColumns) {
|
|
colLookup[col.name.toLowerCase()] = { name: col.name, ddtype: getDdType(col) }
|
|
}
|
|
const augmentedRows = visibleRows.map(r => {
|
|
const normalised = {}
|
|
for (const key of Object.keys(r)) {
|
|
const info = colLookup[key.toLowerCase()]
|
|
const targetKey = info ? info.name : key
|
|
normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
|
|
}
|
|
return { _____DELETE__THIS__RECORD_____: 'No', ...normalised }
|
|
})
|
|
|
|
// $sasdata vars: temporal columns are char ($200.), others keep their SAS type
|
|
const vars = { _____DELETE__THIS__RECORD_____: { format: '$3.', label: '_____DELETE__THIS__RECORD_____', length: '3', type: 'char' } }
|
|
for (const col of visibleColumns) {
|
|
vars[col.name] = sasVarsEntry(col)
|
|
}
|
|
|
|
const colHeaders = ['_____DELETE__THIS__RECORD_____', ...visibleColumns.map(c => c.name)]
|
|
|
|
// Determine DTVARS, DTTMVARS, TMVARS from column formats
|
|
let dtVars = [], dttmVars = [], tmVars = []
|
|
for (const col of visibleColumns) {
|
|
const ddtype = getDdType(col)
|
|
if (ddtype === 'DATETIME') dttmVars.push(col.name)
|
|
else if (ddtype === 'DATE') dtVars.push(col.name)
|
|
else if (ddtype === 'TIME') tmVars.push(col.name)
|
|
}
|
|
|
|
// Build cols array
|
|
const cols = visibleColumns.map(col => {
|
|
const ddtype = getDdType(col)
|
|
const fmtname = col.format
|
|
? col.format.replace(/[\d.]+$/, '').replace('datetime', 'DATETIME').replace('date', 'DATE').replace('time', 'TIME').replace('best', 'BEST').replace('E8601DT', 'DATETIME')
|
|
: ' '
|
|
let coltype
|
|
if (ddtype === 'DATE') coltype = `{"data":"${col.name}","type":"date"}`
|
|
else if (ddtype === 'DATETIME') coltype = `{"data":"${col.name}","type":"datetime"}`
|
|
else if (ddtype === 'TIME') coltype = `{"data":"${col.name}","type":"time"}`
|
|
else if (ddtype === 'N') coltype = `{"data":"${col.name}","type":"numeric","format":"0"}`
|
|
else coltype = `{"data":"${col.name}"}`
|
|
return {
|
|
NAME: col.name,
|
|
LABEL: col.label || col.name,
|
|
FMTNAME: fmtname || ' ',
|
|
DDTYPE: ddtype,
|
|
CLS_RULE: 'READ',
|
|
MEMLABEL: ' ',
|
|
DESC: ' ',
|
|
LONGDESC: ' ',
|
|
COLTYPE: coltype
|
|
}
|
|
}).sort((a, b) => a.NAME.localeCompare(b.NAME))
|
|
|
|
// Build maxvarlengths
|
|
const maxvarlengths = [
|
|
{ NAME: '_____DELETE__THIS__RECORD_____', MAXLEN: 3 },
|
|
...visibleColumns.map(col => {
|
|
let maxLen = 0
|
|
const lcName = col.name.toLowerCase()
|
|
for (const row of augmentedRows) {
|
|
let val = row[col.name]
|
|
if (val === undefined) val = row[lcName]
|
|
if (val !== undefined && val !== null) {
|
|
const len = String(val).length
|
|
if (len > maxLen) maxLen = len
|
|
}
|
|
}
|
|
return { NAME: col.name.toLowerCase(), MAXLEN: maxLen }
|
|
})
|
|
]
|
|
|
|
response = {
|
|
approvers: [{ PERSONNAME: 'sasdemo', EMAIL: 'sasdemo', USERID: 'sasdemo' }],
|
|
cols,
|
|
dqdata, dqrules,
|
|
dsmeta: [
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: String(tableData.rows.length) },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Variables', VALUE: String(tableData.columns.length) },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Engine', VALUE: 'V9' }
|
|
],
|
|
maxvarlengths,
|
|
query: filterQuery,
|
|
sasdata: augmentedRows,
|
|
$sasdata: { vars },
|
|
sasparams: [{
|
|
COLHEADERS: colHeaders.join(','),
|
|
FILTER_TEXT: filterText === '' ? ' ' : filterText,
|
|
PKCNT: tableReg && tableReg.buskey ? tableReg.buskey.trim().split(/\s+/).length : 0,
|
|
PK: tableReg ? tableReg.buskey || '' : '',
|
|
DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
|
|
DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
|
|
TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',
|
|
LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
|
|
RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
|
|
CLS_FLAG: 0,
|
|
ISMAP: 0
|
|
}],
|
|
versions: [],
|
|
xl_rules: []
|
|
}
|
|
} else {
|
|
response = {
|
|
approvers: [], cols: [], dqdata: [], dqrules: [],
|
|
dsmeta: [
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: '0' },
|
|
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' }
|
|
],
|
|
maxvarlengths: [], query: [], sasdata: [],
|
|
$sasdata: { vars: {} },
|
|
sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0 }],
|
|
versions: [], xl_rules: []
|
|
}
|
|
}
|
|
|
|
webOutOpen()
|
|
if (abortMsg) {
|
|
webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
|
|
webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
|
|
} else {
|
|
webOutObj(response.approvers, 'approvers')
|
|
webOutObj(response.cols, 'cols')
|
|
webOutObj(response.dqdata, 'dqdata')
|
|
webOutObj(response.dqrules, 'dqrules')
|
|
webOutObj(response.dsmeta, 'dsmeta')
|
|
webOutObj(response.maxvarlengths, 'maxvarlengths')
|
|
webOutObj(response.query, 'query')
|
|
webOutObj(response.sasdata, 'sasdata', response.$sasdata)
|
|
webOutObj(response.sasparams, 'sasparams')
|
|
webOutObj(response.versions, 'versions')
|
|
webOutObj(response.xl_rules, 'xl_rules')
|
|
}
|
|
webOutClose()
|