Files
dc/sas/mocks/sasjs/services/public/viewdata.js
T
dc 96f70d53b6
Build / Build-and-ng-test (pull_request) Successful in 5m11s
Build / Build-and-test-development (pull_request) Canceled after 3m24s
Build / lighthouse (pull_request) Canceled after 3m22s
feat(mocks): generate the DDL download from the table, not a sample
public/getddl answered one fixed Postgres-flavoured body for every request -
DC_JSLIB.MPE_X_TEST whatever dataset was asked for - which is the last read path
the client calls where the mock invented its answer.

It now builds the SAS-flavour DDL from the table file, the way mp_ds2ddl builds
it from dictionary.columns and dictionary.indexes: the header, the create table
with each column's type, format, NOT NULL and label, and the unique index. The
list-output spacing was derived from the estate's own answers and reproduces
them byte for byte for all forty control tables.

Recorded as public/getddl/mpe_tables, so a mock change that alters the text
fails the gate. The header credits a user and a timestamp, so the projection
masks both - a user id is not something to publish, and the comparison is about
the DDL.

TSQL and PGSQL are not generated yet: a caller asking for one gets the SAS text,
and the README says so rather than inventing a second dialect without a
recording to check it against.
2026-11-07 18:10:11 +00:00

452 lines
20 KiB
JavaScript

const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
// Load shared DC mock utilities, then read the settings they know how to
// read: the control library comes from services/settings.js (dcInit is the
// mock's %mpe_init), with the constant only as the pre-makedata default.
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
const dcLibref = dcInit(driveRoot, appLoc).dcLibref
const mpeDataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', dcLibref)
// Row cap for the viewer - mirrors DC_MAXOBS_WEBVIEW in viewdata.sas, read from
// MPE_CONFIG the way the real service reads it from the settings program (an
// absent or switched-off option falls back to the shipped default). Also the
// MAXROWS value reported in sasparams.
const maxObsWebView = dcConfigNumber(
loadDcConfig(driveRoot, appLoc, mpeDataDir),
'DC_MAXOBS_WEBVIEW',
2000
)
/**
* The text SAS puts in the viewer for a temporal column.
*
* The estate prints the value at the column's own format, and the mock has to
* read that format rather than print one shape for every datetime: a column
* formatted E8601DT26.6 shows "2026-01-02T03:04:05.000000" while one formatted
* DATETIME19.3 shows "01JAN60:00:00:00.00". The default shapes come from the
* recorded response (golden/public/viewdata/mpe_tables.json, `_meta.sas` 9.4M9 -
* a value only a SAS runtime emits, see the Provenance section of
* parity/README.md), which is why they are not simply the format's own
* rendering: SAS prints a two-digit year and a two-decimal fraction there.
*/
const VIEW_MONTHS = ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC']
const SAS_EPOCH_MS = Date.UTC(1960, 0, 1)
function pad2(value) {
return String(value).padStart(2, '0')
}
function pad4(value) {
return String(value).padStart(4, '0')
}
function viewDateTime(seconds) {
const whole = Math.floor(seconds)
const d = new Date(SAS_EPOCH_MS + whole * 1000)
// two printed digits, so a fraction that rounds up must not carry into them
const fraction = Math.min(99, Math.round((seconds - whole) * 100))
return (
pad2(d.getUTCDate()) + VIEW_MONTHS[d.getUTCMonth()] +
String(d.getUTCFullYear()).slice(2) + ':' +
pad2(d.getUTCHours()) + ':' + pad2(d.getUTCMinutes()) + ':' +
pad2(d.getUTCSeconds()) + '.' + pad2(fraction)
)
}
function viewDate(seconds) {
// whole days, rounded as the text branch rounds, so a stored number and a
// stored ISO string of the same instant land on the same day
const d = new Date(SAS_EPOCH_MS + Math.round(seconds) * 86400000)
// DATE9. is four digits wide, so a year below 1000 is zero-padded
return pad2(d.getUTCDate()) + VIEW_MONTHS[d.getUTCMonth()] + pad4(d.getUTCFullYear())
}
function viewTime(seconds) {
const whole = Math.floor(seconds)
const fraction = Math.min(99, Math.round((seconds - whole) * 100))
return (
pad2(Math.floor(whole / 3600)) + ':' +
pad2(Math.floor((whole % 3600) / 60)) + ':' +
pad2(whole % 60) + '.' + pad2(fraction)
)
}
/** The fraction width an E8601 format names: E8601DT26.6 prints six digits. */
function isoDecimals(format) {
const match = /\.(\d+)$/.exec(String(format || ''))
return match ? Number(match[1]) : 0
}
/**
* The whole second and its fraction, after rounding to the printed precision.
* Rounding first is what carries: a value 1e-7 under the next second rounds to
* the next second with a zero fraction, where clamping would print .999999 of
* the second before it.
*/
function splitSeconds(seconds, decimals) {
const scale = Math.pow(10, decimals)
const ticks = Math.round(seconds * scale)
const whole = Math.floor(ticks / scale)
return { whole: whole, fraction: ticks - whole * scale }
}
/** E8601DT<w>.<d>: the ISO 8601 datetime SAS prints, at that precision. */
function viewIsoDateTime(seconds, format) {
const decimals = isoDecimals(format)
const parts = splitSeconds(seconds, decimals)
const d = new Date(SAS_EPOCH_MS + parts.whole * 1000)
let text =
pad4(d.getUTCFullYear()) + '-' + pad2(d.getUTCMonth() + 1) + '-' +
pad2(d.getUTCDate()) + 'T' + pad2(d.getUTCHours()) + ':' +
pad2(d.getUTCMinutes()) + ':' + pad2(d.getUTCSeconds())
if (decimals > 0) {
text += '.' + String(parts.fraction).padStart(decimals, '0')
}
return text
}
/** E8601DA<w>.: the ISO 8601 date. */
function viewIsoDate(seconds) {
const d = new Date(SAS_EPOCH_MS + Math.round(seconds) * 86400000)
return (
pad4(d.getUTCFullYear()) + '-' + pad2(d.getUTCMonth() + 1) + '-' +
pad2(d.getUTCDate())
)
}
/** E8601TM<w>.<d>: the ISO 8601 time. */
function viewIsoTime(seconds, format) {
const decimals = isoDecimals(format)
const parts = splitSeconds(seconds, decimals)
const whole = parts.whole
let text =
pad2(Math.floor(whole / 3600)) + ':' +
pad2(Math.floor((whole % 3600) / 60)) + ':' + pad2(whole % 60)
if (decimals > 0) {
text += '.' + String(parts.fraction).padStart(decimals, '0')
}
return text
}
/**
* A temporal value at the column's own format: E8601DT, E8601DA and E8601TM
* are printed as ISO, and everything else - including the zoned E8601DZ and
* E8601TZ, the interval forms E8601DTM/DN/MN, and every classic SAS format -
* keeps the SAS shape. Only the three are mapped because those are the ISO
* formats the estate's own tables carry; a zoned one would need its offset
* suffix printed too, and no fixture exercises that.
*/
function viewTemporal(seconds, ddtype, format) {
const name = String(format || '').toUpperCase()
if (ddtype === 'DATETIME') {
return name.indexOf('E8601DT') === 0
? viewIsoDateTime(seconds, format)
: viewDateTime(Math.round(seconds * 100) / 100)
}
if (ddtype === 'DATE') {
return name.indexOf('E8601DA') === 0
? viewIsoDate(seconds)
: viewDate(seconds)
}
return name.indexOf('E8601TM') === 0
? viewIsoTime(seconds, format)
: viewTime(Math.round(seconds * 100) / 100)
}
/** A value as the viewer shows it, whatever the row stored. */
function viewCell(value, ddtype, format) {
if (value === undefined || value === null || value === '') return null
let seconds = value
if (typeof value !== 'number') {
// the mock stores a datetime as ISO text in some rows; a text without a
// zone is SAS's own naive value, so it is read as UTC rather than in
// whatever zone the host happens to sit in
const text = String(value)
const zoned = /[Zz]$|[+-]\d{2}:?\d{2}$/.test(text) ? text : text + 'Z'
const parsed = Date.parse(zoned)
if (isNaN(parsed)) return String(value)
if (ddtype === 'DATE') {
seconds = Math.round((parsed - SAS_EPOCH_MS) / 86400000)
} else if (ddtype === 'TIME') {
// a time is seconds since midnight, so the day the text falls on is
// discarded - an ISO '1970-01-01T01:01:01' is 3661, not a datetime
seconds = (((parsed % 86400000) + 86400000) % 86400000) / 1000
} else {
seconds = (parsed - SAS_EPOCH_MS) / 1000
}
}
if (ddtype === 'DATETIME' || ddtype === 'DATE' || ddtype === 'TIME') {
return viewTemporal(seconds, ddtype, format)
}
return String(value)
}
// ─── Parse input ──────────────────────────────────────────────────────────────
const _sctRow = fetchTable('SASControlTable')[0] || {}
let libds = _sctRow.LIBDS || ''
const libref = libds.split('.')[0] || dcLibref
const dsn = libds.split('.')[1] || ''
// FILTER_RK (matching viewdata.sas: if filter_rk le 0 then filter_rk=-1)
const filterRk = Number(colVal(_sctRow, 'FILTER_RK')) > 0 ? Number(colVal(_sctRow, 'FILTER_RK')) : -1
// SEARCHTYPE / SEARCHVAL - only CHAR or NUM triggers a search, anything else
// (blank, NONE) is a normal view. Search values are stripped of the same
// characters viewdata.sas removes before handing them to mp_searchdata.
const rawSearchType = String(colVal(_sctRow, 'SEARCHTYPE') || '').toUpperCase()
const searchType = (rawSearchType === 'CHAR' || rawSearchType === 'NUM') ? rawSearchType : ''
const rawSearchVal = colVal(_sctRow, 'SEARCHVAL')
let searchVal = (rawSearchVal === undefined || rawSearchVal === null) ? '' : String(rawSearchVal)
if (searchType) searchVal = searchVal.replace(/[%&;"]/g, '')
// ─── Load the target table ───────────────────────────────────────────────────
const dataDir = libDataDir(libref)
const loadTableData = makeTableLoader(dataDir)
const loadMpeTable = makeTableLoader(mpeDataDir)
let tableData = loadTableData(dsn)
if (!tableData && dsn) {
const found = loadTableAnyLib(dsn)
if (found) tableData = found.data
}
// ─── Derived variables (mpe_derived.sas) ─────────────────────────────────────
// A stored filter can reference a derived column, so the derivation has to run
// before the filter. The columns themselves are dropped from the output - they
// belong to the EDIT screen, as viewdata.sas drops them.
// a copy: the derivation below returns a new column list, and every use here
// (cols, search, rows, vars) must keep looking at the physical columns only
const physicalColumns = [...(tableData && tableData.columns ? tableData.columns : [])]
const derived = mpeDerived({
libref: libref,
table: dsn,
dataDir: mpeDataDir,
columns: physicalColumns
})
if (tableData && derived.length > 0) {
tableData = mpeDeriveRows(tableData, derived)
}
// ─── Apply the stored filter (mpe_filtermaster, VIEW mode) ───────────────────
// Mirrors viewdata.sas: %mpe_filtermaster(VIEW,&libds,filter_rk=&filter_rk)
// Row Level Security and any saved filter are applied here, before the search.
const { predicate: filterPredicate, filterText } = mpeFilterMaster({
mode: 'VIEW',
libds: (libref + '.' + dsn).toUpperCase(),
filterRk: filterRk,
dataDir: mpeDataDir,
columns: tableData && tableData.columns ? tableData.columns : []
})
// ─── Search predicate (mp_searchdata.sas) ────────────────────────────────────
// mp_searchdata builds one WHERE clause covering every column of the table:
// or ("COL1"n ? "the string") /* character columns: case sensitive CONTAINS */
// or ("COL2"n = the number) /* numeric columns: exact equality */
function rowMatchesSearch(row, columns, type, value) {
for (const col of columns) {
const val = colVal(row, col.name)
if (val === undefined || val === null) continue
if (type === 'CHAR') {
if (col.type !== 'N' && String(val).includes(value)) return true
} else {
if (col.type === 'N' && Number(val) === Number(value)) return true
}
}
return false
}
// ─── Build response ──────────────────────────────────────────────────────────
// Look up PK fields from MPE_TABLES (mirrors mp_getpk).
// MPE_AUDIT is a standard DC audit table whose PK is always the first 5 columns.
const mpeTables = loadMpeTable('MPE_TABLES')
let pkFields = ' '
if (mpeTables && mpeTables.rows) {
const reg = mpeTables.rows.find(r => r.libref === libref && r.dsn === dsn)
if (reg && reg.buskey) {
pkFields = reg.buskey
// viewdata reports the validity column ahead of the business key: it is
// part of what identifies the current version of a row
const validity = String(colVal(reg, 'VAR_TXTO') || '').trim()
if (validity) pkFields = validity.toUpperCase() + ' ' + pkFields
}
}
if (dsn.toUpperCase() === 'MPE_AUDIT' && pkFields.trim() === '') {
pkFields = 'LOAD_REF LIBREF DSN KEY_HASH TGTVAR_NM'
}
let cols = []
let viewdata = []
let vars = {}
let nobs = 0
if (tableData && tableData.columns) {
// mpGetcols, the JS counterpart of %mp_getcols.sas - the macro viewdata.sas
// calls for its own column list.
// the data dictionary supplies a column's business label, as it does for
// getdata - viewdata.sas augments mp_getcols the same way
const dictionaryData = loadMpeTable('MPE_DATADICTIONARY')
const dictionary = {}
if (dictionaryData && dictionaryData.rows) {
const prefix = (libref + '.' + dsn + '.').toUpperCase()
for (const row of dictionaryData.rows) {
const source = String(colVal(row, 'DD_SOURCE') || '').toUpperCase()
if (String(colVal(row, 'DD_TYPE') || '').toUpperCase() !== 'COLUMN') continue
if (source.indexOf(prefix) !== 0) continue
dictionary[source.slice(prefix.length)] = colVal(row, 'DD_SHORTDESC')
}
}
cols = mpGetcols(physicalColumns).map(c => {
const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
const formatStr = c.TYPE === 'N'
? String(c.FORMAT || '8.').toUpperCase()
: ('$' + c.LENGTH + '.')
const label = dictionary[String(c.NAME).toUpperCase()] || c.LABEL || c.NAME
return {
NAME: c.NAME, TYPE: c.TYPE, LENGTH: c.LENGTH,
FORMAT: formatStr, FMTNAME: c.FMTNAME || ' ', DDTYPE: ddtype,
LABEL: label
}
})
// the estate answers the column list in name order
cols.sort((a, b) => (a.NAME < b.NAME ? -1 : a.NAME > b.NAME ? 1 : 0))
// Rows in scope: the stored filter is applied first, then the search
// (matching viewdata.sas, which searches a filtered view of the table).
const filteredRows = (tableData.rows || []).filter(r => filterPredicate(r))
let rows
if (searchType) {
// the derived columns are dropped before the search, so only the
// physical columns are searched (viewdata.sas drops them in the
// vwsearch data step mp_searchdata then reads)
const matches = filteredRows.filter(row => rowMatchesSearch(row, physicalColumns, searchType, searchVal))
// mp_searchdata caps its output at outobs=&DC_MAXOBS_WEBVIEW, and
// viewdata.sas then keeps only `if _n_<&DC_MAXOBS_WEBVIEW` of them.
nobs = Math.min(matches.length, maxObsWebView)
rows = matches.slice(0, maxObsWebView - 1)
} else {
// proc sql select count(*) ... where <filter> (uncapped)
nobs = filteredRows.length
// data work.viewdata; set &libds; where <filter>; if _n_>&DC_MAXOBS_WEBVIEW then stop;
rows = filteredRows.slice(0, maxObsWebView)
}
// viewdata rows: all values as strings (formatted, SAS proc-print style)
viewdata = rows.map(r => {
const row = {}
for (const col of physicalColumns) {
const lcName = col.name.toLowerCase()
let val = r[col.name]
if (val === undefined) val = r[lcName]
if (val === undefined) {
const key = Object.keys(r).find(k => k.toLowerCase() === lcName)
if (key) val = r[key]
}
const ddtype = getDdType(col)
if (val !== undefined && val !== null && val !== '') {
if (ddtype === 'DATE' || ddtype === 'DATETIME' || ddtype === 'TIME') {
val = viewCell(val, ddtype, col.format)
} else if (typeof val === 'number') {
val = formatSasValue(val, ddtype)
}
}
/*
A value the row does not carry is SAS's missing, which the viewer
renders as an empty cell: a space for a character column and '.' for
a number. Building the cell with String() would put the text
"undefined" on screen instead, which is what a missing property
stringifies to.
*/
if (val === undefined || val === null || val === '') {
row[payloadKey(col)] = col.type === 'N' ? '.' : ' '
} else {
/*
Only trailing blanks are noise - the width SAS pads to - and
only blanks, not every whitespace character: a value that ends
in a tab is not padded, and stripping it would change a value
the estate keeps. A leading blank is part of the value and the
viewer shows it.
val is a string by now: the branches above hand it through
viewCell or formatSasValue, both of which return text.
*/
const text = String(val).replace(/ +$/, '')
row[payloadKey(col)] = text === '' ? ' ' : text
}
}
return row
})
// SAS normally sends a single empty row when the table has no data, so
// the client can derive the column set even when empty (the UI iterates
// viewdata[0] to build the grid headers). Numeric columns get "."
// (SAS missing), character columns get "".
// A search that matches nothing sends no rows at all (the client shows
// its "no data found" panel instead of an empty grid).
if (viewdata.length === 0 && !searchType && physicalColumns.length > 0) {
const emptyRow = {}
// a blank character cell is a space everywhere else in this service,
// so the empty row says the same thing
for (const col of physicalColumns) emptyRow[payloadKey(col)] = col.type === 'N' ? '.' : ' '
viewdata = [emptyRow]
}
// $viewdata vars: keep original SAS types (num stays num, char stays char)
/*
The viewer's metadata is the column's own: a datetime arrives as num, 8
bytes, DATETIME19.3. sasVarsEntry describes the *edit grid*, where the
temporal columns have been converted to character, so it is not used here.
*/
/*
The two label members are deliberately different, and the recorded
response proves it: cols[].LABEL for DSN is "Dataset Name to be edited"
(the data dictionary's business label, which viewdata.sas augments in as
it does for getdata) while $viewdata.vars.DSN.label is "DSN", the physical
label. Unifying them would look tidier and would stop matching the estate.
*/
vars = {}
for (const col of physicalColumns) {
const isTemporal = ['DATE', 'DATETIME', 'TIME'].indexOf(getDdType(col)) >= 0
vars[payloadKey(col)] = {
format: isTemporal
? String(
col.format ||
(['DATE', 'TIME'].indexOf(getDdType(col)) >= 0
? (getDdType(col) === 'DATE' ? 'DATE9.' : 'TIME8.')
: 'DATETIME19.3')
).toUpperCase()
: (col.type === 'N' ? (col.format || 'best.') : ('$' + col.length + '.')),
label: col.label || payloadKey(col),
length: String(col.length),
type: col.type === 'N' ? 'num' : 'char'
}
}
}
webout('OPEN')
webout('OBJ', 'cls_rules', [])
webout('OBJ', 'cols', cols)
webout('OBJ', 'dsmeta', [{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: libref + '.' + dsn },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: String(nobs) },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Variables', VALUE: String(cols.length) },
{ ODS_TABLE: 'ATTRIBUTES', NAME: 'Engine', VALUE: 'V9' }])
webout('OBJ', 'query', [])
webout('OBJ', 'sasparams', [{ TABLEURI: ' ', TABLENAME: dsn, FILTER_TEXT: filterText === '' ? ' ' : filterText,
PK_FIELDS: pkFields, NOBS: nobs, VARS: cols.length, MAXROWS: maxObsWebView }])
webout('OBJ', 'versions', [])
webout('OBJ', 'viewdata', viewdata, { showmeta: { vars } })
requireMemberMetadata(weboutPath, 'viewdata')
webout('CLOSE')