dc
74060c0585
Review follow-ups on the rk_underlying handling: - mpeAssignRetainedKey (dcMockUtils) mirrors %mp_retainedkey: any submitted RK column is dropped and every row's RK is re-derived from the business key, so a stale RK in a hand-built payload cannot survive into the diff. Business keys compare the way SAS does - case-sensitive, trailing blanks ignored. - editors/loadfile excludes the RK from its expected-columns check for these tables, as loadfile.sas does, and runs the derivation before staging, so a real CSV upload (which legitimately omits the RK) is accepted and its diff matches instead of reporting every row as NEW. - both staging paths add the derived column to the staged headers, so MPE_SUBMIT.INPUT_VARS counts the width mpe_loader would see.
241 lines
10 KiB
JavaScript
241 lines
10 KiB
JavaScript
const nodePath = require('path')
|
|
|
|
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
|
|
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
|
|
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
|
|
|
|
// Load shared DC mock utilities
|
|
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
|
|
|
|
// DC database location comes from services/settings.js (written by makedata)
|
|
const dcConf = loadDcSettings(driveRoot, appLoc)
|
|
const dcLibref = dcConf.dcLibref
|
|
const mpeDataDir = dcConf.dataDir
|
|
const stagingDir = nodePath.resolve(mpeDataDir, 'staging')
|
|
|
|
const mpeLoadTableData = makeTableLoader(mpeDataDir)
|
|
|
|
// ─── Parse input ──────────────────────────────────────────────────────────────
|
|
// The file arrives as an uploaded fileref (_WEBIN_FILEREF1/2 - it may be either
|
|
// position, so pick the one whose filename ends in .csv), and the target table
|
|
// arrives as the `table` URL param. Mirrors loadfile.sas.
|
|
|
|
// Debug aid: log the uploaded filerefs (mirrors `%put _all_` in the SAS
|
|
// original) so upload issues are diagnosable from the session log.
|
|
for (let i = 1; i <= 10; i++) {
|
|
let fname
|
|
try {
|
|
fname = eval('typeof _WEBIN_FILENAME' + i + ' !== "undefined" ? String(_WEBIN_FILENAME' + i + ') : null')
|
|
} catch (e) { fname = null }
|
|
if (fname === null) break
|
|
let frefLen
|
|
try {
|
|
frefLen = eval('_WEBIN_FILEREF' + i + ' ? _WEBIN_FILEREF' + i + '.length : 0')
|
|
} catch (e) { frefLen = 0 }
|
|
console.log('_WEBIN_FILENAME' + i + ': ' + fname + ' (' + frefLen + ' bytes)')
|
|
}
|
|
|
|
let csvText = ''
|
|
let csvName = ''
|
|
|
|
// The filerefs are module-level consts in the runtime preamble, so dynamic
|
|
// access needs eval (same approach as fetchRaw in dcMockUtils).
|
|
for (let i = 1; i <= 10; i++) {
|
|
let fname
|
|
try {
|
|
fname = eval('typeof _WEBIN_FILENAME' + i + ' !== "undefined" ? String(_WEBIN_FILENAME' + i + ') : ""')
|
|
} catch (e) {
|
|
break
|
|
}
|
|
if (!fname) break
|
|
if (fname.toUpperCase().endsWith('.CSV')) {
|
|
try {
|
|
csvText = eval('_WEBIN_FILEREF' + i + '.toString()')
|
|
csvName = fname
|
|
} catch (e) { /* not defined */ }
|
|
break
|
|
}
|
|
}
|
|
|
|
let libds = ''
|
|
if (typeof table !== 'undefined' && table) libds = String(table).toUpperCase()
|
|
let libref = libds.split('.')[0] || dcLibref
|
|
let dsn = libds.split('.')[1] || ''
|
|
|
|
// ─── Validate (mirrors the mp_abort checks in loadfile.sas) ─────────────────
|
|
|
|
let msg = ''
|
|
if (!csvText) {
|
|
msg = 'File "' + csvName + '" must be a CSV! (Comma separated with .csv extension)'
|
|
} else if (!libds || !dsn) {
|
|
msg = 'Missing: table'
|
|
}
|
|
|
|
// ─── Parse the CSV and stage it ──────────────────────────────────────────────
|
|
// The uploaded CSV is a standard comma-separated file (not the SASjs adapter's
|
|
// %webout(FETCH) format), so it must be parsed with a plain CSV parser, not
|
|
// the parseCsv/parseCsvHeader helpers from dcMockUtils (which expect
|
|
// space-separated "COL:$char10." headers). Column formats are then loaded
|
|
// from the target table's schema (identified by the `table` query parameter)
|
|
// so that formatted date/datetime/time strings are converted back to SAS
|
|
// numeric values, mirroring loadfile.sas + mpe_loader.sas.
|
|
|
|
function parseStandardCsv(text) {
|
|
const lines = text.split(/\r?\n/).filter((l) => l.length > 0)
|
|
if (lines.length < 1) return { headers: [], rows: [] }
|
|
const headers = _parseCsvLine(lines[0])
|
|
const rows = []
|
|
for (let i = 1; i < lines.length; i++) {
|
|
const values = _parseCsvLine(lines[i])
|
|
const row = {}
|
|
for (let j = 0; j < headers.length; j++) {
|
|
row[headers[j]] = values[j] !== undefined ? values[j] : ''
|
|
}
|
|
rows.push(row)
|
|
}
|
|
return { headers, rows }
|
|
}
|
|
|
|
let tableId = ''
|
|
|
|
if (!msg) {
|
|
let headers = []
|
|
let rows = []
|
|
try {
|
|
const parsed = parseStandardCsv(csvText)
|
|
headers = parsed.headers
|
|
rows = parsed.rows
|
|
} catch (err) {
|
|
msg = 'Error parsing CSV: ' + err.message
|
|
}
|
|
|
|
if (!msg) {
|
|
// Convert formatted date/datetime/time strings back to SAS numeric values
|
|
// using the target table's column formats (from the `table` query param).
|
|
// The base table lives in its own libref's data dir.
|
|
const baseDataDir = libDataDir(libref)
|
|
const schema = loadTableSchema(baseDataDir, dsn)
|
|
if (schema.length > 0) {
|
|
const colDdtypes = {}
|
|
for (const col of schema) colDdtypes[col.name.toUpperCase()] = getDdType(col)
|
|
rows = rows.map((row) => {
|
|
const converted = {}
|
|
for (const key of Object.keys(row)) {
|
|
const ddtype = colDdtypes[key.toUpperCase()]
|
|
converted[key] = ddtype ? parseFormattedToSas(row[key], ddtype) : row[key]
|
|
}
|
|
return converted
|
|
})
|
|
}
|
|
|
|
// Validate that the CSV headers match the base table columns (mirrors
|
|
// the mp_abort check for missing columns in loadfile.sas).
|
|
// TXTEMPORAL tables have VAR_TXFROM/VAR_TXTO (and optionally
|
|
// VAR_PROCESSED) columns that are computed at load time, never
|
|
// provided in the uploaded file - they must be excluded from the
|
|
// missing-cols check, same as the real loadfile.sas does.
|
|
if (schema.length > 0 && headers.length > 0) {
|
|
// Load MPE_TABLES to find the temporal column names for this table
|
|
const mpeTablesLoader = makeTableLoader(mpeDataDir)
|
|
const mpeTablesData = mpeTablesLoader('MPE_TABLES')
|
|
let computedCols = []
|
|
if (mpeTablesData && mpeTablesData.rows) {
|
|
const tableEntry = mpeTablesData.rows.find(r =>
|
|
r.libref === libref && r.dsn === dsn
|
|
)
|
|
if (tableEntry) {
|
|
if (tableEntry.var_txfrom) computedCols.push(tableEntry.var_txfrom.toUpperCase())
|
|
if (tableEntry.var_txto) computedCols.push(tableEntry.var_txto.toUpperCase())
|
|
if (tableEntry.var_processed) computedCols.push(tableEntry.var_processed.toUpperCase())
|
|
// loadfile.sas drops the RK from the comparison for a table
|
|
// with rk_underlying (the loader derives it), so an upload
|
|
// for one legitimately omits it.
|
|
if (tableEntry.rk_underlying && tableEntry.buskey) {
|
|
computedCols.push(String(tableEntry.buskey).toUpperCase())
|
|
}
|
|
}
|
|
}
|
|
const baseCols = schema.map((c) => c.name.toUpperCase())
|
|
.filter(c => !computedCols.includes(c))
|
|
const receivedCols = headers.map((h) => h.toUpperCase().trim())
|
|
const missing = baseCols.filter((c) => !receivedCols.includes(c))
|
|
if (missing.length > 0) {
|
|
msg = 'Expected cols: <b>' + baseCols.join(' ') + '</b>' +
|
|
'</br>Received cols: <b>' + receivedCols.join(' ') + '</b>' +
|
|
'</br>Missing cols: <b>' + missing.join(' ') + '</b>'
|
|
}
|
|
}
|
|
|
|
// Reject empty data (mirrors the "no relevant data found" check).
|
|
if (!msg && rows.length === 0) {
|
|
msg = 'No relevant data found in file "' + csvName + '"'
|
|
}
|
|
}
|
|
|
|
if (!msg) {
|
|
// ─── POST_EDIT_HOOK ──────────────────────────────────────────────────
|
|
// Mirrors mpe_loader.sas: the hook runs before the MPE_SUBMIT record is
|
|
// written, while LIBREF / DS are still ordinary macro variables, so it
|
|
// can re-point the changeset at a different table. The mock eval()s the
|
|
// hook's .js counterpart in this scope so it can reassign libref / dsn.
|
|
const hookTables = mpeLoadTableData('MPE_TABLES')
|
|
const hookReg = (hookTables && hookTables.rows)
|
|
? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
|
|
: null
|
|
if (hookReg && hookReg.post_edit_hook) {
|
|
const hookSrc = mockHookSource(hookReg.post_edit_hook)
|
|
if (hookSrc) {
|
|
try {
|
|
eval(hookSrc)
|
|
} catch (err) {
|
|
console.log(
|
|
'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
|
|
)
|
|
}
|
|
libds = libref + '.' + dsn
|
|
}
|
|
}
|
|
|
|
tableId = makeTableId()
|
|
// ─── RK assignment (mpe_loader) ──────────────────────────────────────
|
|
// The uploaded file carries the business key and no RK (see the
|
|
// expected-columns check above), so the loader derives it before the
|
|
// rows are staged.
|
|
const finalTables = mpeLoadTableData('MPE_TABLES')
|
|
const finalReg =
|
|
finalTables && finalTables.rows
|
|
? finalTables.rows.find((r) => r.libref === libref && r.dsn === dsn)
|
|
: null
|
|
const rkName = mpeAssignRetainedKey({
|
|
rows,
|
|
tableReg: finalReg,
|
|
baseData: makeTableLoader(libDataDir(libref))(dsn)
|
|
})
|
|
if (rkName && !headers.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
|
|
headers.push(rkName)
|
|
}
|
|
stageSubmission({
|
|
dataDir: libDataDir(libref),
|
|
stagingDir,
|
|
tableId,
|
|
libds,
|
|
libref,
|
|
dsn,
|
|
message: 'File upload: ' + csvName,
|
|
headers,
|
|
rows
|
|
})
|
|
}
|
|
}
|
|
|
|
// ─── Respond ─────────────────────────────────────────────────────────────────
|
|
|
|
webOutOpen()
|
|
if (msg) {
|
|
webOutObj([{ MSG: msg, MAC: 'loadfile' }], 'sasjsAbort')
|
|
webOutObj([{ STATUS: 'ERROR', DSID: '' }], 'sasparams')
|
|
} else {
|
|
webOutObj([{ STATUS: 'SUCCESS', DSID: tableId, URL: '' }], 'sasparams')
|
|
}
|
|
webOutClose()
|