dc
1b13120839
#359 has landed, so the CLI bump this branch was waiting on is now in the base and the merge brings it in. One conflict, in client/cypress/support/commands.ts: both sides reworked loginAndUpdateValidKey. The base version decides from the body captured before login, which is where the original bug was - on an estate that shows a login form that capture has no nav tree, so the command early-returned and never applied a key, and the upload specs then found no file input. This branch re-reads the loaded page and treats the licence banner as the signal. Kept this side; prettier re-indented the result. Every other file merged automatically, and the hookErrmsg veto in stagedata.js and loadfile.js survives the merge.
257 lines
11 KiB
JavaScript
257 lines
11 KiB
JavaScript
const nodePath = require('path')
|
|
|
|
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
|
|
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
|
|
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
|
|
|
|
// Load shared DC mock utilities
|
|
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
|
|
|
|
// DC database location comes from services/settings.js (written by makedata)
|
|
const dcConf = loadDcSettings(driveRoot, appLoc)
|
|
const dcLibref = dcConf.dcLibref
|
|
const mpeDataDir = dcConf.dataDir
|
|
const stagingDir = nodePath.resolve(mpeDataDir, 'staging')
|
|
|
|
const mpeLoadTableData = makeTableLoader(mpeDataDir)
|
|
|
|
// ─── Parse input ──────────────────────────────────────────────────────────────
|
|
// The file arrives as an uploaded fileref (_WEBIN_FILEREF1/2 - it may be either
|
|
// position, so pick the one whose filename ends in .csv), and the target table
|
|
// arrives as the `table` URL param. Mirrors loadfile.sas.
|
|
|
|
// Debug aid: log the uploaded filerefs (mirrors `%put _all_` in the SAS
|
|
// original) so upload issues are diagnosable from the session log.
|
|
for (let i = 1; i <= 10; i++) {
|
|
let fname
|
|
try {
|
|
fname = eval('typeof _WEBIN_FILENAME' + i + ' !== "undefined" ? String(_WEBIN_FILENAME' + i + ') : null')
|
|
} catch (e) { fname = null }
|
|
if (fname === null) break
|
|
let frefLen
|
|
try {
|
|
frefLen = eval('_WEBIN_FILEREF' + i + ' ? _WEBIN_FILEREF' + i + '.length : 0')
|
|
} catch (e) { frefLen = 0 }
|
|
console.log('_WEBIN_FILENAME' + i + ': ' + fname + ' (' + frefLen + ' bytes)')
|
|
}
|
|
|
|
let csvText = ''
|
|
let csvName = ''
|
|
|
|
// The filerefs are module-level consts in the runtime preamble, so dynamic
|
|
// access needs eval (same approach as fetchRaw in dcMockUtils).
|
|
for (let i = 1; i <= 10; i++) {
|
|
let fname
|
|
try {
|
|
fname = eval('typeof _WEBIN_FILENAME' + i + ' !== "undefined" ? String(_WEBIN_FILENAME' + i + ') : ""')
|
|
} catch (e) {
|
|
break
|
|
}
|
|
if (!fname) break
|
|
if (fname.toUpperCase().endsWith('.CSV')) {
|
|
try {
|
|
csvText = eval('_WEBIN_FILEREF' + i + '.toString()')
|
|
csvName = fname
|
|
} catch (e) { /* not defined */ }
|
|
break
|
|
}
|
|
}
|
|
|
|
let libds = ''
|
|
if (typeof table !== 'undefined' && table) libds = String(table).toUpperCase()
|
|
// captured before any hook can re-point the submission: the cleaned CSV is
|
|
// named after the original table, as loadfile.sas names it
|
|
const origLibds = libds
|
|
let libref = libds.split('.')[0] || dcLibref
|
|
let dsn = libds.split('.')[1] || ''
|
|
|
|
// ─── Validate (mirrors the mp_abort checks in loadfile.sas) ─────────────────
|
|
|
|
let msg = ''
|
|
if (!csvText) {
|
|
msg = 'File "' + csvName + '" must be a CSV! (Comma separated with .csv extension)'
|
|
} else if (!libds || !dsn) {
|
|
msg = 'Missing: table'
|
|
}
|
|
|
|
// ─── Parse the CSV and stage it ──────────────────────────────────────────────
|
|
// The uploaded CSV is a standard comma-separated file (not the SASjs adapter's
|
|
// %webout(FETCH) format), so it must be parsed with a plain CSV parser, not
|
|
// the parseCsv/parseCsvHeader helpers from dcMockUtils (which expect
|
|
// space-separated "COL:$char10." headers). Column formats are then loaded
|
|
// from the target table's schema (identified by the `table` query parameter)
|
|
// so that formatted date/datetime/time strings are converted back to SAS
|
|
// numeric values, mirroring loadfile.sas + mpe_loader.sas.
|
|
|
|
function parseStandardCsv(text) {
|
|
const lines = text.split(/\r?\n/).filter((l) => l.length > 0)
|
|
if (lines.length < 1) return { headers: [], rows: [] }
|
|
const headers = _parseCsvLine(lines[0])
|
|
const rows = []
|
|
for (let i = 1; i < lines.length; i++) {
|
|
const values = _parseCsvLine(lines[i])
|
|
const row = {}
|
|
for (let j = 0; j < headers.length; j++) {
|
|
row[headers[j]] = values[j] !== undefined ? values[j] : ''
|
|
}
|
|
rows.push(row)
|
|
}
|
|
return { headers, rows }
|
|
}
|
|
|
|
let tableId = ''
|
|
|
|
if (!msg) {
|
|
let headers = []
|
|
let rows = []
|
|
try {
|
|
const parsed = parseStandardCsv(csvText)
|
|
headers = parsed.headers
|
|
rows = parsed.rows
|
|
} catch (err) {
|
|
msg = 'Error parsing CSV: ' + err.message
|
|
}
|
|
|
|
if (!msg) {
|
|
// Convert formatted date/datetime/time strings back to SAS numeric values
|
|
// using the target table's column formats (from the `table` query param).
|
|
// The base table lives in its own libref's data dir.
|
|
const baseDataDir = libDataDir(libref)
|
|
const schema = loadTableSchema(baseDataDir, dsn)
|
|
if (schema.length > 0) {
|
|
const colDdtypes = {}
|
|
for (const col of schema) colDdtypes[col.name.toUpperCase()] = getDdType(col)
|
|
rows = rows.map((row) => {
|
|
const converted = {}
|
|
for (const key of Object.keys(row)) {
|
|
const ddtype = colDdtypes[key.toUpperCase()]
|
|
converted[key] = ddtype ? parseFormattedToSas(row[key], ddtype) : row[key]
|
|
}
|
|
return converted
|
|
})
|
|
}
|
|
|
|
// Validate that the CSV headers match the base table columns (mirrors
|
|
// the mp_abort check for missing columns in loadfile.sas).
|
|
// TXTEMPORAL tables have VAR_TXFROM/VAR_TXTO (and optionally
|
|
// VAR_PROCESSED) columns that are computed at load time, never
|
|
// provided in the uploaded file - they must be excluded from the
|
|
// missing-cols check, same as the real loadfile.sas does.
|
|
if (schema.length > 0 && headers.length > 0) {
|
|
// Load MPE_TABLES to find the temporal column names for this table
|
|
const mpeTablesLoader = makeTableLoader(mpeDataDir)
|
|
const mpeTablesData = mpeTablesLoader('MPE_TABLES')
|
|
let computedCols = []
|
|
if (mpeTablesData && mpeTablesData.rows) {
|
|
const tableEntry = mpeTablesData.rows.find(r =>
|
|
r.libref === libref && r.dsn === dsn
|
|
)
|
|
if (tableEntry) {
|
|
if (tableEntry.var_txfrom) computedCols.push(tableEntry.var_txfrom.toUpperCase())
|
|
if (tableEntry.var_txto) computedCols.push(tableEntry.var_txto.toUpperCase())
|
|
if (tableEntry.var_processed) computedCols.push(tableEntry.var_processed.toUpperCase())
|
|
// loadfile.sas drops the RK from the comparison for a table
|
|
// with rk_underlying (the loader derives it), so an upload
|
|
// for one legitimately omits it.
|
|
if (tableEntry.rk_underlying && tableEntry.buskey) {
|
|
computedCols.push(String(tableEntry.buskey).toUpperCase())
|
|
}
|
|
}
|
|
}
|
|
const baseCols = schema.map((c) => c.name.toUpperCase())
|
|
.filter(c => !computedCols.includes(c))
|
|
const receivedCols = headers.map((h) => h.toUpperCase().trim())
|
|
const missing = baseCols.filter((c) => !receivedCols.includes(c))
|
|
if (missing.length > 0) {
|
|
msg = 'Expected cols: <b>' + baseCols.join(' ') + '</b>' +
|
|
'</br>Received cols: <b>' + receivedCols.join(' ') + '</b>' +
|
|
'</br>Missing cols: <b>' + missing.join(' ') + '</b>'
|
|
}
|
|
}
|
|
|
|
// Reject empty data (mirrors the "no relevant data found" check).
|
|
if (!msg && rows.length === 0) {
|
|
msg = 'No relevant data found in file "' + csvName + '"'
|
|
}
|
|
}
|
|
|
|
if (!msg) {
|
|
// ─── POST_EDIT_HOOK ──────────────────────────────────────────────────
|
|
// Mirrors mpe_loader.sas: the hook runs before the MPE_SUBMIT record is
|
|
// written, while LIBREF / DS are still ordinary macro variables, so it
|
|
// can re-point the changeset at a different table. The mock eval()s the
|
|
// hook's .js counterpart in this scope so it can reassign libref / dsn.
|
|
// A validating hook rejects the submission by setting hookErrmsg (see
|
|
// hooks/mpe_derived_postedit.js), which stops the staging below.
|
|
let hookErrmsg = ''
|
|
const hookTables = mpeLoadTableData('MPE_TABLES')
|
|
const hookReg = (hookTables && hookTables.rows)
|
|
? hookTables.rows.find(r => r.libref === libref && r.dsn === dsn)
|
|
: null
|
|
if (hookReg && hookReg.post_edit_hook) {
|
|
const hookSrc = mockHookSource(hookReg.post_edit_hook)
|
|
if (hookSrc) {
|
|
// a hook reads and normalises the submitted rows as jsDataRows
|
|
// (stagedata.js names them that); here they are `rows`
|
|
const jsDataRows = rows
|
|
try {
|
|
eval(hookSrc)
|
|
} catch (err) {
|
|
// a hook that throws has not validated anything, so the
|
|
// submission is refused rather than staged unchecked
|
|
hookErrmsg =
|
|
'post_edit_hook ' + hookReg.post_edit_hook + ' failed: ' + err.message
|
|
}
|
|
libds = libref + '.' + dsn
|
|
}
|
|
}
|
|
if (hookErrmsg) msg = hookErrmsg
|
|
|
|
tableId = makeTableId()
|
|
// ─── RK assignment (mpe_loader) ──────────────────────────────────────
|
|
// The uploaded file carries the business key and no RK (see the
|
|
// expected-columns check above), so the loader derives it before the
|
|
// rows are staged.
|
|
const finalTables = mpeLoadTableData('MPE_TABLES')
|
|
const finalReg =
|
|
finalTables && finalTables.rows
|
|
? finalTables.rows.find((r) => r.libref === libref && r.dsn === dsn)
|
|
: null
|
|
const rkName = mpeAssignRetainedKey({
|
|
rows,
|
|
tableReg: finalReg,
|
|
baseData: makeTableLoader(libDataDir(libref))(dsn)
|
|
})
|
|
if (rkName && !headers.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
|
|
headers.push(rkName)
|
|
}
|
|
if (!msg) stageSubmission({
|
|
dataDir: libDataDir(libref),
|
|
stagingDir,
|
|
tableId,
|
|
libds,
|
|
libref,
|
|
dsn,
|
|
message: 'File upload: ' + csvName,
|
|
headers,
|
|
rows,
|
|
// loadfile.sas leaves the cleaned CSV (%mp_cleancsv) in the approval
|
|
// folder, named after the table as the request found it. It copies
|
|
// the upload as BKP_<name> only when a second, non-CSV fileref came
|
|
// with the request, which the mock does not accept - so no BKP here.
|
|
auditFiles: [{ name: origLibds + '.csv', content: csvText }]
|
|
})
|
|
}
|
|
}
|
|
|
|
// ─── Respond ─────────────────────────────────────────────────────────────────
|
|
|
|
webOutOpen()
|
|
if (msg) {
|
|
webOutObj([{ MSG: msg, MAC: 'loadfile' }], 'sasjsAbort')
|
|
webOutObj([{ STATUS: 'ERROR', DSID: '' }], 'sasparams')
|
|
} else {
|
|
webOutObj([{ STATUS: 'SUCCESS', DSID: tableId, URL: '' }], 'sasparams')
|
|
}
|
|
webOutClose()
|