fix: dynamic js mocks
Build / Build-and-ng-test (pull_request) Successful in 5m16s
Lighthouse Checks / lighthouse (pull_request) Successful in 21m35s
Build / Build-and-test-development (pull_request) Failing after 33m28s

This commit is contained in:
hermes committed 2026-08-28 00:35:31 +01:00
1 parent 0082828543
commit 3b9344cb60
14 files changed
+584 -377

No files matched your search

+10 -2
View File
@@ -125,8 +125,16 @@ jobs:
cd ./sas/mocks/sasjs
npm install -g @sasjs/cli
npm install -g replace-in-files-cli
# Remove any previous deployment (drive folder delete API) so the
# deploy and makedata start from a clean appLoc
curl -sS -X DELETE "http://localhost:5000/SASjsApi/drive/folder/?_folderPath=/Public/app/dc"
sasjs cbd -t server-ci
# sasjs request services/admin/makedata -t server-ci -d ./deploy/makeData4GL.json -c ./deploy/requestConfig.json -o ./output.json
# Seed the DC database (mock data files on the drive).
# makedata replies with HTML (it is normally called as a URL redirect),
# which the CLI reports as "invalid Json string" - ignore that.
# NOTE: -d paths resolve from the sasjs project root (sas/mocks),
# not the cwd (sas/mocks/sasjs).
sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true
- name: Prepare and run frontend and cypress
timeout-minutes: 35
@@ -139,7 +147,7 @@ jobs:
npm run postinstall
# Prepare index.html to SASJS local
replace-in-files --regex='serverUrl=".*?"' --replacement='serverUrl="http://localhost:5000"' ./src/index.html
replace-in-files --regex='appLoc=".*?"' --replacement='appLoc="/Public/app/devtest"' ./src/index.html
replace-in-files --regex='appLoc=".*?"' --replacement='appLoc="/Public/app/dc"' ./src/index.html
replace-in-files --regex='serverType=".*?"' --replacement='serverType="SASJS"' ./src/index.html
replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
cat ./cypress.config.ts
+10 -2
View File
@@ -125,8 +125,16 @@ jobs:
cd ./sas/mocks/sasjs
npm install -g @sasjs/cli
npm install -g replace-in-files-cli
# Remove any previous deployment (drive folder delete API) so the
# deploy and makedata start from a clean appLoc
curl -sS -X DELETE "http://localhost:5000/SASjsApi/drive/folder/?_folderPath=/Public/app/dc"
sasjs cbd -t server-ci
# sasjs request services/admin/makedata -t server-ci -d ./deploy/makeData4GL.json -c ./deploy/requestConfig.json -o ./output.json
# Seed the DC database (mock data files on the drive).
# makedata replies with HTML (it is normally called as a URL redirect),
# which the CLI reports as "invalid Json string" - ignore that.
# NOTE: -d paths resolve from the sasjs project root (sas/mocks),
# not the cwd (sas/mocks/sasjs).
sasjs request services/admin/makedata -t server-ci -d deploy/makedata.json -o ./makedata_out.json || true
- name: Prepare and run frontend and cypress
run: |
@@ -138,7 +146,7 @@ jobs:
npm run postinstall
# Prepare index.html to SASJS local
replace-in-files --regex='serverUrl=".*?"' --replacement='serverUrl="http://localhost:5000"' ./src/index.html
replace-in-files --regex='appLoc=".*?"' --replacement='appLoc="/Public/app/devtest"' ./src/index.html
replace-in-files --regex='appLoc=".*?"' --replacement='appLoc="/Public/app/dc"' ./src/index.html
replace-in-files --regex='serverType=".*?"' --replacement='serverType="SASJS"' ./src/index.html
replace-in-files --regex='"hosturl".*' --replacement='hosturl:"http://localhost:4200",' ./cypress.config.ts
cat ./cypress.config.ts
+2 -2
View File
@@ -91,7 +91,7 @@ npm ci
sasjs cbd -t server-ci
```
This compiles, builds, and deploys the JS mock services plus the Angular frontend (as a streaming app) to the SASjs Server. The `server-ci` target in `sas/mocks/sasjs/sasjsconfig.json` points to `http://localhost:5000` with appLoc `/Public/app/devtest` and stream service name `clickme`.
This compiles, builds, and deploys the JS mock services plus the Angular frontend (as a streaming app) to the SASjs Server. The `server-ci` target in `sas/mocks/sasjs/sasjsconfig.json` points to `http://localhost:5000` with appLoc `/Public/app/dc` and stream service name `clickme`.
After deploy, the streamed app is available at: `http://localhost:5000/AppStream/clickme/`
@@ -101,7 +101,7 @@ For frontend development with live reload, point `client/src/index.html` at the
```
cd client
# Edit src/index.html: set serverUrl="http://localhost:5000", appLoc="/Public/app/devtest", serverType="SASJS"
# Edit src/index.html: set serverUrl="http://localhost:5000", appLoc="/Public/app/dc", serverType="SASJS"
npm start
```
+7
View File
@@ -0,0 +1,7 @@
{
"SASControlTable": [
{
"ADMIN": "AllUsers"
}
]
}
+8 -5
View File
@@ -19,9 +19,6 @@ const appLoc = nodePath.join(..._program.split('services')[0].split('/'))
// This is browsable in SASjs Studio and persists across service redeployes
const dataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', dcLibref)
// Settings file lives alongside the table files
const settingsStore = nodePath.resolve(dataDir, 'settings.json')
// Ensure the data directory exists (create parent dirs as needed)
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true })
@@ -705,7 +702,10 @@ console.log('DC data dir: ' + dataDir)
console.log('DCLIB: ' + dcLibref)
console.log('ADMIN: ' + adminGroup)
// Write settings (what the real settings.sas would contain)
// Write settings (what the real settings.sas would contain) as a JS module at
// services/settings.js, mirroring how makedata.sas writes
// services/public/settings.sas on the real backend. Services eval() it to
// pick up dc_libref, dc_admin_group, dc_staging_area and dcpath.
const settings = {
dc_libref: dcLibref,
dc_admin_group: adminGroup,
@@ -714,7 +714,10 @@ const settings = {
mpelib: dcLibref,
dcpath: dataDir
}
fs.writeFileSync(settingsStore, JSON.stringify(settings, null, 2))
const settingsFile = nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'settings.js')
// `var` (not const/let) so that eval'ing the file in a service's scope makes
// dcSettings visible to the reader, same as the dcMockUtils functions.
fs.writeFileSync(settingsFile, 'var dcSettings = ' + JSON.stringify(settings, null, 2) + '\n')
// Delete makedata from the Drive so the frontend polling detects completion
// The frontend checks if makedata.sas is still in appLoc/services/admin/
+65 -31
View File
@@ -1,39 +1,73 @@
const path = require('path')
const nodePath = require('path')
let appLoc = path.join(..._program.split('services')[0].split('/'))
let licenceKey = ''
let activationKey = ''
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
let writeError = 0
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
if (_webin_fileref1) {
const fileText = _webin_fileref1.toString()
const split = fileText.split('\n')[1].split(',')
activationKey = split[0]
licenceKey = split[1]
// DC database location comes from services/public/settings.js (written by makedata)
const dcConf = loadDcSettings(driveRoot, appLoc)
const dcLibref = dcConf.dcLibref
const dataDir = dcConf.dataDir
const loadTableData = makeTableLoader(dataDir)
// ─── Parse the keyupload table (%webout(FETCH) equivalent) ────────────────────
const keyRow = fetchTable('keyupload')[0] || {}
let activationKey = (keyRow.ACTIVATION_KEY || '').toString()
let licenceKey = (keyRow.LICENCE_KEY || '').toString()
// ─── Validate key lengths (mirrors the mp_abort checks in registerkey.sas) ───
let msg = 'SUCCESS'
if (activationKey.length < 10) {
msg = 'Invalid activation_key'
} else if (licenceKey.length < 10) {
msg = 'Invalid licencekey'
}
const sessionStoragePath = path.resolve(__dirname, '..', '..', 'drive', 'files', appLoc, 'mock-storage')
// ─── Upsert keys into mpe_config (TXTEMPORAL, PK = VAR_SCOPE VAR_NAME) ───────
if (!fs.existsSync(sessionStoragePath)){
fs.mkdirSync(sessionStoragePath);
const TX_HIGH = 253717919999
if (msg === 'SUCCESS') {
let writeError = 0
try {
const configFile = nodePath.resolve(dataDir, 'mpe_config.json')
const tableData = loadTableData('MPE_CONFIG')
if (tableData && tableData.rows) {
const now = Math.floor(Date.now() / 1000)
// Close current rows: set tx_to to now for the two key variables
for (const row of tableData.rows) {
if (
row.var_scope === 'DC' &&
(row.var_name === 'DC_LICENCE_KEY' || row.var_name === 'DC_ACTIVATION_KEY') &&
row.tx_to === TX_HIGH
) {
row.tx_to = now
}
}
// Append the new rows (as bitemporal_dataloader does with LOADTYPE=TXTEMPORAL)
tableData.rows.push({ tx_from: now, tx_to: TX_HIGH, var_scope: 'DC', var_name: 'DC_ACTIVATION_KEY', var_value: activationKey, var_active: 1, var_desc: 'Activation Key' })
tableData.rows.push({ tx_from: now, tx_to: TX_HIGH, var_scope: 'DC', var_name: 'DC_LICENCE_KEY', var_value: licenceKey, var_active: 1, var_desc: 'Licence Key' })
fs.writeFileSync(configFile, JSON.stringify(tableData, null, 2))
} else {
writeError = 1
}
} catch (err) {
writeError = 1
}
if (writeError) msg = 'Error writing licence file'
}
const licenceStore = path.resolve(sessionStoragePath, 'licence.json')
const json = {
licenceKey: licenceKey,
activationKey: activationKey
}
try {
fs.writeFileSync(licenceStore, JSON.stringify(json, null, 2))
} catch (err) {
writeError = 1
}
if (writeError) {
_webout = `{ "return": [{ "MSG": "Error writing licence file" }] }`
} else {
_webout = `{ "return": [{ "MSG": "SUCCESS" }] }`
}
webOutOpen()
webOutObj([{ MSG: msg }], 'return')
webOutClose()
+79 -36
View File
@@ -1,38 +1,81 @@
_webout = `{"SYSDATE" : "27SEP22"
,"SYSTIME" : "07:56"
, "fromsas":
[
{
"RESPONSE": "SUCCESS!",
"TABLE_ID": "DC20220927T075630819_560700_2598",
"REVIEWED_BY_NM": "sasdemo",
"BASE_TABLE": "DC_JSLIB.MPE_X_TEST",
"REVIEW_STATUS_ID": "REJECTED",
"REVIEWED_ON_DTTM": "27SEP22:07:56:53.47",
"REVIEW_REASON_TXT": "Test"
}
]
,"_DEBUG" : ""
,"_METAUSER": "sasdemo@SAS"
,"_METAPERSON": "sasdemo"
,"_PROGRAM" : "/Projects/app/dc/services/approvers/rejection"
,"AUTOEXEC" : "D%3A%5Copt%5Csasinside%5CConfig%5CLev1%5CSASApp%5CStoredProcessServer%5Cautoexec.sas"
,"MF_GETUSER" : "sasdemo"
,"SYSCC" : "0"
,"SYSENCODING" : "wlatin1"
,"SYSERRORTEXT" : ""
,"SYSHOSTNAME" : "SAS"
,"SYSPROCESSID" : "41DD80A91145F3B64092AC0000000000"
,"SYSPROCESSMODE" : "SAS Stored Process Server"
,"SYSPROCESSNAME" : ""
,"SYSJOBID" : "25980"
,"SYSSCPL" : "Linunx"
,"SYSSITE" : "123"
,"SYSUSERID" : "sassrv"
,"SYSVLONG" : "9.04.01M7P080520"
,"SYSWARNINGTEXT" : "ENCODING option ignored for files opened with RECFM=N."
,"END_DTTM" : "2022-09-27T07:56:53.688000"
,"MEMSIZE" : "82GB"
const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
// DC database location comes from services/settings.js (written by makedata)
const dcConf = loadDcSettings(driveRoot, appLoc)
const dataDir = dcConf.dataDir
const loadTableData = makeTableLoader(dataDir)
function saveTableData(tableName, tableData) {
const file = nodePath.resolve(dataDir, tableName.toLowerCase() + '.json')
fs.writeFileSync(file, JSON.stringify(tableData, null, 2))
}
`
// ─── Parse input (BrowserParams: TABLE, STP_REASON) ──────────────────────────
const _bpRow = fetchTable('BrowserParams')[0] || {}
const tableId = (_bpRow.TABLE || '').toString()
const reason = (_bpRow.STP_REASON || '').toString()
// ─── Load the submission and mark it rejected ────────────────────────────────
const submitData = loadTableData('MPE_SUBMIT') || { rows: [] }
const submitRow = submitData.rows.find((r) => r.TABLE_ID === tableId)
const userNm =
typeof _METAUSER !== 'undefined' && _METAUSER ? _METAUSER : 'sasdemo'
const now = Date.now() / 1000
let fromsas = []
if (submitRow) {
// mirror rejection.sas: reject only a still-submitted table
if (submitRow.SUBMIT_STATUS_CD === 'SUBMITTED') {
submitRow.SUBMIT_STATUS_CD = 'REJECTED'
submitRow.NUM_OF_APPROVALS_REMAINING = 0
submitRow.REVIEWED_BY_NM = userNm
submitRow.REVIEWED_ON_DTTM = now
saveTableData('MPE_SUBMIT', submitData)
const reviewData = loadTableData('MPE_REVIEW') || { rows: [] }
reviewData.rows.push({
TABLE_ID: tableId,
BASE_TABLE: submitRow.BASE_LIB + '.' + submitRow.BASE_DS,
REVIEW_STATUS_ID: 'REJECTED',
REVIEWED_BY_NM: userNm,
REVIEWED_ON_DTTM: now,
REVIEW_REASON_TXT: reason
})
saveTableData('MPE_REVIEW', reviewData)
}
fromsas = [
{
RESPONSE: submitRow.SUBMIT_STATUS_CD === 'REJECTED' ? 'SUCCESS!' : 'TABLE NOT SUBMITTED',
TABLE_ID: tableId,
REVIEWED_BY_NM: submitRow.REVIEWED_BY_NM || userNm,
BASE_TABLE: submitRow.BASE_LIB + '.' + submitRow.BASE_DS,
REVIEW_STATUS_ID: submitRow.SUBMIT_STATUS_CD,
REVIEWED_ON_DTTM: submitRow.REVIEWED_ON_DTTM,
REVIEW_REASON_TXT: reason
}
]
} else {
fromsas = [
{
RESPONSE: 'TABLE NOT FOUND',
TABLE_ID: tableId
}
]
}
webOutOpen()
webOutObj(fromsas, 'fromsas')
webOutClose()
+151
View File
@@ -89,6 +89,17 @@ function fetchTable(tableName) {
return parseCsv(raw)
}
/**
* Parses just the header line of a SASjs adapter CSV string
* ("COL1:$char10. COL2:$char8." -> ['COL1', 'COL2']).
*/
function parseCsvHeader(csvText) {
if (!csvText) return []
const lines = csvText.split(/\r?\n/)
if (lines.length < 1) return []
return lines[0].trim().split(/\s+/).map((h) => h.split(':')[0])
}
/**
* Parses a SASjs adapter CSV string into an array of row objects.
* Header line: "COL1:$char10. COL2:$char8." (space-separated, name before colon)
@@ -296,6 +307,146 @@ function sasVarsEntry(col) {
return { format: fmt, label: col.label || col.name, length: String(col.length), type: col.type === 'N' ? 'num' : 'char' }
}
/**
* Stages a submission: writes staged.json under staging/<tableId>/, copies the
* session audit items, and inserts the MPE_SUBMIT row. Shared by the
* editors/stagedata (grid submit) and editors/loadfile (CSV upload) services.
*
* @param {Object} opts - { dataDir, stagingDir, tableId, libds, libref, dsn,
* message, headers, rows, weboutPath }
* @returns {Object} the stageData object written to staged.json
*/
function stageSubmission(opts) {
const { dataDir, stagingDir, tableId, libds, libref, dsn, message, headers, rows } = opts
const stageFolder = nodePath.resolve(stagingDir, tableId)
fs.mkdirSync(stageFolder, { recursive: true })
const stageData = {
table_id: tableId,
libds,
libref,
dsn,
submitted_by_nm:
typeof _METAUSER !== 'undefined' && _METAUSER ? _METAUSER : 'sasdemo',
submitted_on_dttm: Date.now() / 1000,
headers,
rows
}
fs.writeFileSync(
nodePath.resolve(stageFolder, 'staged.json'),
JSON.stringify(stageData, null, 2)
)
// keep audit items from this Stored Program run alongside the staged data
copySessionAuditItems(stageFolder, 'submit')
// Insert a record into MPE_SUBMIT
const loadTableData = makeTableLoader(dataDir)
const submitTable = loadTableData('MPE_SUBMIT') || {
name: 'MPE_SUBMIT',
columns: [],
rows: []
}
submitTable.rows.push({
TABLE_ID: tableId,
SUBMIT_STATUS_CD: 'SUBMITTED',
BASE_LIB: libref,
BASE_DS: dsn,
SUBMITTED_BY_NM: stageData.submitted_by_nm,
SUBMITTED_ON_DTTM: stageData.submitted_on_dttm,
SUBMITTED_REASON_TXT: message || '',
INPUT_OBS: rows.length,
INPUT_VARS: headers.filter((h) => h !== '_____DELETE__THIS__RECORD_____').length,
NUM_OF_APPROVALS_REQUIRED: 1,
NUM_OF_APPROVALS_REMAINING: 1,
REVIEWED_BY_NM: '',
REVIEWED_ON_DTTM: null
})
const submitFile = nodePath.resolve(dataDir, 'mpe_submit.json')
fs.writeFileSync(submitFile, JSON.stringify(submitTable, null, 2))
return stageData
}
/**
* Copies audit items from the Stored Program's session directory (its
* current working directory at runtime) into the staging subfolder:
* - log.log -> <prefix>.log (runtime log of the submission)
* - code.js -> <prefix>_code.js (program source as executed)
* - reqHeaders.txt -> <prefix>_reqHeaders.txt
*/
function copySessionAuditItems(stageFolder, prefix) {
try {
const sessionDir = nodePath.dirname(weboutPath)
const copies = [
['log.log', prefix + '.log'],
['code.js', prefix + '_code.js'],
['reqHeaders.txt', prefix + '_reqHeaders.txt']
]
for (const [src, dest] of copies) {
const srcPath = nodePath.resolve(sessionDir, src)
if (fs.existsSync(srcPath)) {
fs.copyFileSync(srcPath, nodePath.resolve(stageFolder, dest))
}
}
} catch (err) {
console.log('Error copying session audit items: ' + err.message)
}
}
/**
* Generates a DC table_id in the same format as the real backend
* (DC<yyyymmdd-hhmmss.sss>_<rand6>_<pid4>).
*/
function makeTableId() {
const now = new Date()
const pad = (n, w) => String(n).padStart(w, '0')
const dtStr =
now.getFullYear() +
pad(now.getMonth() + 1, 2) +
pad(now.getDate(), 2) +
'-' +
pad(now.getHours(), 2) +
pad(now.getMinutes(), 2) +
pad(now.getSeconds(), 2) +
'.' +
pad(now.getMilliseconds(), 3)
const rand6 = pad(Math.floor(Math.random() * 1000000), 6)
const pid4 = pad(process.pid % 10000, 4)
return `DC${dtStr}_${rand6}_${pid4}`
}
// ─── DC settings (services/settings.js) ──────────────────────────────────────
/**
* Loads the DC settings written by makedata to services/settings.js
* (the JS equivalent of the real backend's settings.sas).
* Returns { dcLibref, dataDir, dcAdminGroup, dcpath } - with defaults when
* makedata has not run yet.
* Requires nodePath, fs, driveRoot and appLoc in the caller's scope.
*/
function loadDcSettings(driveRoot, appLoc) {
const result = {
dcLibref: 'DC_JSLIB',
dataDir: nodePath.resolve(driveRoot, 'files', appLoc, 'data', 'DC_JSLIB'),
dcAdminGroup: null,
dcpath: null
}
try {
const settingsFile = nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'settings.js')
const settingsRaw = fs.readFileSync(settingsFile, {encoding:'utf8'}).toString()
eval(settingsRaw)
if (typeof dcSettings !== 'undefined') {
if (dcSettings.dc_libref) result.dcLibref = dcSettings.dc_libref
if (dcSettings.dc_admin_group) result.dcAdminGroup = dcSettings.dc_admin_group
if (dcSettings.dcpath) result.dcpath = dcSettings.dcpath
result.dataDir = nodePath.resolve(driveRoot, 'files', appLoc, 'data', result.dcLibref)
}
} catch(err) { /* makedata hasn't run - keep defaults */ }
return result
}
// ─── %webout(OPEN/OBJ/CLOSE) emulation ───────────────────────────────────────
/**
@@ -1,3 +1,14 @@
const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
let jsDataFileText = ''
let mpe_tables = {
SYSDATE: '10MAR23',
SYSTIME: '12:56',
@@ -6213,17 +6224,19 @@ let other = {
MEMSIZE: '93GB'
}
if (_webin_filename1.includes('SASControlTable'))
jsDataFileText = _webin_fileref1.toString()
if (_webin_filename2.includes('SASControlTable'))
jsDataFileText = _webin_fileref2.toString()
jsDataFileText = fetchRaw('SASControlTable') || ''
if (jsDataFileText.length > 0) {
if (jsDataFileText.includes('MPE_TABLES')) {
_webout = JSON.stringify(mpe_tables)
} else {
_webout = JSON.stringify(other)
// Send each table from the canned response; webOutClose() supplies the
// system variables (SYSDATE, SYSTIME, _METAUSER etc) that the canned
// objects also carry pre-baked.
const canned = jsDataFileText.includes('MPE_TABLES') ? mpe_tables : other
webOutOpen()
for (const key of Object.keys(canned)) {
if (Array.isArray(canned[key])) webOutObj(canned[key], key)
}
webOutClose()
} else {
console.log('Error reading jsdata input')
}
+126 -31
View File
@@ -1,33 +1,128 @@
_webout = `{"SYSDATE" : "29SEP22"
,"SYSTIME" : "11:57"
, "sasparams":
[
{
"STATUS": "SUCCESS",
"DSID": "DC20220929T115713486_609542_1443"
}
]
,"_DEBUG" : ""
,"_METAUSER": "sasdemo@SAS"
,"_METAPERSON": "sasdemo"
,"_PROGRAM" : "/Projects/app/dc/services/editors/loadfile"
,"AUTOEXEC" : "D%3A%5Copt%5Csasinside%5CConfig%5CLev1%5CSASApp%5CStoredProcessServer%5Cautoexec.sas"
,"MF_GETUSER" : "sasdemo"
,"SYSCC" : "0"
,"SYSENCODING" : "wlatin1"
,"SYSERRORTEXT" : ""
,"SYSHOSTNAME" : "SAS"
,"SYSPROCESSID" : "41DD815FE643C6A840C0278000000000"
,"SYSPROCESSMODE" : "SAS Stored Process Server"
,"SYSPROCESSNAME" : ""
,"SYSJOBID" : "14432"
,"SYSSCPL" : "Linunx"
,"SYSSITE" : "123"
,"SYSUSERID" : "sassrv"
,"SYSVLONG" : "9.04.01M7P080520"
,"SYSWARNINGTEXT" : "ENCODING option ignored for files opened with RECFM=N."
,"END_DTTM" : "2022-09-29T11:57:14.118000"
,"MEMSIZE" : "93GB"
const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
// DC database location comes from services/settings.js (written by makedata)
const dcConf = loadDcSettings(driveRoot, appLoc)
const dcLibref = dcConf.dcLibref
const dataDir = dcConf.dataDir
const stagingDir = nodePath.resolve(dataDir, 'staging')
const loadTableData = makeTableLoader(dataDir)
// ─── Parse input ──────────────────────────────────────────────────────────────
// The file arrives as an uploaded fileref (_WEBIN_FILEREF1/2 - it may be either
// position, so pick the one whose filename ends in .csv), and the target table
// arrives as the `table` URL param. Mirrors loadfile.sas.
// Debug aid: log the uploaded filerefs (mirrors `%put _all_` in the SAS
// original) so upload issues are diagnosable from the session log.
for (let i = 1; i <= 10; i++) {
let fname
try {
fname = eval('typeof _WEBIN_FILENAME' + i + ' !== "undefined" ? String(_WEBIN_FILENAME' + i + ') : null')
} catch (e) { fname = null }
if (fname === null) break
let frefLen
try {
frefLen = eval('_WEBIN_FILEREF' + i + ' ? _WEBIN_FILEREF' + i + '.length : 0')
} catch (e) { frefLen = 0 }
console.log('_WEBIN_FILENAME' + i + ': ' + fname + ' (' + frefLen + ' bytes)')
}
`
let csvText = ''
let csvName = ''
// The filerefs are module-level consts in the runtime preamble, so dynamic
// access needs eval (same approach as fetchRaw in dcMockUtils).
for (let i = 1; i <= 10; i++) {
let fname
try {
fname = eval('typeof _WEBIN_FILENAME' + i + ' !== "undefined" ? String(_WEBIN_FILENAME' + i + ') : ""')
} catch (e) {
break
}
if (!fname) break
if (fname.toUpperCase().endsWith('.CSV')) {
try {
csvText = eval('_WEBIN_FILEREF' + i + '.toString()')
csvName = fname
} catch (e) { /* not defined */ }
break
}
}
let libds = ''
if (typeof table !== 'undefined' && table) libds = String(table).toUpperCase()
const libref = libds.split('.')[0] || dcLibref
const dsn = libds.split('.')[1] || ''
// ─── Validate (mirrors the mp_abort checks in loadfile.sas) ─────────────────
let msg = ''
if (!csvText) {
msg = 'File "' + csvName + '" must be a CSV! (Comma separated with .csv extension)'
} else if (!libds || !dsn) {
msg = 'Missing: table'
}
// ─── Parse the CSV and stage it ──────────────────────────────────────────────
let tableId = ''
if (!msg) {
let headers = []
let rows = []
try {
headers = parseCsvHeader(csvText)
rows = parseCsv(csvText)
} catch (err) {
msg = 'Error parsing CSV: ' + err.message
}
if (!msg) {
// Convert formatted date/datetime/time strings back to SAS numeric values
const schema = loadTableSchema(dataDir, dsn)
if (schema.length > 0) {
const colDdtypes = {}
for (const col of schema) colDdtypes[col.name.toUpperCase()] = getDdType(col)
rows = rows.map((row) => {
const converted = {}
for (const key of Object.keys(row)) {
const ddtype = colDdtypes[key.toUpperCase()]
converted[key] = ddtype ? parseFormattedToSas(row[key], ddtype) : row[key]
}
return converted
})
}
tableId = makeTableId()
stageSubmission({
dataDir,
stagingDir,
tableId,
libds,
libref,
dsn,
message: 'File upload: ' + csvName,
headers,
rows
})
}
}
// ─── Respond ─────────────────────────────────────────────────────────────────
webOutOpen()
if (msg) {
webOutObj([{ MSG: msg, MAC: 'loadfile' }], 'sasjsAbort')
webOutObj([{ STATUS: 'ERROR', DSID: '' }], 'sasparams')
} else {
webOutObj([{ STATUS: 'SUCCESS', DSID: tableId, URL: '' }], 'sasparams')
}
webOutClose()
+17 -126
View File
@@ -16,37 +16,6 @@ eval(
const loadTableData = makeTableLoader(dataDir)
function saveTableData(tableName, tableData) {
const file = nodePath.resolve(dataDir, tableName.toLowerCase() + '.json')
fs.writeFileSync(file, JSON.stringify(tableData, null, 2))
}
/**
* Copies audit items from the Stored Program's session directory (its
* current working directory at runtime) into the staging subfolder:
* - log.log -> submit.log (runtime log of the submission)
* - code.js -> submit_code.js (program source as executed)
* - reqHeaders.txt -> submit_reqHeaders.txt
*/
function copySessionAuditItems(stageFolder, prefix) {
try {
const sessionDir = nodePath.dirname(weboutPath)
const copies = [
['log.log', prefix + '.log'],
['code.js', prefix + '_code.js'],
['reqHeaders.txt', prefix + '_reqHeaders.txt']
]
for (const [src, dest] of copies) {
const srcPath = nodePath.resolve(sessionDir, src)
if (fs.existsSync(srcPath)) {
fs.copyFileSync(srcPath, nodePath.resolve(stageFolder, dest))
}
}
} catch (err) {
console.log('Error copying session audit items: ' + err.message)
}
}
// ─── Parse input ──────────────────────────────────────────────────────────────
const _sctRow = fetchTable('SASControlTable')[0] || {}
@@ -61,41 +30,12 @@ const jsDataRaw = fetchRaw('jsdata')
let jsDataHeaders = []
let jsDataRows = []
if (jsDataRaw) {
try {
const lines = jsDataRaw.split(/\r?\n/)
if (lines.length >= 1) {
jsDataHeaders = lines[0]
.trim()
.split(/\s+/)
.map((h) => h.split(':')[0])
for (let i = 1; i < lines.length; i++) {
const line = lines[i].trim()
if (!line) continue
const values = []
let cur = '',
inQuote = false
for (let j = 0; j < line.length; j++) {
const ch = line[j]
if (ch === '"') {
if (inQuote && line[j + 1] === '"') {
cur += '"'
j++
} else inQuote = !inQuote
} else if (ch === ',' && !inQuote) {
values.push(cur)
cur = ''
} else cur += ch
}
values.push(cur)
const row = {}
for (let k = 0; k < jsDataHeaders.length && k < values.length; k++)
row[jsDataHeaders[k]] = values[k]
jsDataRows.push(row)
}
try {
jsDataHeaders = parseCsvHeader(jsDataRaw)
jsDataRows = parseCsv(jsDataRaw)
} catch (err) {
console.log('Error parsing jsdata: ' + err.message)
}
} catch (err) {
console.log('Error parsing jsdata: ' + err.message)
}
}
// Convert formatted date/datetime/time strings back to SAS numeric values
@@ -113,73 +53,24 @@ if (schema.length > 0) {
})
}
// ─── Generate table_id and stage the data ────────────────────────────────────
// ─── Stage the data ──────────────────────────────────────────────────────────
// One subfolder per submission (DSID) under the staging directory, so that the
// staged data can sit alongside related artifacts (submit/approval logs,
// temp diffs etc) for the audit trail.
const now = new Date()
const pad = (n, w) => String(n).padStart(w, '0')
const dtStr =
now.getFullYear() +
pad(now.getMonth() + 1, 2) +
pad(now.getDate(), 2) +
'-' +
pad(now.getHours(), 2) +
pad(now.getMinutes(), 2) +
pad(now.getSeconds(), 2) +
'.' +
pad(now.getMilliseconds(), 3)
const rand6 = pad(Math.floor(Math.random() * 1000000), 6)
const pid4 = pad(process.pid % 10000, 4)
const tableId = `DC${dtStr}_${rand6}_${pid4}`
const tableId = makeTableId()
const stageFolder = nodePath.resolve(stagingDir, tableId)
fs.mkdirSync(stageFolder, { recursive: true })
const stageData = {
table_id: tableId,
libds,
libref,
dsn,
submitted_by_nm:
typeof _METAUSER !== 'undefined' && _METAUSER ? _METAUSER : 'sasdemo',
submitted_on_dttm: now.getTime() / 1000,
headers: jsDataHeaders,
rows: jsDataRows
}
fs.writeFileSync(
nodePath.resolve(stageFolder, 'staged.json'),
JSON.stringify(stageData, null, 2)
)
// keep audit items from this Stored Program run alongside the staged data
copySessionAuditItems(stageFolder, 'submit')
// Insert a record into MPE_SUBMIT
const submitTable = loadTableData('MPE_SUBMIT') || {
name: 'MPE_SUBMIT',
columns: [],
rows: []
}
submitTable.rows.push({
TABLE_ID: tableId,
SUBMIT_STATUS_CD: 'SUBMITTED',
BASE_LIB: libref,
BASE_DS: dsn,
SUBMITTED_BY_NM: stageData.submitted_by_nm,
SUBMITTED_ON_DTTM: stageData.submitted_on_dttm,
SUBMITTED_REASON_TXT: message,
INPUT_OBS: jsDataRows.length,
INPUT_VARS: jsDataHeaders.filter(
(h) => h !== '_____DELETE__THIS__RECORD_____'
).length,
NUM_OF_APPROVALS_REQUIRED: 1,
NUM_OF_APPROVALS_REMAINING: 1,
REVIEWED_BY_NM: '',
REVIEWED_ON_DTTM: null
stageSubmission({
dataDir,
stagingDir,
tableId,
libds,
libref,
dsn,
message,
headers: jsDataHeaders,
rows: jsDataRows
})
saveTableData('MPE_SUBMIT', submitTable)
webOutOpen()
webOutObj([{ STATUS: 'SUCCESS', DSID: tableId, URL: '' }], 'sasparams')
@@ -3,21 +3,23 @@ const nodePath = require('path')
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
const licenceStore = nodePath.resolve(driveRoot, 'files', appLoc, 'mock-storage', 'licence.json')
const usersStore = nodePath.resolve(driveRoot, 'files', appLoc, 'mock-storage', 'users.json')
let licence = {}
let licence = {
activationKey: '',
licenceKey: ''
}
let users = {}
let dcLibref = 'DC_JSLIB'
let adminGroup = 'Data Management Business Approvers'
let licenceKey = ''
let activationKey = ''
licence = {
activationKey: '',
licenceKey: ''
}
users = {
REGISTERCOUNT: 1,
ISREGISTERED: 1
@@ -48,40 +50,12 @@ try {
}
// Find the DC database created by makedata.
// makedata writes one JSON file per table under <drive>/files/<appLoc>/data/<DCLIB>/
// plus a settings.json with the DCLIB, admin group, and dcpath.
// Scan for the most recent DCLIB directory under <appLoc>/data/
let dataDir = null
try {
const dataRoot = nodePath.resolve(driveRoot, 'files', appLoc, 'data')
if (fs.existsSync(dataRoot)) {
const dirs = fs.readdirSync(dataRoot)
.filter(d => d.toUpperCase().startsWith('DC'))
.map(d => {
const dirPath = nodePath.resolve(dataRoot, d)
const stat = fs.statSync(dirPath)
return { name: d, path: dirPath, mtime: stat.mtime }
})
.sort((a, b) => b.mtime - a.mtime)
if (dirs.length > 0) {
const latest = dirs[0]
dcLibref = latest.name
dataDir = latest.path
// Read settings.json for the admin group
const settingsFile = nodePath.resolve(latest.path, 'settings.json')
try {
const settingsRaw = fs.readFileSync(settingsFile, {encoding:'utf8'}).toString()
const settings = JSON.parse(settingsRaw)
adminGroup = settings.dc_admin_group || adminGroup
} catch(err) {}
}
}
} catch(err) {
// Database not yet created - use defaults (pre-makedata state)
}
// DC database location and admin group come from services/settings.js
// (written by makedata). Defaults apply when makedata hasn't run yet.
const dcConf = loadDcSettings(driveRoot, appLoc)
dcLibref = dcConf.dcLibref
let dataDir = dcConf.dataDir
if (dcConf.dcAdminGroup) adminGroup = dcConf.dcAdminGroup
// Build sasdatasets from the MPE_TABLES data file
let sasdatasets = []
@@ -105,56 +79,56 @@ if (sasdatasets.length === 0) {
if (licence.licenceKey) licenceKey = licence.licenceKey
if (licence.activationKey) activationKey = licence.activationKey
const sasdatasetsJson = sasdatasets.map(d => `{"LIBREF": "${d.LIBREF}","DSN": "${d.DSN}"}`).join(',\n')
_webout = `{"SYSDATE" : "26SEP22"
,"SYSTIME" : "08:29"
, "sasdatasets":
[
${sasdatasetsJson}
]
, "saslibs":
[
{"LIBREF": "${dcLibref}"}
]
, "globvars":
[
{"DCLIB": "${dcLibref}",
"SAS9LINEAGE_ENABLED": 1,
"ISADMIN": 1,
"ISREGISTERED": ${users.ISREGISTERED},
"REGISTERCOUNT": ${users.REGISTERCOUNT},
"DC_ADMIN_GROUP": "${adminGroup}",
"LICENCE_KEY": "${licenceKey}",
"ACTIVATION_KEY": "${activationKey}",
"DC_RESTRICT_EDITRECORD": "NO"
// Keys registered via admin/registerkey are stored in mpe_config
// (DC_LICENCE_KEY / DC_ACTIVATION_KEY), mirroring the real backend's
// dc_getsettings. Prefer the active mpe_config values; mock-storage/licence.json
// remains as a fallback for pre-existing deployments.
if (dataDir) {
try {
const configFile = nodePath.resolve(dataDir, 'mpe_config.json')
const configRaw = fs.readFileSync(configFile, {encoding:'utf8'}).toString()
const config = JSON.parse(configRaw)
if (config && config.rows) {
const getVar = (name) => {
const rows = config.rows.filter(r =>
r.var_scope === 'DC' && r.var_name === name && r.var_active === 1)
if (rows.length === 0) return ''
// TXTEMPORAL: the active row is the one with the high tx_to
const active = rows.find(r => Number(r.tx_to) >= 253717919999) || rows[rows.length - 1]
const val = (active.var_value || '').toString()
return val.trim()
}
const configLicence = getVar('DC_LICENCE_KEY')
const configActivation = getVar('DC_ACTIVATION_KEY')
if (configLicence) licenceKey = configLicence
if (configActivation) activationKey = configActivation
}
} catch(err) { /* mpe_config not available - keep mock-storage values */ }
}
const globvars = [
{
DCLIB: dcLibref,
SAS9LINEAGE_ENABLED: 1,
ISADMIN: 1,
ISREGISTERED: users.ISREGISTERED,
REGISTERCOUNT: users.REGISTERCOUNT,
DC_ADMIN_GROUP: adminGroup,
LICENCE_KEY: licenceKey,
ACTIVATION_KEY: activationKey,
DC_RESTRICT_EDITRECORD: 'NO'
}
]
,"xlmaps":
[
["BASEL-CR2" ,"","${dcLibref}.MPE_XLMAP_DATA" ],
["BASEL-KM1" ,"Basel 3 Key Metrics report","${dcLibref}.MPE_XLMAP_DATA" ],
["SAMPLE" ,"","${dcLibref}.MPE_XLMAP_DATA" ]
const xlmaps = [
['BASEL-CR2', '', dcLibref + '.MPE_XLMAP_DATA'],
['BASEL-KM1', 'Basel 3 Key Metrics report', dcLibref + '.MPE_XLMAP_DATA'],
['SAMPLE', '', dcLibref + '.MPE_XLMAP_DATA']
]
,"_DEBUG" : ""
,"_METAUSER": "sasdemo@SAS"
,"_METAPERSON": "sasdemo"
,"_PROGRAM" : "${appLoc}/services/public/startupservice"
,"AUTOEXEC" : ""
,"MF_GETUSER" : "sasdemo"
,"SYSCC" : "0"
,"SYSENCODING" : "utf-8"
,"SYSERRORTEXT" : ""
,"SYSHOSTNAME" : "sasjs-server"
,"SYSPROCESSID" : "0"
,"SYSPROCESSMODE" : "Stored Program"
,"SYSPROCESSNAME" : ""
,"SYSJOBID" : "0"
,"SYSSCPL" : "LINUX"
,"SYSSITE" : "123"
,"SYSUSERID" : "sasdemo"
,"SYSVLONG" : "9.04.01M7P080520"
,"SYSWARNINGTEXT" : ""
,"END_DTTM" : "2022-09-26T08:29:06.092000"
,"MEMSIZE" : "0KB"
}`
webOutOpen()
webOutObj(sasdatasets, 'sasdatasets')
webOutObj([{ LIBREF: dcLibref }], 'saslibs')
webOutObj(globvars, 'globvars')
webOutObj(xlmaps, 'xlmaps')
webOutClose()
@@ -1,19 +1,21 @@
const path = require('path')
const nodePath = require('path')
let iwantFileText = ''
let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
const sessionStoragePath = nodePath.resolve(driveRoot, 'files', appLoc, 'mock-storage')
let appLoc = path.join(..._program.split('services')[0].split('/'))
const sessionStoragePath = path.resolve(__dirname, '..', '..', 'drive', 'files', appLoc, 'mock-storage')
if (!fs.existsSync(sessionStoragePath)){
fs.mkdirSync(sessionStoragePath);
if (!fs.existsSync(sessionStoragePath)) {
fs.mkdirSync(sessionStoragePath)
}
if (_webin_filename1.includes('filterquery')) iwantFileText = _webin_fileref1.toString()
if (_webin_filename2.includes('filterquery')) iwantFileText = _webin_fileref2.toString()
// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))
if (iwantFileText.length > 0) {
const filterStore = path.resolve(sessionStoragePath, 'filter.txt')
let iwantFileText = fetchRaw('filterquery')
if (iwantFileText && iwantFileText.length > 0) {
const filterStore = nodePath.resolve(sessionStoragePath, 'filter.txt')
const filterText = iwantFileText.split('\n')[1]
try {
@@ -21,36 +23,14 @@ if (iwantFileText.length > 0) {
} catch (err) {}
}
_webout = `{"SYSDATE" : "26SEP22"
,"SYSTIME" : "08:49"
, "result":
[
{
"FILTER_RK": 1,
"FILTER_HASH": "FFE9C1E5F7AEC3B71F315EF2FEFC2296",
"FILTER_TABLE": "DC_JSLIB.MPE_X_TEST"
}
const result = [
{
FILTER_RK: 1,
FILTER_HASH: 'FFE9C1E5F7AEC3B71F315EF2FEFC2296',
FILTER_TABLE: 'DC_JSLIB.MPE_X_TEST'
}
]
,"_DEBUG" : ""
,"_METAUSER": "sasdemo@SAS"
,"_METAPERSON": "sasdemo"
,"_PROGRAM" : "/Projects/app/dc/services/public/validatefilter"
,"AUTOEXEC" : "D%3A%5Copt%5Csasinside%5CConfig%5CLev1%5CSASApp%5CStoredProcessServer%5Cautoexec.sas"
,"MF_GETUSER" : "sasdemo"
,"SYSCC" : "0"
,"SYSENCODING" : "wlatin1"
,"SYSERRORTEXT" : ""
,"SYSHOSTNAME" : "SAS"
,"SYSPROCESSID" : "41DD8057CAEDE35440BF960000000000"
,"SYSPROCESSMODE" : "SAS Stored Process Server"
,"SYSPROCESSNAME" : ""
,"SYSJOBID" : "27448"
,"SYSSCPL" : "Linunx"
,"SYSSITE" : "123"
,"SYSUSERID" : "sassrv"
,"SYSVLONG" : "9.04.01M7P080520"
,"SYSWARNINGTEXT" : "ENCODING option ignored for files opened with RECFM=N."
,"END_DTTM" : "2022-09-26T08:49:48.696000"
,"MEMSIZE" : "46GB"
}
`
webOutOpen()
webOutObj(result, 'result')
webOutClose()
@@ -13,7 +13,7 @@ let mptables = []
try {
const files = fs.readdirSync(dataDir)
for (const f of files) {
if (f.endsWith('.json') && f !== 'settings.json') {
if (f.endsWith('.json')) {
try {
const raw = fs.readFileSync(nodePath.resolve(dataDir, f), {encoding:'utf8'}).toString()
const tableData = JSON.parse(raw)