Merge pull request 'fix(mocks): getrawdata reads the library it was asked for, and aborts when it cannot' (#518) from fix/mock-getrawdata-resolution into release/7.23

Reviewed-on: #518

sas/mocks/sasjs/services/public/getrawdata.js

@@ -27,6 +27,17 @@ function loadTableData(tableName) {
     } catch(err) { return null }
 }
 
+/* The same, from the folder of any library on the Drive: makedata and
+   createdemodata write each library's tables under data/<libref>, and this
+   service downloads from the library it was asked for rather than from the
+   control one. */
+function loadTableFrom(libref, tableName) {
+    try {
+        const file = nodePath.resolve(dataDir, '..', libref, tableName.toLowerCase() + '.json')
+        return JSON.parse(fs.readFileSync(file, {encoding:'utf8'}).toString())
+    } catch(err) { return null }
+}
+
 // SAS epoch for date/datetime/time formatting
 const SAS_EPOCH_MS = Date.UTC(1960, 0, 1)
 const MONTHS = ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG','SEP','OCT','NOV','DEC']
@@ -100,27 +111,45 @@ const DOWNLOAD_TYPES = {
   helpers collide with it, so the envelope is written here the way mpAbort
   writes it: the abort member, no CLOSE.
 */
-function abortInput(message) {
+function abortInput(message, mac) {
     webout('OPEN')
-    webout('OBJ', 'sasjsAbort', [{ MSG: message, MAC: _program + '.sas' }])
+    webout('OBJ', 'sasjsAbort', [{ MSG: message, MAC: mac || _program + '.sas' }])
     const stop = new Error(message)
     stop.mockAbort = true
     throw stop
 }
 
-const requestTable = String(typeof table === 'undefined' || table === null ? '' : table).trim()
+const rawTable = String(typeof table === 'undefined' || table === null ? '' : table).trim()
 const TABLE_SHAPE = /^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i
+/* %mpe_validatecol(...,LIBDS,...) strips a trailing -FC before validating, and
+   getrawdata.sas reads LIBREF.CAT-FC as a format catalog and then works with
+   the stripped pair (`table=cats("&lib..",ds)`), so the mock keeps ONE
+   canonical name from here on: the guard, the filename, the library it reads
+   and the name in an abort all use the stripped form rather than validating
+   one name and using another. */
 const withoutCatalog =
-    requestTable.length > 3 && requestTable.slice(-3).toUpperCase() === '-FC'
-        ? requestTable.slice(0, -3)
-        : requestTable
-if (!requestTable || !TABLE_SHAPE.test(withoutCatalog)) abortInput('Invalid table')
+    rawTable.length > 3 && rawTable.slice(-3).toUpperCase() === '-FC'
+        ? rawTable.slice(0, -3)
+        : rawTable
+if (!rawTable || !TABLE_SHAPE.test(withoutCatalog)) abortInput('Invalid table')
+const requestTable = withoutCatalog.toUpperCase()
 
 const requestFilter = String(
     typeof filter === 'undefined' || filter === null ? '' : filter
 ).trim() || '0'
+/* ISINT is input(&filter,??best32.) with mod 1 = 0, and the estate's recorded
+   answers show the whole value has to read as a number: it rejects 1,000,
+   1abc and 1.5x, which a leading-numeric-prefix read would take, and accepts
+   7.0, 1e3, +5, 007 and -3. Number() alone is the wrong shape test - it reads
+   0x10 as 16, which the informat does not - so the number is matched first and
+   then asked whether it is whole. */
+const FILTER_NUMBER = /^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$/
 const filterValue = Number(requestFilter)
-if (!isFinite(filterValue) || Math.floor(filterValue) !== filterValue) {
+if (
+    !FILTER_NUMBER.test(requestFilter) ||
+    !isFinite(filterValue) ||
+    Math.floor(filterValue) !== filterValue
+) {
     abortInput('Invalid filter')
 }
 
@@ -130,90 +159,104 @@ const writesWeboutDirectly = requestedType === 'WEBCSV' || requestedType === 'WE
 
 if (!download && !writesWeboutDirectly) abortInput('Invalid type')
 
-const filename = download ? requestTable.toUpperCase() + '.' + download.ext : ''
+const filename = download ? requestTable + '.' + download.ext : ''
 
 // Generate CSV from the table data (data-driven, not hardcoded)
 let csvLines = []
 
-// Determine the table name from the `table` param (format: LIBREF.TABLE)
 const tableName = requestTable.split('.')[1]
+const exportLibref = requestTable.split('.')[0]
 
-const tableData = loadTableData(tableName)
-if (tableData && tableData.columns && tableData.rows) {
-    /*
-      getrawdata.sas drops the registration's validity columns before it writes
-      the download (`data staged(drop=&txfrom &txto)`), so a control table never
-      carries its history columns into a user's file.
-    */
-    const exportLibref = requestTable.split('.')[0]
-    const registry = loadTableData('MPE_TABLES')
-    const registration =
-        registry && registry.rows
-            ? registry.rows.find(
-                  (row) => String(row.libref).toUpperCase() === exportLibref.toUpperCase() &&
-                      String(row.dsn).toUpperCase() === tableName.toUpperCase()
-              )
-            : null
-    const dropped = ['var_txfrom', 'var_txto']
-        .map((key) => (registration ? registration[key] : ''))
-        .filter((name) => name && String(name).trim() !== '')
-        .map((name) => String(name).trim().toUpperCase())
-    const exportColumns = tableData.columns.filter(
-        (col) => dropped.indexOf(col.name.toUpperCase()) < 0
+/*
+  getrawdata.sas assigns the library it was asked for - %dc_assignlib(READ,&lib)
+  - and reads the data from there; the control library supplies only the
+  registration row. A table that cannot be resolved aborts before anything is
+  written, and the estate gives one message for both cases, a library it does
+  not have and a member missing from one it does:
+
+    NOLIB.MPE_TABLES        -> Target table not found: NOLIB.MPE_TABLES
+    DC634301.NO_SUCH_TABLE  -> Target table not found: DC634301.NO_SUCH_TABLE
+
+  The abort is mp_derivedvars', so the MAC is that macro's own default and not
+  the service, and the message carries the upcased LIBDS getrawdata.sas builds.
+*/
+const tableData = loadTableFrom(exportLibref, tableName)
+if (!tableData || !tableData.columns || !tableData.rows) {
+    abortInput('Target table not found: ' + requestTable, 'MP_ABORT')
+}
+
+/*
+  getrawdata.sas drops the registration's validity columns before it writes
+  the download (`data staged(drop=&txfrom &txto)`), so a control table never
+  carries its history columns into a user's file.
+*/
+const registry = loadTableData('MPE_TABLES')
+const registration =
+    registry && registry.rows
+        ? registry.rows.find(
+              (row) => String(row.libref).toUpperCase() === exportLibref.toUpperCase() &&
+                  String(row.dsn).toUpperCase() === tableName.toUpperCase()
+          )
+        : null
+const dropped = ['var_txfrom', 'var_txto']
+    .map((key) => (registration ? registration[key] : ''))
+    .filter((name) => name && String(name).trim() !== '')
+    .map((name) => String(name).trim().toUpperCase())
+const exportColumns = tableData.columns.filter(
+    (col) => dropped.indexOf(col.name.toUpperCase()) < 0
+)
+
+// a CSV download opens with a header row, as the SAS export writes one. The
+// other types stream the same lines under their own content type, and none
+// of them is a CSV, so they keep starting at the first row
+if (requestedType === 'CSV') {
+    /* the header the SAS export writes is upper case, whatever case the
+       table was created with. This service carries its own helpers rather
+       than eval'ing dcMockUtils, so it upcases here rather than calling
+       payloadKey. */
+    csvLines.push(
+        exportColumns.map((col) => String(col.name).toUpperCase()).join(',')
     )
+}
 
-    // a CSV download opens with a header row, as the SAS export writes one. The
-    // other types stream the same lines under their own content type, and none
-    // of them is a CSV, so they keep starting at the first row
-    if (requestedType === 'CSV') {
-        /* the header the SAS export writes is upper case, whatever case the
-           table was created with. This service carries its own helpers rather
-           than eval'ing dcMockUtils, so it upcases here rather than calling
-           payloadKey. */
-        csvLines.push(
-            exportColumns.map((col) => String(col.name).toUpperCase()).join(',')
-        )
-    }
+// Build column ddtype lookup
+const colDdtypes = {}
+for (const col of exportColumns) {
+    const fmt = (col.format || '').toLowerCase()
+    let ddtype = col.type === 'N' ? 'N' : 'C'
+    if (fmt.startsWith('datetime') || fmt.startsWith('e8601dt')) ddtype = 'DATETIME'
+    else if (fmt.startsWith('date')) ddtype = 'DATE'
+    else if (fmt.startsWith('time')) ddtype = 'TIME'
+    colDdtypes[col.name.toUpperCase()] = ddtype
+}
 
-    // Build column ddtype lookup
-    const colDdtypes = {}
+for (const row of tableData.rows) {
+    const values = []
     for (const col of exportColumns) {
-        const fmt = (col.format || '').toLowerCase()
-        let ddtype = col.type === 'N' ? 'N' : 'C'
-        if (fmt.startsWith('datetime') || fmt.startsWith('e8601dt')) ddtype = 'DATETIME'
-        else if (fmt.startsWith('date')) ddtype = 'DATE'
-        else if (fmt.startsWith('time')) ddtype = 'TIME'
-        colDdtypes[col.name.toUpperCase()] = ddtype
-    }
-
-    for (const row of tableData.rows) {
-        const values = []
-        for (const col of exportColumns) {
-            let val = row[col.name]
-            if (val === undefined) val = row[col.name.toLowerCase()]
-            if (val === undefined) {
-                const key = Object.keys(row).find(k => k.toLowerCase() === col.name.toLowerCase())
-                if (key) val = row[key]
-            }
-            if (val === null || val === undefined) {
-                val = ''
-            } else {
-                const ddtype = colDdtypes[col.name.toUpperCase()]
-                if (ddtype === 'DATE' || ddtype === 'DATETIME' || ddtype === 'TIME') {
-                    val = formatSasValue(val, ddtype)
-                } else {
-                    val = String(val)
-                }
-                // Quote if it contains a delimiter, a quote or a line break -
-                // a description with a newline is still one CSV record
-                if (val.includes(',') || val.includes('"') || /[\r\n]/.test(val)) {
-                    val = '"' + val.replace(/"/g, '""') + '"'
-                }
-            }
-            values.push(val)
+        let val = row[col.name]
+        if (val === undefined) val = row[col.name.toLowerCase()]
+        if (val === undefined) {
+            const key = Object.keys(row).find(k => k.toLowerCase() === col.name.toLowerCase())
+            if (key) val = row[key]
         }
-        csvLines.push(values.join(','))
+        if (val === null || val === undefined) {
+            val = ''
+        } else {
+            const ddtype = colDdtypes[col.name.toUpperCase()]
+            if (ddtype === 'DATE' || ddtype === 'DATETIME' || ddtype === 'TIME') {
+                val = formatSasValue(val, ddtype)
+            } else {
+                val = String(val)
+            }
+            // Quote if it contains a delimiter, a quote or a line break -
+            // a description with a newline is still one CSV record
+            if (val.includes(',') || val.includes('"') || /[\r\n]/.test(val)) {
+                val = '"' + val.replace(/"/g, '""') + '"'
+            }
+        }
+        values.push(val)
     }
+    csvLines.push(values.join(','))
 }
 
 _webout = csvLines.join('\n')

sas/mocks/test/download-headers.test.js

@@ -209,11 +209,15 @@ test('getrawdata validates the table before it streams, as the estate does', ()
   }
 })
 
-test('getrawdata validates the filter as an integer, as ISINT reads it', () => {
+test('getrawdata reads the filter the way the estate does', () => {
   const estate = seededEstate({ demo: false })
-  const rejected = ['abc', '1.5', '1,000']
 
-  for (const filter of rejected) {
+  /* Recorded from the synthetic estate: each of these is answered "Invalid
+     filter" and nothing is written. 1,000 and 1abc are the interesting ones -
+     ISINT's input(&filter,??best32.) reading would take a leading numeric
+     prefix, but the estate rejects them, so the whole value has to read as a
+     number - and 0x10 is 16 to Number() and invalid there. */
+  for (const filter of ['abc', '1.5', '1,000', '1abc', '1.5x', '0x10']) {
     const body = runService(estate, 'public/getrawdata', {
       params: { type: 'CSV', table: 'DC_JSLIB.MPE_SELECTBOX', filter }
     })
@@ -225,20 +229,82 @@ test('getrawdata validates the filter as an integer, as ISINT reads it', () => {
     assert.equal(headersOf(estate), '', filter)
   }
 
-  /* %mp_validatecol(...,ISINT,...) is input(&filter,??best32.) with mod 1 = 0,
-     so a decimal written as one and an exponent are both integers to it. The
-     estate's own answer for these three is a downstream abort (the filter RK
-     does not exist), not "Invalid filter", which is what shows validation
-     passed; the mock does not resolve filter RKs, so it streams. */
-  for (const filter of ['7.0', '1e3', ' 0 ']) {
+  /* And these pass validation. The estate's own answer for them is a downstream
+     abort (the filter RK does not exist), not "Invalid filter"; the mock does
+     not resolve filter RKs, so it streams. */
+  for (const filter of ['7.0', '1e3', '+5', '007', ' 7 ', '-3']) {
     const body = runService(estate, 'public/getrawdata', {
       params: { type: 'CSV', table: 'DC_JSLIB.MPE_SELECTBOX', filter }
     })
-    assert.doesNotMatch(body, /sasjsAbort/, filter)
+    assert.doesNotMatch(body, /Invalid filter/, filter)
     assert.equal(headersOf(estate).split('\n')[0], 'Content-Type: application/csv', filter)
   }
 })
 
+test('getrawdata resolves the catalog form to one canonical name', () => {
+  const estate = seededEstate({ demo: false })
+
+  /*
+    %mpe_validatecol(...,LIBDS,...) strips the -FC suffix before validating, and
+    getrawdata.sas then works with the stripped pair - so a request must not
+    validate one name and name the file after another. The estate answers a real
+    LIBREF.CAT-FC by running %mp_cntlout on the catalog; the mock has no catalog
+    support, so the body is the member's rows and its filename is the stripped
+    LIBREF.DS.
+  */
+  const body = runService(estate, 'public/getrawdata', {
+    params: { type: 'CSV', table: 'DC_JSLIB.MPE_SELECTBOX-FC', filter: '0' }
+  })
+
+  assert.doesNotMatch(body, /sasjsAbort/)
+  assert.deepEqual(headersOf(estate).split('\n'), [
+    'Content-Type: application/csv',
+    'Content-disposition: attachment; filename=DC_JSLIB.MPE_SELECTBOX.csv'
+  ])
+})
+
+test('getrawdata downloads from the library it was asked for', () => {
+  /* DC_JSLIB is the control library; the data comes from the library named in
+     the LIBDS, which is what %dc_assignlib(READ,&lib) in getrawdata.sas does
+     and what createdemodata writes TESTDATA for. */
+  const estate = seededEstate()
+  const body = runService(estate, 'public/getrawdata', {
+    params: { type: 'CSV', table: 'TESTDATA.DEMO_ORDERS', filter: '0' }
+  })
+
+  assert.doesNotMatch(body, /sasjsAbort/)
+  assert.ok(body.split('\n').length > 2, 'the export holds rows of the requested library')
+  assert.deepEqual(headersOf(estate).split('\n'), [
+    'Content-Type: application/csv',
+    'Content-disposition: attachment; filename=TESTDATA.DEMO_ORDERS.csv'
+  ])
+})
+
+test('getrawdata aborts a table it cannot resolve, as the estate does', () => {
+  const estate = seededEstate({ demo: false })
+  const cases = [
+    // a library the estate does not have: recorded as Target table not found
+    ['NOLIB.MPE_TABLES', 'Target table not found: NOLIB.MPE_TABLES'],
+    // a member missing from a library it does have: the same message
+    ['DC_JSLIB.NO_SUCH_TABLE', 'Target table not found: DC_JSLIB.NO_SUCH_TABLE'],
+    // the message carries the upcased LIBDS getrawdata.sas builds
+    ['nolib.mpe_tables', 'Target table not found: NOLIB.MPE_TABLES']
+  ]
+
+  for (const [table, message] of cases) {
+    const body = runService(estate, 'public/getrawdata', {
+      params: { type: 'CSV', table, filter: '0' }
+    })
+    // the abort is mp_derivedvars', so the MAC is that macro's default
+    assert.deepEqual(
+      JSON.parse(body).sasjsAbort,
+      [{ MSG: message, MAC: 'MP_ABORT' }],
+      table
+    )
+    assert.equal(headersOf(estate), '', table)
+  }
+})
+
 // ─── admin/exportdb and the shared mpStreamfile ──────────────────────────────
 
 test('exportdb streams the DDL as application/text, named <libref>.ddl', async () => {

sas/mocks/test/parity/golden/public/getrawdata/missing_table.json

@@ -0,0 +1,24 @@
+{
+  "_meta": {
+    "scenario": "missing_table",
+    "service": "public/getrawdata",
+    "estate": "synthetic",
+    "targetType": "SASJS",
+    "sas": "9.4M9",
+    "dc": "7.22.0",
+    "captured": "2026-11-07",
+    "tls": "verified",
+    "members": [
+      "sasjsAbort"
+    ]
+  },
+  "body": {
+    "sasjsAbort": [
+      {
+        "MSG": "Target table not found: <LIBREF>.NO_SUCH_TABLE",
+        "MAC": "MP_ABORT"
+      }
+    ],
+    "SYSCC": "0"
+  }
+}

sas/mocks/test/parity/scenarios.json

@@ -272,6 +272,30 @@
       "note": "the download path: a table as the client saves it, CSV with a header row and the registration's validity columns dropped. Parsed to rows on both sides, because SAS quotes and pads each cell to the variable's width",
       "accepted": []
     },
+    {
+      "id": "missing_table",
+      "service": "public/getrawdata",
+      "params": {
+        "type": "CSV",
+        "table": "{{LIBREF}}.NO_SUCH_TABLE",
+        "filter": "0"
+      },
+      "mockParams": {
+        "type": "CSV",
+        "table": "DC_JSLIB.NO_SUCH_TABLE",
+        "filter": "0"
+      },
+      "members": [
+        "sasjsAbort"
+      ],
+      "note": "a download for a table that cannot be resolved: the abort names the full LIBDS and comes from mp_derivedvars, so the MAC is the macro's default rather than the service. The same answer covers a library the estate does not have",
+      "accepted": [
+        {
+          "path": ".SYSCC",
+          "reason": "the SAS estate's wrapper adds SYSCC to an aborted response; a JS program's envelope is finalised at CLOSE, and the abort stops before it, so the mock's abort carries none. The message and the macro name - what the client renders - do match"
+        }
+      ]
+    },
     {
       "id": "control_library",
       "service": "public/viewtables",