Merge pull request 'fix(mocks): key rk_underlying tables on their business key' (#357) from fix/mock-rk-underlying-pk into feat/derived-vars

Reviewed-on: #357

client/package.json

@@ -147,7 +147,7 @@
     "ajv": "8.18.0",
     "uuid": "11.1.1",
     "lighthouse": "13.4.0",
-    "fast-uri": "3.1.8",
+    "fast-uri": "4.2.3",
     "readdir-glob": {
       "brace-expansion": "^5.0.9"
     },

sas/mocks/sasjs/services/dcMockUtils.js

@@ -460,6 +460,60 @@ function sasVarsEntry(col) {
     return { format: fmt, label: col.label || col.name, length: String(col.length), type: col.type === 'N' ? 'num' : 'char' }
 }
 
+/**
+ * Assigns the retained key (RK) to staged rows for a table registered with
+ * rk_underlying, mirroring %mp_retainedkey as %mpe_loader invokes it: any RK
+ * column in the input is dropped, and every row's RK is derived from its
+ * business key - the value that key already holds on the base table, or the
+ * next key value for a new one. Business keys are matched the way SAS compares
+ * them: case-sensitive, trailing blanks ignored.
+ *
+ * @param {Object} opts - { rows, tableReg, baseData }
+ *   tableReg - the table's MPE_TABLES row (needs rk_underlying and buskey)
+ *   baseData - { columns, rows } of the base table, for the key lookup
+ * @returns {String} the RK column name when the assignment ran, else ''
+ */
+function mpeAssignRetainedKey(opts) {
+    const { rows, tableReg, baseData } = opts
+    const rkUnderlying = tableReg && tableReg.rk_underlying
+        ? String(tableReg.rk_underlying).trim()
+        : ''
+    const rkCol = tableReg && tableReg.buskey ? String(tableReg.buskey).trim() : ''
+    if (!rkUnderlying || !rkCol || !rows || rows.length === 0) return ''
+
+    const base = baseData || { columns: [], rows: [] }
+    const submitted = Object.keys(rows[0] || {}).find(
+        (k) => k.toUpperCase() === rkCol.toUpperCase()
+    )
+    const rkName = submitted ||
+        (base.columns.find((c) => c.name.toUpperCase() === rkCol.toUpperCase()) || {}).name ||
+        rkCol
+    const keys = rkUnderlying.split(/\s+/)
+    const keyValue = (value) => {
+        if (value === null || value === undefined) return ''
+        return typeof value === 'number' ? String(value) : String(value).replace(/\s+$/, '')
+    }
+    const businessKey = (row) => keys.map((k) => keyValue(colVal(row, k))).join('|')
+
+    const baseByKey = new Map()
+    let nextRk = 0
+    for (const row of base.rows) {
+        const rk = colVal(row, rkCol)
+        baseByKey.set(businessKey(row), rk)
+        const num = Number(rk)
+        if (isFinite(num) && num > nextRk) nextRk = num
+    }
+
+    for (const row of rows) {
+        Object.keys(row).forEach((k) => {
+            if (k.toUpperCase() === rkCol.toUpperCase()) delete row[k]
+        })
+        const existing = baseByKey.get(businessKey(row))
+        row[rkName] = existing === undefined || existing === null ? ++nextRk : existing
+    }
+    return rkName
+}
+
 /**
  * Stages a submission: writes staged.json under staging/<tableId>/, copies the
  * session audit items, and inserts the MPE_SUBMIT row. Shared by the

sas/mocks/sasjs/services/editors/getdata.js

@@ -77,6 +77,27 @@ if (mpeTablesData && mpeTablesData.rows) {
     tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
 }
 
+// The variables exposed to the editor as the primary key, matching getdata.sas:
+// pk = coalesce(rk_underlying, buskey). A table registered with rk_underlying
+// is keyed on its business key in the editor - the RK itself is assigned by the
+// backend loader, so it is neither shown nor sent by the user.
+const rkUnderlying = tableReg && tableReg.rk_underlying ? tableReg.rk_underlying.trim() : ''
+const buskey = tableReg && tableReg.buskey ? tableReg.buskey.trim() : ''
+const pkFields = rkUnderlying || buskey
+const pkColumns = pkFields ? pkFields.split(/\s+/) : []
+
+// Order columns with the PK variables first, in PK order, then the remaining
+// columns in schema order. getdata.sas does the same ("order delete var and pk
+// fields at start of table") and the client reads the first PKCNT columns
+// positionally when it detects duplicate keys.
+function pkFirst(columns, keys) {
+    const rank = new Map(keys.map((name, i) => [name.toUpperCase(), i]))
+    const keyColumns = columns
+        .filter(col => rank.has(col.name.toUpperCase()))
+        .sort((a, b) => rank.get(a.name.toUpperCase()) - rank.get(b.name.toUpperCase()))
+    return keyColumns.concat(columns.filter(col => !rank.has(col.name.toUpperCase())))
+}
+
 // ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
 // Applied before the filter, so a filter can reference them.  The derived
 // columns are added to the schema too, which is what puts them in COLS and in
@@ -100,10 +121,7 @@ const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
 let dqrules = []
 let dqdata = []
 if (validationsData && validationsData.rows) {
-    const pkCols = new Set()
-    if (tableReg && tableReg.buskey) {
-        tableReg.buskey.trim().split(/\s+/).forEach(c => pkCols.add(c.toUpperCase()))
-    }
+    const pkCols = new Set(pkColumns.map(c => c.toUpperCase()))
     dqrules = validationsData.rows
         .filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
         .map(r => ({
@@ -220,8 +238,17 @@ if (tableData && tableData.columns && tableData.rows) {
         if (tableReg.var_busfrom) excludeCols.add(tableReg.var_busfrom.toUpperCase())
         if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
     }
+    // getdata.sas drops the RK field from the editor view when the table is
+    // registered with rk_underlying (drop_rk=&buskey), and exposes the business
+    // key instead.
+    if (rkUnderlying && buskey) {
+        buskey.split(/\s+/).forEach(c => excludeCols.add(c.toUpperCase()))
+    }
 
-    let visibleColumns = tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase()))
+    let visibleColumns = pkFirst(
+        tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase())),
+        pkColumns
+    )
     let visibleRows = tableData.rows
         .filter(r => filterPredicate(r))
         .map(r => {
@@ -389,8 +416,8 @@ if (tableData && tableData.columns && tableData.rows) {
         sasparams: [{
             COLHEADERS: colHeaders.join(','),
             FILTER_TEXT: filterText === '' ? ' ' : filterText,
-            PKCNT: tableReg && tableReg.buskey ? tableReg.buskey.trim().split(/\s+/).length : 0,
-            PK: tableReg ? tableReg.buskey || '' : '',
+            PKCNT: pkColumns.length,
+            PK: pkFields,
             DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
             DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
             TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',

sas/mocks/sasjs/services/editors/loadfile.js

@@ -147,6 +147,12 @@ if (!msg) {
                     if (tableEntry.var_txfrom) computedCols.push(tableEntry.var_txfrom.toUpperCase())
                     if (tableEntry.var_txto) computedCols.push(tableEntry.var_txto.toUpperCase())
                     if (tableEntry.var_processed) computedCols.push(tableEntry.var_processed.toUpperCase())
+                    // loadfile.sas drops the RK from the comparison for a table
+                    // with rk_underlying (the loader derives it), so an upload
+                    // for one legitimately omits it.
+                    if (tableEntry.rk_underlying && tableEntry.buskey) {
+                        computedCols.push(String(tableEntry.buskey).toUpperCase())
+                    }
                 }
             }
             const baseCols = schema.map((c) => c.name.toUpperCase())
@@ -191,6 +197,23 @@ if (!msg) {
         }
 
         tableId = makeTableId()
+        // ─── RK assignment (mpe_loader) ──────────────────────────────────────
+        // The uploaded file carries the business key and no RK (see the
+        // expected-columns check above), so the loader derives it before the
+        // rows are staged.
+        const finalTables = mpeLoadTableData('MPE_TABLES')
+        const finalReg =
+            finalTables && finalTables.rows
+                ? finalTables.rows.find((r) => r.libref === libref && r.dsn === dsn)
+                : null
+        const rkName = mpeAssignRetainedKey({
+            rows,
+            tableReg: finalReg,
+            baseData: makeTableLoader(libDataDir(libref))(dsn)
+        })
+        if (rkName && !headers.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
+            headers.push(rkName)
+        }
         stageSubmission({
             dataDir: libDataDir(libref),
             stagingDir,

sas/mocks/sasjs/services/editors/stagedata.js

@@ -81,6 +81,26 @@ if (hookReg && hookReg.post_edit_hook) {
     }
 }
 
+// ─── RK assignment (mpe_loader) ──────────────────────────────────────────────
+// A table registered with rk_underlying is edited on its business key and the
+// RK column is withheld from the editor (see getdata.js), so the submitted rows
+// carry no RK. The loader derives it, and MPE_SUBMIT records the staged width
+// - which includes the derived column.
+
+const finalTables = makeTableLoader(mpeDataDir)('MPE_TABLES')
+const finalReg =
+    finalTables && finalTables.rows
+        ? finalTables.rows.find((r) => r.libref === libref && r.dsn === dsn)
+        : null
+const rkName = mpeAssignRetainedKey({
+    rows: jsDataRows,
+    tableReg: finalReg,
+    baseData: loadTableData(dsn)
+})
+if (rkName && !jsDataHeaders.some((h) => h.toUpperCase() === rkName.toUpperCase())) {
+    jsDataHeaders.push(rkName)
+}
+
 // ─── Stage the data ──────────────────────────────────────────────────────────
 // One subfolder per submission (DSID) under the staging directory, so that the
 // staged data can sit alongside related artifacts (submit/approval logs,