chore(release): 7.11.0 [skip ci]

# [7.11.0](https://git.datacontroller.io/dc/dc/compare/v7.10.1...v7.11.0) (2026-07-20)

### Bug Fixes

* audit-skipping syntax correction ([ad7efc9](ad7efc90bc))
* **editor:** ignore cell validation on rows marked for delete ([70c5805](70c5805743))
* explicit use of .sashdat in dc_casload ([6a95067](6a95067618))
* refactor mf_wordsinstr1butnotstr2 and added a wide table to test suite ([be94c68](be94c68f35))
* retry the startup service request once before failing ([5391df8](5391df8f04))
* run dc_casload for dq tables in getdata.  Closes [#264](#264). ([6b16f9b](6b16f9b364))
* updating tests, moving mpe_rowhash to core library ([c5f0e4f](c5f0e4f106))

### Features

* bumping core to get the mp_rowhash definition, [#269](#269) ([f73f5e4](f73f5e47ac))
* new hash algorithm - allows 1000+ cols.  Closes [#269](#269) ([08947f0](08947f0fc8))

CHANGELOG.md

@@ -1,3 +1,29 @@
+# [7.11.0](https://git.datacontroller.io/dc/dc/compare/v7.10.1...v7.11.0) (2026-07-20)
+
+
+### Bug Fixes
+
+* audit-skipping syntax correction ([ad7efc9](https://git.datacontroller.io/dc/dc/commit/ad7efc90bc33c93383e17359d14c0605cded69a9))
+* **editor:** ignore cell validation on rows marked for delete ([70c5805](https://git.datacontroller.io/dc/dc/commit/70c5805743c8c6e61bc216f37cbe8cd4f0614c98))
+* explicit use of .sashdat in dc_casload ([6a95067](https://git.datacontroller.io/dc/dc/commit/6a95067618600d7b03b6af46bf3d74721e42c66a))
+* refactor mf_wordsinstr1butnotstr2 and added a wide table to test suite ([be94c68](https://git.datacontroller.io/dc/dc/commit/be94c68f354d0f9b1eb52213807d1874a74d9b8a))
+* retry the startup service request once before failing ([5391df8](https://git.datacontroller.io/dc/dc/commit/5391df8f04cf40205a600b23c207f97b6fdf5913))
+* run dc_casload for dq tables in getdata.  Closes [#264](https://git.datacontroller.io/dc/dc/issues/264). ([6b16f9b](https://git.datacontroller.io/dc/dc/commit/6b16f9b3647fca2e39d44544201ce972046e01a3))
+* updating tests, moving mpe_rowhash to core library ([c5f0e4f](https://git.datacontroller.io/dc/dc/commit/c5f0e4f106aa4e6a7c2a2fd0ebb2e85d1e7bee7b))
+
+
+### Features
+
+* bumping core to get the mp_rowhash definition, [#269](https://git.datacontroller.io/dc/dc/issues/269) ([f73f5e4](https://git.datacontroller.io/dc/dc/commit/f73f5e47ac38a305bac4ebd7eac48d1a3edf4a83))
+* new hash algorithm - allows 1000+ cols.  Closes [#269](https://git.datacontroller.io/dc/dc/issues/269) ([08947f0](https://git.datacontroller.io/dc/dc/commit/08947f0fc8458be098e3aaf769b1da7fadf480cb))
+
+## [7.10.1](https://git.datacontroller.io/dc/dc/compare/v7.10.0...v7.10.1) (2026-07-15)
+
+
+### Bug Fixes
+
+* **editor:** order validation rules by COLHEADERS, not cols[].VARNUM ([4c2aca3](https://git.datacontroller.io/dc/dc/commit/4c2aca34951d6d5c5cf2df69d5bf4256a9290d53))
+
 # [7.10.0](https://git.datacontroller.io/dc/dc/compare/v7.9.1...v7.10.0) (2026-07-13)
 
 

CONTRIBUTING.md

@@ -16,6 +16,21 @@ Add `client/.npmrc` file with following content:
 //pylon.sheetjs.com:54111/:_authToken="TOKEN-GOES-HERE"
 ```
 
+In addition to the registry auth, the `@sheet/crypto` package is installed from a local tarball
+(`client/libraries/sheet-crypto.tgz`) rather than the registry. This tarball is not committed to
+the repo directly - only an encrypted copy is: `client/libraries/sheet-crypto.tgz.gpg`.
+
+Before running `npm i`/`npm ci` in `client`, you must decrypt it using the passphrase
+(stored as the `SHEET_PWD` CI secret - ask a repo maintainer if you don't have it):
+```bash
+cd client
+echo "YOUR_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 \
+  --output ./libraries/sheet-crypto.tgz \
+  --decrypt ./libraries/sheet-crypto.tgz.gpg
+
+npm i
+```
+
 [Handsontable](https://www.npmjs.com/package/handsontable)
 
 Licence should be inserted in the `client/src/index.html` file:
@@ -66,6 +81,15 @@ To remove the release, you need to do it with repo administration over at [https
 
 # Troubleshooting
 
+## `npm i` fails with ENOENT on `sheet-crypto.tgz`
+
+If `npm i`/`npm ci` in `client` fails with:
+```
+npm error enoent ENOENT: no such file or directory, open '.../client/libraries/sheet-crypto.tgz'
+```
+it means the encrypted `sheet-crypto.tgz.gpg` hasn't been decrypted yet. See
+[Dependencies that requires licences](#dependencies-that-requires-licences) above for how to decrypt it.
+
 ## Makedata service "could not create directory" error
 
 The dcpath folder should have its permissions set so that the system account (SYSUSERID) can both read and write to it.

client/cypress/e2e/editor.cy.ts

@@ -115,8 +115,109 @@ context('editor tests: ', function () {
       })
     })
   })
+
+  // An invalid cell shouldn't block submission if its row
+  // is marked for delete
+  it('5 | Submits a delete-marked row with an invalid cell', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    clickOnEdit(() => {
+      cy.get('.btn.btn-sm.btn-icon.btn-outline-danger', {
+        timeout: longerCommandTimeout
+      }).then(() => {
+        cy.get('.ht_master tbody tr').then((rows: any) => {
+          // childNodes[5] = SOME_NUM (numeric) — feed it a non-numeric value
+          // to trigger HOT's native invalid mark.
+          cy.get(rows[1].childNodes[5])
+            .dblclick({ force: true })
+            .then(() => {
+              cy.focused()
+                .clear()
+                .type('not a number{enter}')
+                .then(() => {
+                  // childNodes[1] = _____DELETE__THIS__RECORD_____
+                  setDeleteFlag(rows[1].childNodes[1], 'Yes', () => {
+                    submitTable(() => {
+                      // Validation passed despite the invalid SOME_NUM cell:
+                      // the confirm-submit modal's Submit button is enabled
+                      // (validationDone === 1), not the "invalid values" abort.
+                      cy.get('#submitBtn', {
+                        timeout: longerCommandTimeout
+                      })
+                        .should('exist')
+                        .should('not.be.disabled')
+                        .then(() => done())
+                    })
+                  })
+                })
+            })
+        })
+      })
+    })
+  })
+
+  // The delete exemption must not affect the uniqueness (primary key) check.
+  it('6 | Still blocks a duplicate primary key even on a delete-marked row', (done) => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    clickOnEdit(() => {
+      cy.get('.btn.btn-sm.btn-icon.btn-outline-danger', {
+        timeout: longerCommandTimeout
+      }).then(() => {
+        cy.get('.ht_master tbody tr').then((rows: any) => {
+          cy.get(rows[0].childNodes[2]).then((firstPk) => {
+            const duplicateValue = firstPk.text()
+
+            // childNodes[2] = PRIMARY_KEY_FIELD
+            cy.get(rows[1].childNodes[2])
+              .dblclick({ force: true })
+              .then(() => {
+                cy.focused()
+                  .clear()
+                  .type(`${duplicateValue}{enter}`)
+                  .then(() => {
+                    setDeleteFlag(rows[1].childNodes[1], 'Yes', () => {
+                      submitTable()
+
+                      cy.get('.duplicate-keys-modal', {
+                        timeout: longerCommandTimeout
+                      })
+                        .should('exist')
+                        .then(() => done())
+                    })
+                  })
+              })
+          })
+        })
+      })
+    })
+  })
 })
 
+// Opens the _____DELETE__THIS__RECORD_____ dropdown editor on the given cell
+// and picks the given choice ('Yes'/'No') — same technique as
+// coltype-delete-record.cy.ts (arrow -> autocompleteEditor choice list).
+const setDeleteFlag = (
+  deleteCell: any,
+  value: 'Yes' | 'No',
+  callback?: any
+) => {
+  cy.get(deleteCell)
+    .click({ force: true })
+    .then(($td) => {
+      cy.get('.htAutocompleteArrow', { withinSubject: $td })
+        .should('exist')
+        .click({ force: true })
+
+      cy.get('.autocompleteEditor .htCore tbody td')
+        .contains(value)
+        .click({ force: true })
+        .then(() => {
+          if (callback) callback()
+        })
+    })
+}
+
 const clickOnEdit = (callback?: any) => {
   cy.get('.btnCtrl button.btn-primary', { timeout: longerCommandTimeout })
     .click()

client/cypress/e2e/liveness.cy.ts

@@ -36,6 +36,38 @@ context('liveness tests: ', function () {
     })
   })
 
+  // Multiple concurrent DC instances can cause the startupservice request
+  // to time out even though a retry succeeds. Fail only the first
+  // startupservice call (any later ones, e.g. from other tests/reloads,
+  // pass through untouched) and confirm the app still reaches the nav tree
+  // instead of showing the startup error modal.
+  it('2 | Recovers from a failed startupservice request via the single retry', () => {
+    let startupServiceCallCount = 0
+
+    cy.intercept('POST', '**/SASjsApi/stp/execute*', (req) => {
+      const isStartupService = JSON.stringify(req.body || '').includes(
+        'startupservice'
+      )
+
+      if (isStartupService) startupServiceCallCount++
+
+      if (isStartupService && startupServiceCallCount === 1) {
+        req.reply({ statusCode: 503, body: 'Service unavailable' })
+      } else {
+        req.continue()
+      }
+    }).as('stpExecute')
+
+    visitPage('home')
+
+    cy.get('.nav-tree clr-tree > clr-tree-node', {
+      // Longer than the retry's own 0-3s pause on top of the normal timeout.
+      timeout: longerCommandTimeout + 3000
+    }).should('exist')
+
+    cy.get('.abortMsg').should('not.exist')
+  })
+
   /**
    * Thist part will be needed if we add more tests in future
    */

client/src/app/editor/editor.component.ts

@@ -3455,6 +3455,14 @@ export class EditorComponent implements OnInit, AfterViewInit, OnDestroy {
         if (cellMeta && cellMeta.filter === false)
           hot.setCellMeta(row, col, 'filter', true)
 
+        // Toggling Delete? changes every other cell's validation exemption
+        // in this row (see DcValidator.setDefaultValidator) — re-validate so
+        // invalid highlights clear/reappear immediately instead of only at
+        // submit time.
+        if (colProp === '_____DELETE__THIS__RECORD_____') {
+          hot.validateRows([row], () => hot.render())
+        }
+
         this.executeDynamicCellValidationIfApplicable(colProp, col, row)
       }
     })

client/src/app/services/app.service.spec.ts

@@ -0,0 +1,141 @@
+import { EventEmitter } from '@angular/core'
+import { BehaviorSubject, Subject } from 'rxjs'
+import { AppService } from './app.service'
+
+/**
+ * AppService's constructor subscribes to several emitters/subjects (see
+ * subscribe() and the persistSelectedTheme check) and startUpData() touches
+ * a handful of methods on each collaborator — these stubs cover exactly
+ * that surface, not the full real services (no TestBed/DI needed, same
+ * plain-instantiation-with-stubs precedent as va-filter.service.spec.ts).
+ */
+const buildDeps = () => {
+  const licenceService: any = {
+    isAppActivated: new BehaviorSubject<boolean | null>(null),
+    activation: jasmine.createSpy('activation').and.resolveTo(undefined)
+  }
+  const eventService: any = {
+    showInfoModal: jasmine.createSpy('showInfoModal'),
+    startupDataLoaded: jasmine.createSpy('startupDataLoaded'),
+    toggleDarkMode: jasmine.createSpy('toggleDarkMode')
+  }
+  const sasService: any = {
+    loadStartupServiceEmitter: new EventEmitter<any>(),
+    requestSiteIdEmitter: new EventEmitter<string>(),
+    request: jasmine.createSpy('request')
+  }
+  const loggerService: any = {
+    log: jasmine.createSpy('log')
+  }
+  const appSettingsService: any = {
+    settings: new BehaviorSubject({ persistSelectedTheme: false } as any)
+  }
+  const router: any = {
+    events: new Subject(),
+    url: '',
+    navigateByUrl: jasmine.createSpy('navigateByUrl')
+  }
+  const appStoreService: any = {
+    getDcAdapterSettings: () => undefined
+  }
+
+  return {
+    licenceService,
+    eventService,
+    sasService,
+    loggerService,
+    appSettingsService,
+    router,
+    appStoreService
+  }
+}
+
+const buildAppService = (deps: ReturnType<typeof buildDeps>) =>
+  new AppService(
+    deps.licenceService,
+    deps.eventService,
+    deps.sasService,
+    deps.loggerService,
+    deps.appSettingsService,
+    deps.router,
+    deps.appStoreService
+  )
+
+// Minimal valid startupservice payload — just enough to pass startUpData()'s
+// missing-props check (Globvars/Sasdatasets/Saslibs/XLMaps).
+const validStartupResponse = () => ({
+  adapterResponse: {
+    SYSSITE: 'SITE1',
+    globvars: [{ ISADMIN: false, DC_ADMIN_GROUP: '', DCLIB: 'DCLIB' }],
+    sasdatasets: [],
+    saslibs: {},
+    xlmaps: []
+  }
+})
+
+describe('AppService - startup retry', () => {
+  it('retries once and succeeds without showing the error modal when only the first attempt rejects', async () => {
+    const deps = buildDeps()
+    deps.sasService.request.and.returnValues(
+      Promise.reject('timeout'),
+      Promise.resolve(validStartupResponse())
+    )
+
+    const appService = buildAppService(deps)
+    // Keep the retry pause instant in tests — see STARTUP_RETRY_PLAN.md.
+    ;(appService as any).retryOptions = {
+      wait: () => Promise.resolve(),
+      random: () => 0
+    }
+
+    await appService.startUpData()
+
+    expect(deps.sasService.request).toHaveBeenCalledTimes(2)
+    expect(deps.eventService.showInfoModal).not.toHaveBeenCalled()
+    expect(deps.eventService.startupDataLoaded).toHaveBeenCalled()
+  })
+
+  it('shows the error modal once (not twice) when both attempts reject', async () => {
+    const deps = buildDeps()
+    deps.sasService.request.and.returnValues(
+      Promise.reject('timeout-1'),
+      Promise.reject('timeout-2')
+    )
+
+    const appService = buildAppService(deps)
+    ;(appService as any).retryOptions = {
+      wait: () => Promise.resolve(),
+      random: () => 0
+    }
+
+    await appService.startUpData()
+
+    expect(deps.sasService.request).toHaveBeenCalledTimes(2)
+    expect(deps.eventService.showInfoModal).toHaveBeenCalledTimes(1)
+    expect(deps.licenceService.isAppActivated.value).toBeFalse()
+  })
+
+  it('does not retry a response that arrives but is missing required properties', async () => {
+    const deps = buildDeps()
+    deps.sasService.request.and.resolveTo({
+      adapterResponse: {
+        SYSSITE: 'SITE1',
+        // globvars deliberately omitted
+        sasdatasets: [],
+        saslibs: {},
+        xlmaps: []
+      }
+    })
+
+    const appService = buildAppService(deps)
+    ;(appService as any).retryOptions = {
+      wait: () => Promise.resolve(),
+      random: () => 0
+    }
+
+    await appService.startUpData()
+
+    expect(deps.sasService.request).toHaveBeenCalledTimes(1)
+    expect(deps.eventService.showInfoModal).toHaveBeenCalledTimes(1)
+  })
+})

client/src/app/services/app.service.ts

@@ -11,11 +11,15 @@ import { AppSettingsService } from './app-settings.service'
 import { AppThemes } from '../models/AppSettings'
 import { RequestWrapperResponse } from '../models/request-wrapper/RequestWrapperResponse'
 import { AppStoreService } from './app-store.service'
+import { retryOnce, RetryOnceOptions } from '../shared/utils/retry-once'
 
 @Injectable()
 export class AppService {
   public syssite = new BehaviorSubject<string[] | null>(null)
   private environmentInfo: EnvironmentInfo = {}
+  // Overridable in tests to keep the retry pause instant — see
+  // retry-once.spec.ts and app.service.spec.ts.
+  private retryOptions: RetryOnceOptions = {}
 
   constructor(
     private licenceService: LicenceService,
@@ -82,8 +86,13 @@ export class AppService {
   public async startUpData() {
     let startupServiceError = false
 
-    await this.sasService
-      .request('public/startupservice', null)
+    // Retry once on failure — multiple concurrent DC instances (e.g. VA
+    // multi-page reports) can cause this request to time out even though a
+    // second attempt succeeds. See STARTUP_RETRY_PLAN.md.
+    await retryOnce(
+      () => this.sasService.request('public/startupservice', null),
+      this.retryOptions
+    )
       .then(async (res: RequestWrapperResponse) => {
         this.syssite.next([res.adapterResponse.SYSSITE])
 

client/src/app/shared/dc-validator/dc-validator.ts

@@ -51,24 +51,33 @@ export class DcValidator {
 
     this.sasparams = sasparams
     this.hotInstance = hotInstance
-    // Each cols[i].COLTYPE is now a single JSON-object string (one per
-    // column, as of the vertical-array getdata.sas fix — issue #253), not
-    // the old comma-joined sasparams.COLTYPE list. DELETE_RECORD_COLUMN_RULE
-    // is prepended separately — see its own doc comment for why.
+    // Each cols[i].COLTYPE is a single JSON-object string (one per column, as
+    // of the vertical-array getdata.sas fix — issue #253), not the old
+    // comma-joined sasparams.COLTYPE list. DELETE_RECORD_COLUMN_RULE is
+    // injected separately — see its own doc comment for why.
     //
-    // cols[] is NOT guaranteed to arrive in COLHEADERS/display order — e.g.
-    // %mp_getcols can return it sorted by NAME. editor.component.ts pairs
-    // getRules()[i] with headerColumns[i] positionally, so this array must
-    // be built in VARNUM order (which does match COLHEADERS) rather than
-    // cols[]'s own order, or every rule after the first alphabetically-out-
-    // of-place column ends up bound to the wrong header.
-    const colsByVarnum = [...cols].sort((a, b) => a.VARNUM - b.VARNUM)
-    this.rules = [
-      DELETE_RECORD_COLUMN_RULE,
-      ...colsByVarnum
-        .map((col) => parseColTypeRow(col.COLTYPE))
-        .filter((rule): rule is DcValidation => rule !== undefined)
-    ]
+    // COLHEADERS is the authoritative grid order and the only safe key to
+    // build against: it is the variable list of the *output* dataset, where
+    // getdata.sas hoists the delete column and the PK to the front ("order
+    // delete var and pk fields at start of table"), whereas cols[].VARNUM
+    // describes the *source* table and cols[] itself can arrive sorted by
+    // NAME. Those orders diverge whenever the PK isn't already the source's
+    // first column, so rules are keyed by name and emitted in COLHEADERS
+    // order — never paired positionally against cols[] or VARNUM.
+    //
+    // A column with no parseable COLTYPE falls back to a plain text rule
+    // rather than being dropped: getRules() is consumed positionally against
+    // COLHEADERS, so a missing entry would shift every later column.
+    const ruleByName = new Map<string, DcValidation>([
+      [DELETE_RECORD_COLUMN_RULE.data, { ...DELETE_RECORD_COLUMN_RULE }]
+    ])
+    for (const col of cols) {
+      const rule = parseColTypeRow(col.COLTYPE)
+      if (rule) ruleByName.set(col.NAME, rule)
+    }
+    this.rules = sasparams.COLHEADERS.split(',').map(
+      (name) => ruleByName.get(name) ?? { data: name }
+    )
     this.rules = mergeColsRules(cols, this.rules, $dataFormats)
     this.rules = applyNumericFormats(this.rules)
     this.rules = mapIntlCellTypes(this.rules)
@@ -407,6 +416,27 @@ export class DcValidator {
       const colType = self.rules[i].type || ''
       let handsontableValid = null
 
+      // A row marked for delete is exempt from cell-level validation — its
+      // values are about to be removed. Two deliberate exceptions:
+      // - PK columns still validate: the key identifies WHICH record to
+      //   delete, and blank/invalid PKs are also skipped by the separate
+      //   duplicate-key check (validatePrimaryKeys), so this is the only
+      //   net that would otherwise catch them.
+      // - Uniqueness checks are unaffected by construction: they run in a
+      //   separate pass (editor.component.ts validatePrimaryKeys) that
+      //   never goes through this validator.
+      const markedForDelete =
+        this.instance?.getDataAtRowProp?.(
+          this.row,
+          '_____DELETE__THIS__RECORD_____'
+        ) === 'Yes'
+
+      if (markedForDelete && !self.primaryKeys.includes(col || '')) {
+        callback(true)
+
+        return
+      }
+
       // We call handsontable predefined validators, if it returns false, we validate as FALSE
       // If return is true, we validate with our custom DQ validations
 
@@ -558,8 +588,11 @@ export class DcValidator {
 
     if (rule.clsRule === 'EDIT') {
       rule.readOnly = false
+      // Guard the -1: splice(-1, 1) drops the LAST hidden column rather than
+      // no-op'ing, so an EDIT column that was never hidden would un-hide an
+      // unrelated one (e.g. a column hidden by a HIDDEN DQ rule).
       const hiddenIndex = this.hiddenColumns.indexOf(index)
-      this.hiddenColumns.splice(hiddenIndex, 1)
+      if (hiddenIndex !== -1) this.hiddenColumns.splice(hiddenIndex, 1)
     }
   }
 }

client/src/app/shared/dc-validator/tests/dc-validator.spec.ts

@@ -288,14 +288,11 @@ describe('DC Validator', () => {
     expect(dcValidator.getRoundDigits('SOME_NUM')).toBeUndefined()
   })
 
-  it('5 | orders rules by VARNUM even when cols[] arrives in a different order', () => {
-    // Regression test: %mp_getcols/cols1 can serialize cols[] alphabetically
-    // by NAME (confirmed against a real getdata.sas response) rather than in
-    // VARNUM/COLHEADERS order. editor.component.ts pairs
-    // dcValidator.getRules()[i] with headerColumns[i] positionally, so if
-    // DcValidator built rules in cols[]'s own (alphabetical) order, every
-    // rule after the first out-of-place column would end up bound to the
-    // wrong header — silently mis-rendering cell types/dropdowns.
+  it('5 | orders rules by COLHEADERS when cols[] arrives in a different order', () => {
+    // %mp_getcols/cols1 can serialize cols[] alphabetically by NAME rather
+    // than in COLHEADERS order (confirmed against a real getdata.sas
+    // response). getRules() is consumed positionally against headerColumns,
+    // so rule order must follow COLHEADERS regardless of cols[]'s own order.
     const alphabeticalCols = [...example_cols].sort((a, b) =>
       a.NAME.localeCompare(b.NAME)
     )
@@ -308,17 +305,243 @@ describe('DC Validator', () => {
       example_dqData
     )
 
-    const rules = dcValidator.getRules()
-    const colsByVarnum = [...example_cols].sort((a, b) => a.VARNUM - b.VARNUM)
+    expect(dcValidator.getRules().map((rule) => rule.data)).toEqual(
+      example_sasparams.COLHEADERS.split(',')
+    )
+  })
 
-    // rules[0] is the injected DELETE_RECORD_COLUMN_RULE (no VARNUM of its
-    // own — see its own doc comment); the rest must follow ascending VARNUM.
-    for (let i = 0; i < colsByVarnum.length; i++) {
-      expect(rules[i + 1].data).toEqual(colsByVarnum[i].NAME)
-    }
+  it("6 | keeps rules aligned with COLHEADERS when the PK is not the source table's first column", () => {
+    // Regression test for the real-world misalignment: getdata.sas hoists the
+    // delete column and PK to the front of the *output* dataset ("order
+    // delete var and pk fields at start of table"), so COLHEADERS order and
+    // cols[].VARNUM order (which describes the *source* table) diverge
+    // whenever the PK isn't already the source's first column. Worst case (as
+    // seen in the wild): the PK is the source's LAST column, so pairing rules
+    // to headers by VARNUM shifts every single column by one.
+    const cols: Col[] = [
+      // deliberately alphabetical, as the backend serializes it
+      makeCol('PRIMARY_KEY_FIELD', 6), // the PK — last in the source table
+      makeCol('SOME_CHAR', 1),
+      makeCol('SOME_DATE', 2),
+      makeCol('SOME_DROPDOWN', 3),
+      makeCol('SOME_NUM', 4),
+      makeCol('SOME_TIME', 5)
+    ]
+    const sasparams: SASParam = {
+      ...example_sasparams,
+      // PK hoisted to the front by getdata.sas, ahead of VARNUM 1..5
+      COLHEADERS:
+        '_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,' +
+        'SOME_DATE,SOME_DROPDOWN,SOME_NUM,SOME_TIME',
+      PK: 'PRIMARY_KEY_FIELD'
+    } as SASParam
+
+    const dcValidator: DcValidator = new DcValidator(
+      sasparams,
+      { vars: {} } as $DataFormats,
+      cols,
+      [],
+      []
+    )
+
+    expect(dcValidator.getRules().map((rule) => rule.data)).toEqual(
+      sasparams.COLHEADERS.split(',')
+    )
+  })
+
+  it('7 | falls back to a text rule rather than dropping a column with an unparseable COLTYPE', () => {
+    // A dropped rule would shorten getRules() and shift every later column
+    // against headerColumns. getdata.sas LEFT JOINs COLTYPE in, so an
+    // unmatched row can legitimately leave it unset.
+    const cols: Col[] = [
+      makeCol('PRIMARY_KEY_FIELD', 1),
+      { ...makeCol('SOME_CHAR', 2), COLTYPE: undefined },
+      { ...makeCol('SOME_NUM', 3), COLTYPE: 'not valid json' }
+    ]
+    const sasparams: SASParam = {
+      ...example_sasparams,
+      COLHEADERS:
+        '_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,SOME_NUM',
+      PK: 'PRIMARY_KEY_FIELD'
+    } as SASParam
+
+    const dcValidator: DcValidator = new DcValidator(
+      sasparams,
+      { vars: {} } as $DataFormats,
+      cols,
+      [],
+      []
+    )
+
+    expect(dcValidator.getRules().map((rule) => rule.data)).toEqual(
+      sasparams.COLHEADERS.split(',')
+    )
+  })
+
+  it('8 | does not un-hide an unrelated column when a CLS EDIT column was never hidden', () => {
+    // hiddenColumns.indexOf() returns -1 for a column that isn't hidden, and
+    // splice(-1, 1) drops the LAST element instead of no-op'ing — so an EDIT
+    // column used to silently un-hide a column hidden by a HIDDEN DQ rule.
+    //
+    // The EDIT column must not be the PK: setColumnLevelSecurity forces PK
+    // columns to clsRule 'READ' before the EDIT branch is reached, which would
+    // skip the splice entirely and leave this test passing against the bug.
+    const cols: Col[] = [
+      { ...makeCol('PK_COL', 1), CLS_RULE: 'READ' },
+      { ...makeCol('SECRET_COL', 2), CLS_RULE: 'READ' },
+      { ...makeCol('OPEN_COL', 3), CLS_RULE: 'EDIT' }
+    ]
+    const sasparams: SASParam = {
+      ...example_sasparams,
+      CLS_FLAG: 1,
+      COLHEADERS: '_____DELETE__THIS__RECORD_____,PK_COL,SECRET_COL,OPEN_COL',
+      PK: 'PK_COL'
+    } as SASParam
+    const dqRules: DQRule[] = [
+      { BASE_COL: 'SECRET_COL', RULE_TYPE: 'HIDDEN', RULE_VALUE: '', X: 0 }
+    ]
+
+    const dcValidator: DcValidator = new DcValidator(
+      sasparams,
+      { vars: {} } as $DataFormats,
+      cols,
+      dqRules,
+      []
+    )
+
+    // SECRET_COL is rules[2]; CLS_FLAG also hides the delete column (index 0).
+    // OPEN_COL (rules[3]) is EDIT and was never hidden, so the splice guard is
+    // what keeps SECRET_COL hidden here.
+    expect(dcValidator.getHiddenColumns()).toContain(2)
+  })
+
+  describe('9 | ignores cell validation on rows marked for delete', () => {
+    // An invalid cell shouldn't block submission if its row
+    // is about to be deleted anyway. The PK is exempt from the exemption — it
+    // identifies WHICH record to delete, and a blank/invalid PK on a delete
+    // row would also slip past the separate uniqueness check (which skips
+    // blank PKs).
+    //
+    // The validator is a plain `function` invoked with cellProperties as
+    // `this` (row + instance), so a minimal fake stands in for a real HOT
+    // instance here — only `instance.getDataAtRowProp` is read by the guard.
+    const fakeCellProps = (markedForDelete: boolean): any => ({
+      row: 0,
+      editor: undefined,
+      instance: {
+        getDataAtRowProp: (_row: number, prop: string) =>
+          prop === '_____DELETE__THIS__RECORD_____'
+            ? markedForDelete
+              ? 'Yes'
+              : 'No'
+            : undefined
+      }
+    })
+
+    const buildValidator = () =>
+      new DcValidator(
+        example_sasparams,
+        example_dataformats,
+        example_cols,
+        example_dqRules,
+        example_dqData
+      )
+
+    it('exempts an invalid non-PK cell on a delete-marked row', () => {
+      const dcValidator = buildValidator()
+      const someNumRule = dcValidator.getRule('SOME_NUM')
+
+      someNumRule!.validator!.call(
+        fakeCellProps(true),
+        'not a number',
+        (valid: boolean) => {
+          expect(valid).toBeTrue()
+        }
+      )
+    })
+
+    it('exempts a NOTNULL-empty non-PK cell on a delete-marked row', () => {
+      const dcValidator = buildValidator()
+      const someNumRule = dcValidator.getRule('SOME_NUM')
+
+      someNumRule!.validator!.call(
+        fakeCellProps(true),
+        null,
+        (valid: boolean) => {
+          expect(valid).toBeTrue()
+        }
+      )
+    })
+
+    it('still rejects an invalid non-PK cell when the row is not marked for delete', () => {
+      const dcValidator = buildValidator()
+      const someNumRule = dcValidator.getRule('SOME_NUM')
+
+      someNumRule!.validator!.call(
+        fakeCellProps(false),
+        'not a number',
+        (valid: boolean) => {
+          expect(valid).toBeFalse()
+        }
+      )
+    })
+
+    it('still accepts a valid non-PK cell when the row is not marked for delete', () => {
+      const dcValidator = buildValidator()
+      const someNumRule = dcValidator.getRule('SOME_NUM')
+
+      someNumRule!.validator!.call(
+        fakeCellProps(false),
+        2,
+        (valid: boolean) => {
+          expect(valid).toBeTrue()
+        }
+      )
+    })
+
+    it('still rejects an invalid PK cell even when the row is marked for delete', () => {
+      const dcValidator = buildValidator()
+      const pkRule = dcValidator.getRule('PRIMARY_KEY_FIELD')
+
+      pkRule!.validator!.call(
+        fakeCellProps(true),
+        'not a number',
+        (valid: boolean) => {
+          expect(valid).toBeFalse()
+        }
+      )
+    })
+
+    it('treats the edit-record modal path (dummy HOT instance, no data) as not marked for delete', () => {
+      const dcValidator = buildValidator()
+      const someNumRule = dcValidator.getRule('SOME_NUM')
+
+      // executeHotValidator builds cellProperties against a real but
+      // data-less Handsontable instance — getDataAtRowProp() there returns
+      // undefined, never 'Yes', so the exemption never applies in the modal.
+      dcValidator.executeHotValidator(someNumRule!, null, (valid: boolean) => {
+        expect(valid).toBeFalse()
+      })
+    })
   })
 })
 
+/** Minimal cols[] entry — only the fields rule ordering depends on. */
+const makeCol = (name: string, varnum: number): Col =>
+  ({
+    NAME: name,
+    VARNUM: varnum,
+    LABEL: name,
+    FMTNAME: '',
+    DDTYPE: 'CHARACTER',
+    TYPE: '',
+    CLS_RULE: 'READ',
+    MEMLABEL: '',
+    DESC: '',
+    LONGDESC: '',
+    COLTYPE: `{"data":"${name}"}`
+  }) as Col
+
 const example_dqData = [
   {
     BASE_COL: 'SOME_DROPDOWN',
@@ -588,15 +811,22 @@ const example_cols = [
   }
 ]
 
+// COLHEADERS mirrors a real getdata.sas response: the *output* dataset's
+// variable list, i.e. the delete column and PK hoisted to the front, then the
+// remaining source columns in VARNUM order. It drives rule order, so it must
+// list every example_cols entry.
 const example_sasparams = {
   CLS_FLAG: 0,
-  COLHEADERS: 'head1 head2',
+  COLHEADERS:
+    '_____DELETE__THIS__RECORD_____,PRIMARY_KEY_FIELD,SOME_CHAR,SOME_DROPDOWN,' +
+    'SOME_NUM,SOME_CHAR_LOW,SOME_CHAR_ANY,SOME_DROPDOWN_HARD,SOME_DATE,' +
+    'SOME_DATETIME,SOME_TIME,SOME_SHORTNUM,SOME_BESTNUM',
   DTTMVARS: 'dttm vars',
   DTVARS: 'dt vars',
   FILTER_TEXT: 'filter text',
   LOADTYPE: 'load type',
-  PK: 'primary key',
-  PKCNT: 0,
+  PK: 'PRIMARY_KEY_FIELD',
+  PKCNT: 1,
   RK_FLAG: 0,
   TMVARS: 'string'
 }

client/src/app/shared/dc-validator/utils/mergeColsRules.ts

@@ -27,7 +27,7 @@ export const mergeColsRules = (
         : colFormats.type
 
     if (rule && col.DESC) rule.desc = col.DESC
-    if (rule && colFormats.length) rule.length = parseInt(colFormats.length)
+    if (rule && colFormats?.length) rule.length = parseInt(colFormats.length)
     if (rule && col.CLS_RULE) rule.clsRule = col.CLS_RULE
     if (rule && colFormats?.type) rule.sasType = colFormats.type
   }

client/src/app/shared/utils/retry-once.spec.ts

@@ -0,0 +1,63 @@
+import { retryOnce } from './retry-once'
+
+describe('retryOnce', () => {
+  it("resolves with the first attempt's value and never waits when it succeeds", async () => {
+    const requestFn = jasmine.createSpy('requestFn').and.resolveTo('ok')
+    const wait = jasmine.createSpy('wait').and.resolveTo(undefined)
+
+    const result = await retryOnce(requestFn, { wait })
+
+    expect(result).toBe('ok')
+    expect(requestFn).toHaveBeenCalledTimes(1)
+    expect(wait).not.toHaveBeenCalled()
+  })
+
+  it('retries exactly once after a rejection and resolves with the retry value', async () => {
+    const requestFn = jasmine
+      .createSpy('requestFn')
+      .and.returnValues(Promise.reject('first-error'), Promise.resolve('ok'))
+    const wait = jasmine.createSpy('wait').and.resolveTo(undefined)
+
+    const result = await retryOnce(requestFn, { wait })
+
+    expect(result).toBe('ok')
+    expect(requestFn).toHaveBeenCalledTimes(2)
+    expect(wait).toHaveBeenCalledTimes(1)
+  })
+
+  it('derives the pre-retry pause from random within [0, maxDelayMs)', async () => {
+    const requestFn = jasmine
+      .createSpy('requestFn')
+      .and.returnValues(Promise.reject('err'), Promise.resolve('ok'))
+    const wait = jasmine.createSpy('wait').and.resolveTo(undefined)
+
+    await retryOnce(requestFn, { wait, random: () => 0.5 })
+    expect(wait).toHaveBeenCalledWith(1500)
+
+    wait.calls.reset()
+    requestFn.and.returnValues(Promise.reject('err'), Promise.resolve('ok'))
+    await retryOnce(requestFn, { wait, random: () => 0 })
+    expect(wait).toHaveBeenCalledWith(0)
+
+    wait.calls.reset()
+    requestFn.and.returnValues(Promise.reject('err'), Promise.resolve('ok'))
+    await retryOnce(requestFn, { wait, random: () => 0.999999 })
+    expect(wait).toHaveBeenCalledWith(jasmine.any(Number))
+    expect(wait.calls.mostRecent().args[0]).toBeLessThan(3000)
+  })
+
+  it('rejects with the second error when both attempts fail, never trying a third time', async () => {
+    const requestFn = jasmine
+      .createSpy('requestFn')
+      .and.returnValues(
+        Promise.reject('first-error'),
+        Promise.reject('second-error')
+      )
+    const wait = jasmine.createSpy('wait').and.resolveTo(undefined)
+
+    await expectAsync(retryOnce(requestFn, { wait })).toBeRejectedWith(
+      'second-error'
+    )
+    expect(requestFn).toHaveBeenCalledTimes(2)
+  })
+})

client/src/app/shared/utils/retry-once.ts

@@ -0,0 +1,33 @@
+export interface RetryOnceOptions {
+  /** Upper bound of the random pre-retry delay, in ms. Default 3000. */
+  maxDelayMs?: number
+  /** Injectable for tests. Default Math.random. */
+  random?: () => number
+  /** Injectable for tests. Default setTimeout-based wait. */
+  wait?: (ms: number) => Promise<void>
+}
+
+const defaultWait = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms))
+
+/**
+ * Runs requestFn; if it rejects, waits a random 0..maxDelayMs and tries
+ * exactly once more. A second rejection propagates as-is. Multiple
+ * concurrent DC instances (e.g. VA multi-page reports) can cause the
+ * startup request to time out even though a second attempt succeeds.
+ */
+export const retryOnce = async <T>(
+  requestFn: () => Promise<T>,
+  options?: RetryOnceOptions
+): Promise<T> => {
+  const maxDelayMs = options?.maxDelayMs ?? 3000
+  const random = options?.random ?? Math.random
+  const wait = options?.wait ?? defaultWait
+
+  try {
+    return await requestFn()
+  } catch {
+    await wait(random() * maxDelayMs)
+    return requestFn()
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dcfrontend",
-  "version": "7.10.0",
+  "version": "7.11.0",
   "description": "Data Controller",
   "devDependencies": {
     "@saithodev/semantic-release-gitea": "^2.1.0",

sas/package-lock.json

@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "4.18.0",
-        "@sasjs/core": "4.67.3"
+        "@sasjs/core": "4.68.1"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -124,7 +124,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -147,7 +146,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       }
@@ -281,6 +279,12 @@
         "sasjs": "build/index.js"
       }
     },
+    "node_modules/@sasjs/cli/node_modules/@sasjs/core": {
+      "version": "4.67.3",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.67.3.tgz",
+      "integrity": "sha512-7s4UeVN4slF1URo1C5/UnNVv+TonUg+vBktFRiFhcbFYr3RnfXyClBPED6nQOF8AFd2xdXEss58hsCeIebr4lg==",
+      "license": "MIT"
+    },
     "node_modules/@sasjs/cli/node_modules/@sasjs/utils": {
       "version": "3.5.8",
       "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.5.8.tgz",
@@ -317,9 +321,9 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "4.67.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.67.3.tgz",
-      "integrity": "sha512-7s4UeVN4slF1URo1C5/UnNVv+TonUg+vBktFRiFhcbFYr3RnfXyClBPED6nQOF8AFd2xdXEss58hsCeIebr4lg==",
+      "version": "4.68.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.68.1.tgz",
+      "integrity": "sha512-BjoOzV7h7guCShW4JsFHvB1NNLCKHhPI5708OtEk29itQbgVCMRlEXKFRbnMiuNklktDQDi8dR4XZZ0aK7/4Wg==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
@@ -466,7 +470,6 @@
       "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz",
       "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "follow-redirects": "^1.16.0",
         "form-data": "^4.0.5",
@@ -2156,7 +2159,6 @@
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
       "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
       "license": "BSD-3-Clause",
-      "peer": true,
       "dependencies": {
         "psl": "^1.1.33",
         "punycode": "^2.1.1",

sas/package.json

@@ -29,6 +29,6 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "4.18.0",
-    "@sasjs/core": "4.67.3"
+    "@sasjs/core": "4.68.1"
   }
 }

sas/sasjs/macros/bitemporal_closeouts.sas

@@ -35,6 +35,7 @@
   @li mf_getuniquename.sas
   @li mf_getuser.sas
   @li mf_getvartype.sas
+  @li mf_mkdir.sas
   @li mp_lockanytable.sas
   @li dc_assignlib.sas
 
@@ -136,6 +137,7 @@ run;
   libname &tmplib (work);
 %end;
 %else %do;
+  %mf_mkdir(&AUDITFOLDER)
   filename tmp "&AUDITFOLDER/deleterecords.sas" lrecl=10000;
   libname &tmplib "&AUDITFOLDER";
 %end;

sas/sasjs/macros/bitemporal_dataloader.sas

@@ -111,6 +111,7 @@ Areas for optimisation
   @li mp_lockfilecheck.sas
   @li mp_retainedkey.sas
   @li mp_storediffs.sas
+  @li mp_rowhash.sas
 
   @version 9.3
   @author 4GL Apps Ltd.
@@ -193,7 +194,7 @@ Areas for optimisation
 
 /* when changing this macro, update the version num here */
 %local ver;
-%let ver=32;
+%let ver=33;
 %put &sysmacroname entry vars:;
 %put _local_;
 
@@ -246,7 +247,7 @@ run;
 )
 
 
-%local cols idx_pk md5_col ;
+%local cols idx_pk md5_col hash_char_vars hash_num_vars ;
 %let md5_col=___TMP___md5;
 %let check_uniqueness=%upcase(&check_uniqueness);
 %let RK_UPDATE_MAXKEYTABLE=%upcase(&RK_UPDATE_MAXKEYTABLE);
@@ -321,14 +322,20 @@ select name into: cols separated by ','
     (%upcase("&bus_from","&bus_to"
             ,"&tech_from","&tech_to"
             ,"&processed","&delete_col")) ;
-select case when type in (2,6) then cats('put(md5(trim(',name,')),$hex32.)')
-        /* multiply by 1 to strip precision errors (eg 0 != 0) */
-        /* but ONLY if not missing, else will lose any special missing values */
-        else cats('put(md5(trim(put(ifn(missing('
-          ,name,'),',name,',',name,'*1),binary64.))),$hex32.)') end
-        into: stripcols separated by '||'
+/* Character variables are hashed separately from numerics so that the
+   iterative hash can be built with arrays rather than by concatenating
+   hundreds of 32-byte hex strings. */
+select name into: hash_char_vars separated by ' '
   from work.bitemp_cols
-  where upcase(name) not in
+  where type in (2,6)
+    and upcase(name) not in
+    (%upcase("&bus_from","&bus_to"
+            ,"&tech_from","&tech_to"
+            ,"&processed","&delete_col")) ;
+select name into: hash_num_vars separated by ' '
+  from work.bitemp_cols
+  where type not in (2,6)
+    and upcase(name) not in
     (%upcase("&bus_from","&bus_to"
             ,"&tech_from","&tech_to"
             ,"&processed","&delete_col")) ;
@@ -342,7 +349,8 @@ select case when type in (2,6) then cats('put(md5(trim(',name,')),$hex32.)')
 %let tech_to_fmt = format=datetime19.;
 
 
-%put &=stripcols;
+%put &=hash_char_vars;
+%put &=hash_num_vars;
 %put &=pk;
 
 data _null_;
@@ -368,8 +376,8 @@ data _null_;
     call symputx('processed_fmt',fmt,'L');
 run;
 
-%if %index(%quote(&cols),___TMP___) %then %do;
-  %let msg=%str(Table contains a variable name containing "___TMP___".%trim(
+%if %index(%quote(&cols),___TMP___) or %index(%quote(&cols),_____) %then %do;
+  %let msg=%str(Table contains a variable name containing "___TMP___" or "_____".%trim(
     ) This may conflict with temp variable generation!!);
   %mp_abort(msg=&msg,mac=bitemporal_dataloader);
   %let syscc=5;
@@ -485,7 +493,10 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
     &bus_to= %unquote(&bus_to_override) ;
   %end;
   length &md5_col $32;
-  &md5_col=put(md5(&stripcols),hex32.);
+  %mp_rowhash(md5_col=&md5_col
+    ,cvars=&hash_char_vars
+    ,nvars=&hash_num_vars
+  )
   %if %length(&processed)>0 %then %do;
     format &processed &processed_fmt;
     &processed=&now;
@@ -772,13 +783,17 @@ or &engine_type=SNOW or &engine_type=SASIOSNF
   ,msg=%str(syscc=&syscc sqlrc=&sqlrc)
 )
 
-%put hashcols2=&stripcols;
-proc sql;
-create table work.bitemp1_current(drop=___TMP___NEW_FLG) as
-  select *
-    , put(md5(&stripcols),$hex32.) as &md5_col
-  from work.bitemp0_base (drop=&md5_col)
-  where ___TMP___NEW_FLG=0;
+%put hash_char_vars=&hash_char_vars;
+%put hash_num_vars=&hash_num_vars;
+data work.bitemp1_current(drop=___TMP___NEW_FLG);
+  set work.bitemp0_base(drop=&md5_col);
+  if ___TMP___NEW_FLG=0;
+  length &md5_col $32;
+  %mp_rowhash(md5_col=&md5_col
+    ,cvars=&hash_char_vars
+    ,nvars=&hash_num_vars
+  )
+run;
 
 /**
 * NEW records were identified in ___TMP___NEW_FLG in bitemp0_base
@@ -1061,7 +1076,9 @@ run;
 */
 
 %if &loadtype=BITEMPORAL %then %do;
-  %let cat_string=catx('|' ,&bus_from,&bus_to);
+  /* For bitemporal we also include the business dates in the comparison hash.
+     The business dates are passed as prefix numerics so that both lookup and
+     update tables use exactly the same hashing order. */
 
   data work.bitemp5a_lkp (keep=&md5_col)
     %if "%substr(&sysver,1,1)" ne "4" & "%substr(&sysver,1,1)" ne "5" %then %do;
@@ -1069,8 +1086,10 @@ run;
     %end;
     ;
     set work.bitemp0_base;
-    /* for BITEMPORAL we need to compare business dates also */
-    &md5_col=put(md5(&cat_string!!'|'!!&stripcols),$hex32.);
+    %mp_rowhash(md5_col=&md5_col
+      ,cvars=&hash_char_vars
+      ,nvars=&bus_from &bus_to &hash_num_vars
+    )
   run;
 
   data bitemp5b_updates;
@@ -1081,7 +1100,10 @@ run;
       md5_lkp.definedone();
     end;
     /* drop old md5 col as will rebuild with new business dates */
-    &md5_col=put(md5(&cat_string!!'|'!!&stripcols),$hex32.) ;
+    %mp_rowhash(md5_col=&md5_col
+      ,cvars=&hash_char_vars
+      ,nvars=&bus_from &bus_to &hash_num_vars
+    )
     if md5_lkp.check()=0 then delete;
   run;
 

sas/sasjs/macros/bitemporal_dataloader.test.2.sas

@@ -1035,7 +1035,7 @@ run;
     set appendtable&testnum;
     /* delete even numbered observations */
     if mod(_n_,2)=0 and _n_ ne 2 then _____DELETE__THIS__RECORD_____='YES';
-    else _____DELETE__THIS__RECORD_____=0;
+    else _____DELETE__THIS__RECORD_____='NO';
   run;
 
   %bitemporal_dataloader(dclib=work

sas/sasjs/macros/mpe_checkrestore.sas

@@ -54,7 +54,7 @@
     */
   %local audtab;
   proc sql noprint;
-  select coalescec(audit_libds,"&dc_libref..MPE_AUDIT") into: audtab
+  select coalescec(audit_libds,"&dc_libref..MPE_AUDIT") into: audtab trimmed
   from &dc_libref..MPE_TABLES
   where &dc_dttmtfmt. lt tx_to
     and libref="%scan(&libds,1,.)" and dsn="%scan(&libds,2,.)";

sas/sasjs/macros/mpe_loader.sas

@@ -16,7 +16,6 @@
   @li dc_assignlib.sas
   @li mf_getattrn.sas
   @li mf_getuser.sas
-  @li mf_mkdir.sas
   @li mf_verifymacvars.sas
   @li mp_abort.sas
   @li mp_cntlout.sas

sas/sasjs/macros/mpe_refreshtables.test.sas

@@ -37,12 +37,12 @@ create table work.libinfo as
   from &mpelib..mpe_datacatalog_libs(where=(&dc_dttmtfmt. lt tx_to)) a
   left join &mpelib..mpe_datastatus_libs(where=(&dc_dttmtfmt. lt tx_to)) b
   on a.libref=b.libref
-  where a.libref="&libref";
+  where a.libref="FMTONLY";
 
 %let test1=0;
 data _null_;
   set work.libinfo;
-  call symputx('test1',table_cnt);
+  call symputx('test1',catalog_cnt);
 run;
 %mp_assert(
   iftrue=(&test1>0),

sas/sasjs/services/editors/getdata.sas

@@ -626,6 +626,7 @@ create table dqdata as
     %put &=source;
     %put &=lib;
     %dc_assignlib(READ,&lib)
+    %dc_casload(&lib..&ds)
     proc sql;
     create table dqdata&x as
       select distinct "&&base_col&x" as base_col length=32

sas/sasjs/services/editors/getdynamiccolvals.test.2.sas

@@ -57,7 +57,7 @@ data work.check;
   val='VAR_BUSFROM';output;
   val='VAR_BUSTO';output;
 run;
-%mp_assertcolvals(work.DYNAMIC_EXTENDED_VALUES.element2,
+%mp_assertcolvals(work.DYNAMIC_EXTENDED_VALUES.EXTRA_COL_NAME,
   checkvals=work.check.val,
   desc=Correct values found in EXTRA_COL_NAME response,
   test=ALLVALS

sas/sasjs/services/editors/restore.sas

@@ -7,6 +7,7 @@
   @li dc_assignlib.sas
   @li mcf_getfmttype.sas
   @li mf_nobs.sas
+  @li mf_mkdir.sas
   @li mp_abort.sas
   @li mp_applyformats.sas
   @li mp_ds2csv.sas

sas/sasjs/services/public/getrawdata.test.sas

@@ -30,11 +30,12 @@ run;
 data work.results;
   infile webout;
   input;
-  if _infile_=:'PRIMARY_KEY_FIELD,SOME_CHAR' then do;
+  if _infile_=:'PRIMARY_KEY_FIELD, SOME_CHAR' then do;
     output;
     output;
     stop;
   end;
+  else if _n_<5 then putlog _infile_;
 run;
 
 %mp_assertdsobs(work.results,

sas/sasjs/targets/viya/macros_viya/dc_casload.sas

@@ -66,7 +66,7 @@
   %end;
 
   proc casutil;
-    load casdata="&ds"
+    load casdata="&ds..sashdat"
         incaslib="&caslib"
         casout="&ds"
         outcaslib="&caslib"

sas/sasjs/tests/testsetup.sas

@@ -64,6 +64,38 @@ data dctest.mpe_x_test;
   set &DC_LIBREF..mpe_x_test;
 run;
 
+data dctest.wideboy;
+  /* Create 500 character variables and 500 numeric variables */
+  array char_fields[500] $20 char1-char500;
+  array num_fields[500]       num1-num500;
+  length alphabet $62;
+  alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+
+  /* Set a seed for reproducible random data */
+  call streaminit(12345);
+
+  do row_id = 1 to 1000;
+
+    /* Generate random 20-character alphanumeric values */
+    do field = 1 to dim(char_fields);
+      do position = 1 to 20;
+        substr(char_fields[field], position, 1) =
+          substr(alphabet,
+            floor(rand('uniform') * length(alphabet)) + 1,
+            1
+        );
+      end;
+    end;
+    /* Generate random numeric values from a normal distribution */
+    do field = 1 to dim(num_fields);
+      num_fields[field] = ceil(rand('normal', 0, 1));
+    end;
+    output;
+  end;
+  stop;
+run;
+
+
 /* test library with only one format catalog */
 %mf_mkdir(&testloc/fmtonly)
 libname fmtonly "&testloc/fmtonly";
@@ -90,8 +122,7 @@ proc format library=dctest.dcfmts;
   invalue indays (default=13) other=42;
 run;
 proc sql;
-delete from &DC_LIBREF..mpe_tables
-  where libref="DCTEST" and dsn='DCFMTS-FC';
+delete from &DC_LIBREF..mpe_tables where libref="DCTEST";
 data work.append;
   if 0 then set &DC_LIBREF..mpe_tables;
   tx_from=0;
@@ -102,6 +133,11 @@ data work.append;
   loadtype='FORMAT_CAT';
   num_of_approvals_required=1;
   output;
+  dsn='WIDEBOY';
+  buskey='ROW_ID CHAR1 NUM1';
+  loadtype='UPDATE';
+  output;
+  stop;
 run;
 proc append base=&dc_libref..mpe_tables data=work.append;
 run;