test(embed): track the held-out spec, and keep its comments self-contained

Review follow-up on the browser harness.

The exclusion of va-ddc-host now names the issue that tracks re-enabling it
(dc/dc#449), so the coverage gate cannot report a rotted exclusion as accounted
for. The comment also states the ordering constraint the spec carries: it clears
the DC_JSLIB filter tables per case, and every spec in that job shares one mock
estate, so it has to run last or be given its own.

The spec's comments and test name describe the behaviour and the harness, with
no ticket numbers, per AGENTS.md. The va-filter docstring says what the inference
is: a guess from the value's shape alone, which can disagree with the column in
either direction, and which costs the deferred call rather than the page because
the attempt suppresses the abort modal and falls back to the unfiltered load.

.gitea/scripts/cypress-specs.sh

@@ -29,7 +29,16 @@ SPECS="csv-limited hook-programs liveness editor excel-multi-load excel csv filt
 
 # Specs that exist but are held out of CI, each with the reason. Listed here so
 # the coverage check below can tell a deliberate exclusion from a forgotten file.
-EXCLUDED=""
+# va-ddc-host drives the app through sasjs/server's /ddc-host page with
+# `timing=early`, which needs a server release that carries it - the version this
+# job pins does not, so the spec cannot run until SASJS_SERVER_VERSION moves.
+# https://git.datacontroller.io/dc/dc/issues/449 tracks re-enabling it.
+#
+# When it is re-enabled it has to run LAST, or be given its own estate: it clears
+# the DC_JSLIB filter tables (mpe_filtersource, mpe_filteranytable,
+# mpe_maxkeyvalues) before each case, and every spec in this job shares one mock
+# estate. See resetFilterStore in client/cypress/e2e/va-ddc-host.cy.ts.
+EXCLUDED="va-ddc-host"
 
 case "${1:-}" in
   all)

client/cypress.config.ts

@@ -20,6 +20,12 @@ export default defineConfig({
   env: {
     hosturl: 'http://localhost:4200',
     appLocation: '',
+    // The SASjs server the mock services run against, and the location the CI
+    // job streams the built app to (see sas/mocks/sasjs/sasjsconfig.json and
+    // CONTRIBUTING.md). `va-ddc-host.cy.ts` frames that build from the server's
+    // own /ddc-host page, so it needs both rather than `hosturl`.
+    sasjsServerUrl: 'http://localhost:5000',
+    streamedAppLocation: '/Public/app/dc',
     site_id_SAS9: '70221618',
     site_id_SASVIYA: '70253615',
     site_id_SASJS: '123',

client/cypress/e2e/va-ddc-host.cy.ts

@@ -0,0 +1,199 @@
+/**
+ * SAS Visual Analytics data-driven content, end to end.
+ *
+ * VA hands its DDC message to the embedded object as the document appears -
+ * before the app boots - so the app's earliest listener (src/assets/va-early.js)
+ * captures it, and the editor applies a pre-set filter on the very first load.
+ * Posting a message into a running page exercises a different path: it arrives
+ * after getdata, with column metadata available, so a date is formatted
+ * correctly there whatever the client does. Those two paths take the same
+ * clause builder with different inputs, so a value can be formatted correctly on
+ * one and not the other - which is why this spec pins the timing as well as the
+ * clause.
+ *
+ * sasjs/server's /ddc-host supplies the missing half: an iframe carrying VA's
+ * sandbox attribute set, and `timing=early` to deliver the message repeatedly
+ * from the moment the frame is created, so a copy lands before the app
+ * bootstraps. The app under test is the build streamed by the same server, so
+ * everything here runs against one origin and needs no SAS.
+ *
+ * Where the evidence is: the mock public/validatefilter stores what it is sent
+ * (`mpe_filtersource.json` on the drive), so the clause is read back from there
+ * rather than inferred from a request payload. The host page's own log is the
+ * second witness - it records the capture before the frame finishes loading,
+ * which is the timing that makes the case meaningful. The frame's internals are
+ * deliberately not read: Cypress proxies the framed document, so its
+ * `contentWindow` and navigation timings cannot be trusted, and the drive is the
+ * authority anyway.
+ */
+
+const serverUrl = (
+  Cypress.env('sasjsServerUrl') || 'http://localhost:5000'
+).replace(/\/$/, '')
+const appLoc = Cypress.env('streamedAppLocation') || '/Public/app/dc'
+const table = 'DC_JSLIB.MPE_X_TEST'
+const filterSourcePath = `${appLoc}/data/DC_JSLIB/mpe_filtersource.json`
+const appUrl = `${serverUrl}/AppStream/clickme/#/editor/${table}?embed=va`
+const longerCommandTimeout = Cypress.env('longerCommandTimeout') || 50000
+
+interface StoredClause {
+  FILTER_HASH: string
+  VARIABLE_NM: string
+  OPERATOR_NM: string
+  RAW_VALUE: string
+  PROCESSED_DTTM: number
+}
+
+/**
+ * A VA message carrying one parameter. `dataType: 'string'` is what a date
+ * parameter carries when VA displays it with a month name, and on the
+ * metadata-less path this type and the value's shape are all the builder has to
+ * go on. The `dataType: 'number'` variant is covered by the unit spec, where the
+ * dropped-clause branch can be driven directly.
+ */
+const vaMessage = (label: string, value: string, dataType = 'string') => ({
+  version: '1',
+  resultName: 'dd1',
+  rowCount: 0,
+  availableRowCount: 0,
+  data: [],
+  columns: [],
+  parameters: [{ name: 'pr1', label, dataType, value }]
+})
+
+const ddcHostUrl = (message: object) =>
+  `${serverUrl}/ddc-host?` +
+  new URLSearchParams({
+    url: appUrl,
+    message: JSON.stringify(message),
+    timing: 'early'
+  })
+
+/** Everything the mock validatefilter has stored. */
+const readStoredClauses = (): Cypress.Chainable<StoredClause[]> =>
+  cy
+    .request({
+      url: `${serverUrl}/SASjsApi/drive/file`,
+      qs: { _filePath: filterSourcePath },
+      failOnStatusCode: false
+    })
+    .then((res: any) => {
+      if (res.status === 404) return []
+      const body =
+        typeof res.body === 'string' ? JSON.parse(res.body) : res.body
+      return (body?.rows ?? []) as StoredClause[]
+    })
+
+/**
+ * Empty the filter store before a case.
+ *
+ * The mock `mpFilterStore` derives a hash from the clause and reuses the existing
+ * FILTER_RK when it has seen that clause before, so a repeated case writes
+ * nothing - which would make "the clause this run stored" indistinguishable from
+ * whatever an earlier run (or a retry) left behind. Starting from an empty store
+ * keeps the assertion about this run, and it also keeps FILTER_RK allocation
+ * predictable.
+ */
+const resetFilterStore = () => {
+  for (const name of [
+    'mpe_filtersource',
+    'mpe_filteranytable',
+    'mpe_maxkeyvalues'
+  ]) {
+    cy.request({
+      method: 'DELETE',
+      url: `${serverUrl}/SASjsApi/drive/file`,
+      qs: { _filePath: `${appLoc}/data/DC_JSLIB/${name}.json` },
+      failOnStatusCode: false
+    })
+  }
+}
+
+/** The clause stored for `variable`, retried until the service has written it. */
+const storedClauseFor = (
+  variable: string,
+  attempt = 0
+): Cypress.Chainable<StoredClause> =>
+  readStoredClauses().then((rows: StoredClause[]) => {
+    const stored = rows.filter(
+      (row) => String(row.VARIABLE_NM).toUpperCase() === variable
+    )
+    if (stored.length) return stored[stored.length - 1]
+    if (attempt > 60) {
+      throw new Error(
+        `no clause for ${variable} was stored (60 reads, ${rows.length} rows)`
+      )
+    }
+    cy.wait(250)
+    return storedClauseFor(variable, attempt + 1)
+  })
+
+/**
+ * Mount the host page, then assert the two facts that make a case valid: the app
+ * captured the message, and it did so before the frame finished loading (the
+ * app's earliest listener runs at document parse, so this is the window a report
+ * refresh exercises). A capture that only lands after load takes the metadata
+ * path, which formats a date correctly whatever the client does, and so would
+ * prove nothing about the metadata-less path.
+ */
+const mountWithEarlyMessage = (message: object) => {
+  cy.visit(ddcHostUrl(message))
+  cy.get('#log', { timeout: longerCommandTimeout }).should(($log) => {
+    const lines = $log.text()
+    const captured = lines.indexOf('early replay: the app captured the message')
+    const loaded = lines.indexOf('frame loaded')
+    expect(captured, 'the app captured the message').to.be.greaterThan(-1)
+    expect(loaded, 'the frame finished loading').to.be.greaterThan(-1)
+    expect(captured, 'capture happened before the frame loaded').to.be.lessThan(
+      loaded
+    )
+  })
+}
+
+/**
+ * Run one case and return the clause the service received. Cypress queues the
+ * commands in call order, so the store is empty before the page is visited and
+ * the read runs only once the page has captured and applied the message.
+ */
+const applyVaFilter = (
+  message: object,
+  variable: string
+): Cypress.Chainable<StoredClause> => {
+  resetFilterStore()
+  mountWithEarlyMessage(message)
+  return storedClauseFor(variable)
+}
+
+context('VA data-driven content through /ddc-host:', () => {
+  it('1 | stores a month-name date as a date literal', () => {
+    // `August 02, 1971` is how VA displays SOME_DATE=4231 under WORDDATE. A
+    // month-name value formatted as character cannot be compared with a numeric
+    // date column: on a live estate that is the PROC SQL type clash inside
+    // mp_filtervalidate, and on this mock estate it is the quoted literal this
+    // assertion rejects.
+    applyVaFilter(vaMessage('SOME_DATE', 'August 02, 1971'), 'SOME_DATE').then(
+      (clause: StoredClause) => {
+        expect(clause.OPERATOR_NM).to.equal('=')
+        expect(clause.RAW_VALUE).to.equal("'02AUG1971'd")
+      }
+    )
+  })
+
+  it('2 | stores a DATE9 date as a date literal (control)', () => {
+    // DATE9, the control: it separates a parsing problem in the month-name case
+    // from a harness that stopped delivering dates at all.
+    applyVaFilter(vaMessage('SOME_DATE', '01DEC2021'), 'SOME_DATE').then(
+      (clause: StoredClause) => {
+        expect(clause.RAW_VALUE).to.equal("'01DEC2021'd")
+      }
+    )
+  })
+
+  it('3 | stores a character parameter as a quoted string (control)', () => {
+    applyVaFilter(vaMessage('SOME_DROPDOWN', 'Option 2'), 'SOME_DROPDOWN').then(
+      (clause: StoredClause) => {
+        expect(clause.RAW_VALUE).to.equal("'Option 2'")
+      }
+    )
+  })
+})

client/src/app/services/va-filter.service.spec.ts

@@ -958,6 +958,58 @@ describe('VaFilterService', () => {
         ['SOME_DTTM', "'01JAN1960:00:06:05'dt"]
       ])
     })
+
+    it('infers a month-name date, which is what a WORDDATE parameter sends', () => {
+      // The reported shape.  Before the shape was recognised this became the
+      // quoted literal 'December 01, 2021', which a numeric date column cannot
+      // be compared with - the service aborted with a type error.
+      const clauses = service.buildInitialClauses(
+        msg([
+          { name: 'pr1', label: 'SOME_DATE', value: 'December 01, 2021' },
+          { name: 'pr2', label: 'SOME_DATE2', value: 'Dec 1, 2021' },
+          { name: 'pr3', label: 'SOME_DATE3', value: '1 December 2021' },
+          { name: 'pr4', label: 'SOME_DATE4', value: '1-Dec-2021' }
+        ])
+      )
+      expect(clauses.map((c) => [c.VARIABLE_NM, c.RAW_VALUE])).toEqual([
+        ['SOME_DATE', "'01DEC2021'd"],
+        ['SOME_DATE2', "'01DEC2021'd"],
+        ['SOME_DATE3', "'01DEC2021'd"],
+        ['SOME_DATE4', "'01DEC2021'd"]
+      ])
+    })
+
+    it('infers a month-name date whatever the VA dataType says', () => {
+      // The VA dataType is a fallback, not a fact: a date parameter can arrive
+      // as 'number' (where the value was previously dropped, leaving the first
+      // load silently unfiltered) or as 'string' (where it was quoted).
+      for (const dataType of ['number', 'string', undefined]) {
+        const clauses = service.buildInitialClauses(
+          msg([
+            {
+              name: 'pr1',
+              label: 'SOME_DATE',
+              dataType,
+              value: 'December 01, 2021'
+            }
+          ])
+        )
+        expect(clauses.map((c) => c.RAW_VALUE)).toEqual(["'01DEC2021'd"])
+      }
+    })
+
+    it('infers a month-name datetime', () => {
+      const clauses = service.buildInitialClauses(
+        msg([
+          { name: 'pr1', label: 'SOME_DTTM', value: 'Dec 1, 2021 10:30' },
+          { name: 'pr2', label: 'SOME_DTTM2', value: '01DEC2021:10:30' }
+        ])
+      )
+      expect(clauses.map((c) => c.RAW_VALUE)).toEqual([
+        "'01DEC2021:10:30:00'dt",
+        "'01DEC2021:10:30:00'dt"
+      ])
+    })
   })
 
   describe('signature', () => {

client/src/app/services/va-filter.service.ts

@@ -588,20 +588,25 @@ export class VaFilterService {
 
   /**
    * Infers a column's SAS kind from a VA value string (deferred load, no
-   * metadata). VA's temporal strings are distinctive — DATE9 `ddMMMyyyy` and ISO
-   * `yyyy-MM-dd`/`yyyy/MM/dd`, optionally with a `[ T:]HH:MM[:SS]` time part, and
-   * `H:MM:SS` — so they're detected by shape; otherwise fall back to the VA
-   * `dataType`.
+   * metadata).
+   *
+   * The value is offered to the same parsers the clause builder formats with, so
+   * any date shape that can be formatted is also recognised here. Datetime is
+   * tried first because it needs a time part, then date, then time; anything left
+   * falls back to the VA `dataType`, and finally to character.
+   *
+   * The result is a guess by construction: the deferred path has no column
+   * metadata, so a shape this function cannot name is formatted as character,
+   * and a character parameter whose value happens to look like a date ("1 May
+   * 2024") is formatted as a date. Either way the clause can disagree with the
+   * column, which costs the deferred `validatefilter` call: it is attempted with
+   * `suppressErrorAbortModal`, and when it fails the editor falls back to the
+   * unfiltered load and the post-load replay rebuilds the filter against the
+   * real metadata.
    */
   private inferKindFromValue(value: string, dataType?: string): VaColumnKind {
-    const monName = String.raw`\d{1,2}[A-Za-z]{3}\d{2,4}`
-    const ymd = String.raw`\d{4}[-/.]\d{1,2}[-/.]\d{1,2}` // 4-digit year first
-    const dmy = String.raw`\d{1,2}[-/.]\d{1,2}[-/.]\d{4}` // 4-digit year last
-    const date = `${monName}|${ymd}|${dmy}`
-    if (new RegExp(`^(?:${date})[ T:]\\d{1,2}:\\d{2}(:\\d{2})?$`).test(value)) {
-      return 'datetime'
-    }
-    if (new RegExp(`^(?:${date})$`).test(value)) return 'date'
+    if (this.parseDatetime(value)) return 'datetime'
+    if (this.parseDate(value)) return 'date'
     if (/^\d{1,2}:\d{2}(:\d{2})?(\.\d+)?$/.test(value)) return 'time'
     if (dataType === 'number') return 'numeric'
     return 'char'