From 180c2477ed9ae44b4e479e81ec642f40580bcbc8 Mon Sep 17 00:00:00 2001 From: 4gl <@> Date: Mon, 27 Jul 2026 17:51:00 +0100 Subject: [PATCH] fix(regex): special missing handling --- .agent/docs/regex-validations.md | 86 +++++++++++++++++++ AGENTS.md | 11 ++- client/cypress/e2e/editor.cy.ts | 10 ++- .../utils/regex-warning-renderer.spec.ts | 38 +++++++- .../editor/utils/regex-warning-renderer.ts | 5 +- .../app/shared/dc-validator/dc-validator.ts | 13 ++- .../dc-validator/tests/dc-validator.spec.ts | 20 ++++- .../utils/isRegexRuleExempt.spec.ts | 31 +++++-- .../dc-validator/utils/isRegexRuleExempt.ts | 25 ++++-- .../validations/dq-validation.spec.ts | 37 +++++++- .../dc-validator/validations/dq-validation.ts | 24 ++++-- .../app/shared/utils/col-info-html.spec.ts | 12 +-- client/src/app/shared/utils/col-info-html.ts | 16 ++-- 13 files changed, 277 insertions(+), 51 deletions(-) create mode 100644 .agent/docs/regex-validations.md diff --git a/.agent/docs/regex-validations.md b/.agent/docs/regex-validations.md new file mode 100644 index 0000000..4985375 --- /dev/null +++ b/.agent/docs/regex-validations.md @@ -0,0 +1,86 @@ +# Regex Validations (HARDREGEX / SOFTREGEX) + +This document describes how the regex validation rules work internally. For user-facing documentation see `docs/dcc-validations.md` in the `docs.datacontroller.io` repo. + +## Overview + +Two validation rule types in `MPE_VALIDATIONS` validate cell values against a regular expression supplied in `RULE_VALUE`: + +- `HARDREGEX` - submission-blocking. A failing value is rejected by the cell validator and painted red (HOT's own `htInvalid` class), so the row cannot be submitted. +- `SOFTREGEX` - display-only warning. A failing value is painted yellow (`dc-warning-cell` class) with a tooltip, but submission is not blocked. + +Both rule types are selectable in the MPE_VALIDATIONS RULE_TYPE dropdown; they were added to the selectbox seed data in `sas/sasjs/macros/mpe_makedata.sas` and via the optional migration `sas/sasjs/db/migrations/20260720_v7.12_release.sas`. `RULE_VALUE` is limited to 128 characters, which constrains very long patterns. + +## Config-time validation (SAS side) + +`sas/sasjs/services/hooks/mpe_validations_postedit.sas` runs `prxparse()` on any staged HARDREGEX/SOFTREGEX rule value and aborts the edit with a list of offending `libref.table.column` references if the pattern is invalid. This is a best-effort syntax check to catch typos at config time; an empty pattern is treated as valid (it matches everything in JS). Rows marked for delete are skipped. + +Because patterns must pass `prxparse`, rule values are authored in the SAS PRX delimiter form `/pattern/flags` (although a bare pattern is also accepted for backwards compatibility). + +## Frontend evaluation + +All regex handling lives in the client; there is no server-side re-validation of data values. The per-cell decision flow: + +```mermaid +flowchart TD + A[Cell value] --> B{Row marked for delete
and not a PK column?} + B -- Yes --> Z[No validation / no warning] + B -- No --> C{isRegexRuleExempt?
blank, or "." on a numeric column} + C -- Yes --> Z + C -- No --> D{HARDREGEX rule on column?} + D -- Yes --> E{Pattern matches?} + E -- No --> F[Invalid: submission blocked,
red htInvalid + REGEX tooltip] + E -- Yes --> G{SOFTREGEX rule on column?} + D -- No --> G + G -- Yes --> H{Pattern matches?} + H -- No --> I[Warning: yellow dc-warning-cell
+ REGEX tooltip, submission allowed] + H -- Yes --> J[Valid] + G -- No --> J +``` + +A malformed pattern never reaches this flow: it is treated as always-valid (HARDREGEX) / never-warn (SOFTREGEX) with a `console.warn`, rather than breaking the editor. + +### `client/src/app/shared/dc-validator/utils/parseRegexRule.ts` + +Converts an authored SAS PRX pattern into a JavaScript `RegExp`: + +1. If the value matches `/^\/(.*)\/([a-z]*)$/s`, the delimiters are stripped and the trailing flags are passed to `new RegExp(body, flags)`. Otherwise the value is used as-is (backwards compatibility with bare patterns). +2. Three mechanical Perl→JS translations are applied to the body: + - a leading `(?i)` inline modifier is removed and folded into the `i` flag; + - `\Q...\E` literal sequences are replaced with escaped literal text; + - `\A` → `^` and `\z` → `(?![\s\S])` (end-of-string anchor). +3. Perl-only constructs that would need a capture-group-renumbering rewrite (atomic groups `(?>...)`, possessive quantifiers `a++`) are deliberately NOT translated. They throw from `new RegExp`, and every caller treats a throw as "always valid / never warn" (see below) rather than breaking the editor. + +### `client/src/app/shared/dc-validator/utils/isRegexRuleExempt.ts` + +Blank values (`undefined`, `null`, `''`) are exempt from pattern matching on any column type - use a separate NOTNULL rule if populated values must also be enforced. On numeric columns the plain SAS missing (`.`) is also exempt; special missings (`.A`-`.Z`, `._`, bare letters) are NOT exempt anywhere - being deliberately set, they are real values the pattern must match (and on a character column even `.` is real text). `isSpecialMissing` from `@sasjs/utils` is deliberately not used: its optional-dot regex would exempt any single-letter character value ("d", "z") before the regex ever ran. The check takes an `isNumeric` flag, passed by all callers (the dq validator via `dqValidate(rules, value, colType === 'numeric')`, the warning renderer via a `makeRegexWarningRenderer` argument, and `failsSoftRegex` via the column's HOT type). + +### HARDREGEX - blocking validation + +`HARDREGEX` is implemented as a cell validator in `client/src/app/shared/dc-validator/validations/dq-validation.ts`. It returns `true` (valid) for exempt values and for patterns that fail to compile (with a `console.warn`), and otherwise returns `parseRegexRule(ruleValue).test(value.toString())`. Returning `false` makes HOT mark the cell invalid, block submission, and paint it red via its standard `htInvalid` styling. + +### SOFTREGEX - warning renderer + +`client/src/app/editor/utils/regex-warning-renderer.ts` builds a display-only Handsontable renderer (registered per-column by `DcValidator.setupRules` in `client/src/app/shared/dc-validator/dc-validator.ts`). It never returns false; it only: + +- adds a `REGEX: ` tooltip (`td.title`) when a rule fails; +- adds the yellow `dc-warning-cell` class when only SOFTREGEX fails. + +`DcValidator.failsSoftRegex(col, value)` provides the same logic outside the grid (e.g. the edit-record screen). + +### Precedence: HARD and SOFT on the same column + +A column may carry both rules. Both the renderer and `failsSoftRegex` evaluate them independently but with hard-first precedence: HARDREGEX is checked first; a value failing it gets the red invalid styling and a tooltip, and SOFTREGEX is never evaluated for that value (the yellow warning would be redundant). If HARDREGEX passes (or is absent), SOFTREGEX is evaluated independently. + +### Other behaviour + +- Rows marked for delete (`_____DELETE__THIS__RECORD_____ = 'Yes'`) are not warned/validated by the renderer (except primary key columns, which still validate). +- A malformed pattern never breaks the editor: the dq validator treats it as always-valid and the renderer disables the warning, logging to the console instead. +- The pattern is used as authored - it is NOT auto-anchored. Authors must include `^`/`$` to match the entire cell value. +- Column info: `client/src/app/shared/utils/col-info-html.ts` shows the applied pattern in the column-info dropdown - the HARDREGEX pattern if one exists (it is the rule actually applied when both are present), otherwise the SOFTREGEX pattern, otherwise nothing. + +## Testing + +- Unit tests: `parseRegexRule.spec.ts`, `isRegexRuleExempt.spec.ts`, `dq-validation.spec.ts`, `dc-validator.spec.ts` (under `client/src/app/shared/dc-validator/`), `client/src/app/editor/utils/regex-warning-renderer.spec.ts`, `client/src/app/shared/utils/col-info-html.spec.ts`. +- E2E: `client/cypress/e2e/editor.cy.ts`. +- SAS side: `sas/sasjs/services/editors/stagedata.test.3.sas`, plus seed data in `mpe_makedata.sas` (demo HARDREGEX rule "SOME_CHAR must contain 'the' or 'data'" (`/the|data/i`) and SOFTREGEX "should contain the letter 't'"). diff --git a/AGENTS.md b/AGENTS.md index c6fa123..0942de9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,9 +12,12 @@ This is different from `.sas` files, where a maximum line length applies. The no Rationale: hard-wrapped prose produces noisy diffs when sentences are edited and reflowed, and Markdown renderers already handle wrapping. -## SAS files +## Linting (required before "done") -After creating or modifying any `.sas` files, run `sasjs lint` from the `sas/` directory and ensure the files you touched have no lint warnings (the repo currently has pre-existing warnings in other files, which can be ignored). +Never consider a change complete until the relevant linters pass on the files you touched — do not rely on the user's pre-commit hooks to catch it: + +- **Client (TypeScript/HTML/etc.)**: run `npm run lint:check` from the `client/` directory (prettier). Fix any failures with `npm run lint:fix`. +- **SAS**: after creating or modifying any `.sas` files, run `sasjs lint` from the `sas/` directory and ensure the files you touched have no lint warnings (the repo currently has pre-existing warnings in other files, which can be ignored). ## No external assets @@ -23,3 +26,7 @@ Data Controller must run entirely locally (offline / on-prem, no internet access ## The .agent folder Agent-related content lives in `.agent/`: technical/agent-facing documentation goes in `.agent/docs/` (not `docs/`), and skills in `.agent/skills/`. When writing explanatory or technical docs about the codebase, put them in `.agent/docs/`. + +## Code comments and test names + +Never reference items that are not active parts of the repository — no "the original bug", "regression from this fix", "this session/PR/commit", or similar ephemeral context. Comments and test names must be self-contained: describe the behaviour being asserted, not the history of how it was discovered. The one exception is a literal link to a ticket/issue tracker. diff --git a/client/cypress/e2e/editor.cy.ts b/client/cypress/e2e/editor.cy.ts index d54655c..76d8362 100644 --- a/client/cypress/e2e/editor.cy.ts +++ b/client/cypress/e2e/editor.cy.ts @@ -285,21 +285,23 @@ context('editor tests: ', function () { openColumnDropdown('REGEX_HARD_COL') cy.get('.htDropdownMenu').should(($menu) => { - expect($menu.text()).to.include('REGEX: /[\\w.]+@[\\w]+\\.[a-z]{2,}/') + expect($menu.text()).to.include( + 'HARDREGEX: /[\\w.]+@[\\w]+\\.[a-z]{2,}/' + ) }) cy.get('body').click(0, 0) // close menu openColumnDropdown('REGEX_SOFT_COL') cy.get('.htDropdownMenu').should(($menu) => { expect($menu.text()).to.include( - 'REGEX: /[A-Z]{1,2}\\d{1,2}[A-Z]?\\s?\\d[A-Z]{2}/' + 'SOFTREGEX: /[A-Z]{1,2}\\d{1,2}[A-Z]?\\s?\\d[A-Z]{2}/' ) }) }) }) }) - it('10 | Info dropdown labels HARDREGEX/SOFTREGEX separately when a column has both', () => { + it('10 | Info dropdown shows only the applied HARDREGEX pattern when a column has both', () => { openTableFromTree(libraryToOpenIncludes, 'mpe_x_new') clickOnEdit(() => { @@ -313,7 +315,7 @@ context('editor tests: ', function () { const text = $menu.text() expect(text).to.include('HARDREGEX: /^[A-Z0-9_-]+$/') - expect(text).to.include('SOFTREGEX: /^.{5,10}$/') + expect(text).to.not.include('SOFTREGEX: /^.{5,10}$/') }) }) }) diff --git a/client/src/app/editor/utils/regex-warning-renderer.spec.ts b/client/src/app/editor/utils/regex-warning-renderer.spec.ts index a26e266..da1c1ca 100644 --- a/client/src/app/editor/utils/regex-warning-renderer.spec.ts +++ b/client/src/app/editor/utils/regex-warning-renderer.spec.ts @@ -19,6 +19,26 @@ describe('makeRegexWarningRenderer', () => { return { hot, container } } + const buildHotNumeric = (data: any[]) => { + const container = document.createElement('div') + document.body.appendChild(container) + + const hot = new Handsontable(container, { + data, + columns: [ + { + data: 'val', + renderer: makeRegexWarningRenderer('^[A-Z]{3}$', undefined, true) + }, + { data: '_____DELETE__THIS__RECORD_____' } + ], + licenseKey: 'non-commercial-and-evaluation' + }) + hot.render() + + return { hot, container } + } + it('adds dc-warning-cell and a REGEX: title when the value fails the pattern', () => { const { hot, container } = buildHot([ { val: 'abc', _____DELETE__THIS__RECORD_____: 'No' } @@ -56,9 +76,9 @@ describe('makeRegexWarningRenderer', () => { container.remove() }) - it('does not add dc-warning-cell for a SAS special missing value', () => { - const { hot, container } = buildHot([ - { val: '.a', _____DELETE__THIS__RECORD_____: 'No' } + it('does not add dc-warning-cell for the plain SAS missing (".") on a numeric column', () => { + const { hot, container } = buildHotNumeric([ + { val: '.', _____DELETE__THIS__RECORD_____: 'No' } ]) const td = hot.getCell(0, 0) @@ -68,6 +88,18 @@ describe('makeRegexWarningRenderer', () => { container.remove() }) + it('DOES add dc-warning-cell for a special missing on a numeric column (a deliberately-set value)', () => { + const { hot, container } = buildHotNumeric([ + { val: '.a', _____DELETE__THIS__RECORD_____: 'No' } + ]) + + const td = hot.getCell(0, 0) + expect(td?.classList.contains('dc-warning-cell')).toBeTrue() + + hot.destroy() + container.remove() + }) + it('does not throw and never warns on a malformed pattern', () => { const container = document.createElement('div') document.body.appendChild(container) diff --git a/client/src/app/editor/utils/regex-warning-renderer.ts b/client/src/app/editor/utils/regex-warning-renderer.ts index 9465dff..5874eb9 100644 --- a/client/src/app/editor/utils/regex-warning-renderer.ts +++ b/client/src/app/editor/utils/regex-warning-renderer.ts @@ -35,7 +35,8 @@ const compileRegex = ( */ export const makeRegexWarningRenderer = ( softPattern?: string, - hardPattern?: string + hardPattern?: string, + isNumeric: boolean = false ) => { const softRegex = compileRegex(softPattern, 'SOFTREGEX') const hardRegex = compileRegex(hardPattern, 'HARDREGEX') @@ -55,7 +56,7 @@ export const makeRegexWarningRenderer = ( const markedForDelete = instance.getDataAtRowProp(row, '_____DELETE__THIS__RECORD_____') === 'Yes' - const exempt = isRegexRuleExempt(value) + const exempt = isRegexRuleExempt(value, isNumeric) const failsHard = !!hardRegex && !exempt && !hardRegex.test(value.toString()) diff --git a/client/src/app/shared/dc-validator/dc-validator.ts b/client/src/app/shared/dc-validator/dc-validator.ts index 7e9f59f..e8dff4f 100644 --- a/client/src/app/shared/dc-validator/dc-validator.ts +++ b/client/src/app/shared/dc-validator/dc-validator.ts @@ -282,7 +282,9 @@ export class DcValidator { * independently — same precedence as makeRegexWarningRenderer. */ failsSoftRegex(col: string, value: any): boolean { - if (isRegexRuleExempt(value)) return false + const isNumeric = + this.rules.find((rule) => rule.data === col)?.type === 'numeric' + if (isRegexRuleExempt(value, isNumeric)) return false const hardRegexRule = this.dqrules.find( (rule: DQRule) => rule.BASE_COL === col && rule.RULE_TYPE === 'HARDREGEX' @@ -480,7 +482,8 @@ export class DcValidator { this.getRegexRuleValues(ruleColName) this.rules[i].renderer = makeRegexWarningRenderer( softRegexValue, - hardRegexValue + hardRegexValue, + this.rules[i].type === 'numeric' ) } } @@ -569,7 +572,11 @@ export class DcValidator { } if (self.isDqCol(col || '')) { - const dqValid = dqValidate(self.getDqDetails(col || ''), value) + const dqValid = dqValidate( + self.getDqDetails(col || ''), + value, + colType === 'numeric' + ) if (!dqValid) { console.warn(`DQ Validation - invalid (Value: ${value})`) diff --git a/client/src/app/shared/dc-validator/tests/dc-validator.spec.ts b/client/src/app/shared/dc-validator/tests/dc-validator.spec.ts index 334c091..17c6345 100644 --- a/client/src/app/shared/dc-validator/tests/dc-validator.spec.ts +++ b/client/src/app/shared/dc-validator/tests/dc-validator.spec.ts @@ -724,7 +724,7 @@ describe('DC Validator', () => { ).toBeFalse() }) - it('is false for blank/special-missing values, same exemption as HARDREGEX', () => { + it('is false for blank values; special-missing-looking values are real text on a character column', () => { const dcValidator = buildValidator([ { BASE_COL: 'SOME_CHAR_ANY', @@ -735,7 +735,23 @@ describe('DC Validator', () => { ]) expect(dcValidator.failsSoftRegex('SOME_CHAR_ANY', '')).toBeFalse() - expect(dcValidator.failsSoftRegex('SOME_CHAR_ANY', '.a')).toBeFalse() + // ".a" is real text here and fails ^[A-Z]+$ + expect(dcValidator.failsSoftRegex('SOME_CHAR_ANY', '.a')).toBeTrue() + }) + + it('exempts the plain missing (".") but not special missings on a numeric column', () => { + const dcValidator = buildValidator([ + { + BASE_COL: 'SOME_NUM', + RULE_TYPE: 'SOFTREGEX', + RULE_VALUE: '^[0-9]+$', + X: 0 + } + ]) + + expect(dcValidator.failsSoftRegex('SOME_NUM', '.')).toBeFalse() + // Special missings are deliberately-set values and face the pattern. + expect(dcValidator.failsSoftRegex('SOME_NUM', '.a')).toBeTrue() }) it('is false when a value fails both HARDREGEX and SOFTREGEX (precedence — no yellow on a red cell)', () => { diff --git a/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.spec.ts b/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.spec.ts index ccf33f3..20e16bb 100644 --- a/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.spec.ts +++ b/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.spec.ts @@ -1,19 +1,40 @@ import { isRegexRuleExempt } from './isRegexRuleExempt' describe('isRegexRuleExempt', () => { - it('exempts blank/undefined/null', () => { + it('exempts blank/undefined/null on any column type', () => { expect(isRegexRuleExempt('')).toBeTrue() expect(isRegexRuleExempt(undefined)).toBeTrue() expect(isRegexRuleExempt(null)).toBeTrue() + expect(isRegexRuleExempt('', true)).toBeTrue() }) - it('exempts SAS special missing values', () => { - expect(isRegexRuleExempt('.')).toBeTrue() - expect(isRegexRuleExempt('.a')).toBeTrue() - expect(isRegexRuleExempt('_')).toBeTrue() + it('exempts the plain SAS missing (".") on numeric columns', () => { + expect(isRegexRuleExempt('.', true)).toBeTrue() + }) + + it('does not exempt special missings on numeric columns (they are deliberately-set values)', () => { + expect(isRegexRuleExempt('.a', true)).toBeFalse() + expect(isRegexRuleExempt('._', true)).toBeFalse() + expect(isRegexRuleExempt('_', true)).toBeFalse() + expect(isRegexRuleExempt('d', true)).toBeFalse() + }) + + it('exempts nothing but blank on character columns', () => { + expect(isRegexRuleExempt('.')).toBeFalse() + expect(isRegexRuleExempt('.a')).toBeFalse() + expect(isRegexRuleExempt('_')).toBeFalse() + }) + + it('does not exempt a bare single letter on character columns (real value, not a missing)', () => { + // A bare single letter is real character data, not a SAS special + // missing - it must reach the pattern, not be exempted. + expect(isRegexRuleExempt('d')).toBeFalse() + expect(isRegexRuleExempt('z')).toBeFalse() + expect(isRegexRuleExempt('A')).toBeFalse() }) it('does not exempt an ordinary value', () => { expect(isRegexRuleExempt('ABC123')).toBeFalse() + expect(isRegexRuleExempt('ABC123', true)).toBeFalse() }) }) diff --git a/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.ts b/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.ts index 53d21a6..2c3b3aa 100644 --- a/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.ts +++ b/client/src/app/shared/dc-validator/utils/isRegexRuleExempt.ts @@ -1,13 +1,22 @@ -import { isSpecialMissing } from '@sasjs/utils/input/validators' - /** - * HARDREGEX/SOFTREGEX both skip pattern-matching for blank and SAS special - * missing values (., .a-.z, _) — the same exemption NOTNULL/MINVAL/MAXVAL - * already apply elsewhere, since neither convention represents a real - * formatted value the pattern is meant to check. + * HARDREGEX/SOFTREGEX both skip pattern-matching for: + * + * - blank values (undefined, null, '') on any column type - enforcing + * populated values is NOTNULL's job, not the pattern's; and + * - the plain SAS numeric missing (".") on numeric columns - it + * represents the absence of a value, same as blank. + * + * SPECIAL missings (.a-.z, ._, bare letters) are NOT exempt, even on + * numeric columns: being deliberately set, they are real values the + * pattern is meant to check. This also means isSpecialMissing from + * @sasjs/utils (which would match them, with an optional dot) is + * deliberately not used here. */ -export const isRegexRuleExempt = (value: any): boolean => { +export const isRegexRuleExempt = ( + value: any, + isNumeric: boolean = false +): boolean => { if (value === undefined || value === null || value === '') return true - return isSpecialMissing(value) + return isNumeric && value === '.' } diff --git a/client/src/app/shared/dc-validator/validations/dq-validation.spec.ts b/client/src/app/shared/dc-validator/validations/dq-validation.spec.ts index 7703dbb..1776822 100644 --- a/client/src/app/shared/dc-validator/validations/dq-validation.spec.ts +++ b/client/src/app/shared/dc-validator/validations/dq-validation.spec.ts @@ -30,12 +30,28 @@ describe('dqValidate - HARDREGEX', () => { expect(dqValidate(rules, null)).toBeTrue() }) - it('treats SAS special missing values as valid regardless of the pattern', () => { + it('treats the plain SAS missing (".") as valid on numeric columns, regardless of the pattern', () => { const rules = [rule({ RULE_VALUE: '^[A-Z]{3}\\d{4}$' })] - expect(dqValidate(rules, '.')).toBeTrue() - expect(dqValidate(rules, '.a')).toBeTrue() - expect(dqValidate(rules, '_')).toBeTrue() + expect(dqValidate(rules, '.', true)).toBeTrue() + }) + + it('applies the pattern to special missings on numeric columns (deliberately-set values)', () => { + const rules = [rule({ RULE_VALUE: '^[A-Z]{3}\\d{4}$' })] + + expect(dqValidate(rules, '.a', true)).toBeFalse() + expect(dqValidate(rules, '_', true)).toBeFalse() + expect(dqValidate(rules, 'd', true)).toBeFalse() + }) + + it('applies the pattern to special-missing-looking values on character columns', () => { + const rules = [rule({ RULE_VALUE: '^[A-Z]{3}\\d{4}$' })] + + expect(dqValidate(rules, '.')).toBeFalse() + expect(dqValidate(rules, '.a')).toBeFalse() + expect(dqValidate(rules, '_')).toBeFalse() + // A bare single letter is a real character value and must be matched. + expect(dqValidate(rules, 'd')).toBeFalse() }) it('fails open (treats as valid) when the pattern is malformed', () => { @@ -44,6 +60,19 @@ describe('dqValidate - HARDREGEX', () => { expect(dqValidate(rules, 'anything')).toBeTrue() }) + it('validates a single letter on a character column against the pattern', () => { + // A bare single letter ("d", "z") is real character data, not a SAS + // special missing - it must reach the pattern, not be exempted. + const rules = [rule({ RULE_VALUE: '/the|data/i' })] + + expect(dqValidate(rules, 'd')).toBeFalse() + expect(dqValidate(rules, 'z')).toBeFalse() + expect(dqValidate(rules, 'the')).toBeTrue() + expect(dqValidate(rules, 'DATA')).toBeTrue() + expect(dqValidate(rules, 'some data here')).toBeTrue() + expect(dqValidate(rules, '')).toBeTrue() // blank stays exempt + }) + it('handles a more elaborate pattern (multiple character classes, quantifiers, an escaped literal dot)', () => { // Same email pattern used in the getdata.js mock's REGEX_HARD_COL demo. const rules = [rule({ RULE_VALUE: '[\\w.]+@[\\w]+\\.[a-z]{2,}' })] diff --git a/client/src/app/shared/dc-validator/validations/dq-validation.ts b/client/src/app/shared/dc-validator/validations/dq-validation.ts index b470460..80b9ff2 100644 --- a/client/src/app/shared/dc-validator/validations/dq-validation.ts +++ b/client/src/app/shared/dc-validator/validations/dq-validation.ts @@ -4,7 +4,11 @@ import { isRegexRuleExempt } from '../utils/isRegexRuleExempt' import { parseRegexRule } from '../utils/parseRegexRule' const dqValidation: { - [key: string]: (value: any, ruleValue: string | number) => boolean + [key: string]: ( + value: any, + ruleValue: string | number, + isNumeric?: boolean + ) => boolean } = { CASE: (value: any, ruleValue: string | number): boolean => { switch (ruleValue) { @@ -51,8 +55,12 @@ const dqValidation: { }, // Pattern is used as authored, not auto-anchored — a rule author who // wants a full-value match must write ^...$ themselves. - HARDREGEX: (value: any, ruleValue: string | number): boolean => { - if (isRegexRuleExempt(value)) return true + HARDREGEX: ( + value: any, + ruleValue: string | number, + isNumeric: boolean = false + ): boolean => { + if (isRegexRuleExempt(value, isNumeric)) return true try { return parseRegexRule(ruleValue.toString()).test(value.toString()) @@ -65,10 +73,16 @@ const dqValidation: { } } -export const dqValidate = (dqRules: DQRule[], value: any): boolean => { +export const dqValidate = ( + dqRules: DQRule[], + value: any, + isNumeric: boolean = false +): boolean => { for (let detail of dqRules) { if (dqValidation[detail.RULE_TYPE]) { - if (!dqValidation[detail.RULE_TYPE](value, detail.RULE_VALUE)) { + if ( + !dqValidation[detail.RULE_TYPE](value, detail.RULE_VALUE, isNumeric) + ) { console.warn( `DQ Invalid Reason: ${ detail.RULE_TYPE diff --git a/client/src/app/shared/utils/col-info-html.spec.ts b/client/src/app/shared/utils/col-info-html.spec.ts index 7ea48bc..44eef63 100644 --- a/client/src/app/shared/utils/col-info-html.spec.ts +++ b/client/src/app/shared/utils/col-info-html.spec.ts @@ -19,7 +19,7 @@ describe('buildColInfoHtml', () => { ) }) - it('appends a REGEX line when only a HARDREGEX value is provided', () => { + it('appends a HARDREGEX line when only a HARDREGEX value is provided', () => { const colInfo: DataFormat = { label: 'Some Character Column', type: 'char', @@ -30,11 +30,11 @@ describe('buildColInfoHtml', () => { expect( buildColInfoHtml('SOME_CHAR', colInfo, '/^[A-Z]+$/i', undefined) ).toBe( - 'NAME: SOME_CHAR
LABEL: Some Character Column
TYPE: char
LENGTH: 1024
FORMAT: $1024.
REGEX: /^[A-Z]+$/i' + 'NAME: SOME_CHAR
LABEL: Some Character Column
TYPE: char
LENGTH: 1024
FORMAT: $1024.
HARDREGEX: /^[A-Z]+$/i' ) }) - it('appends a REGEX line when only a SOFTREGEX value is provided', () => { + it('appends a SOFTREGEX line when only a SOFTREGEX value is provided', () => { const colInfo: DataFormat = { label: 'Some Character Column', type: 'char', @@ -45,11 +45,11 @@ describe('buildColInfoHtml', () => { expect( buildColInfoHtml('SOME_CHAR', colInfo, undefined, '/^[a-z]+$/') ).toBe( - 'NAME: SOME_CHAR
LABEL: Some Character Column
TYPE: char
LENGTH: 1024
FORMAT: $1024.
REGEX: /^[a-z]+$/' + 'NAME: SOME_CHAR
LABEL: Some Character Column
TYPE: char
LENGTH: 1024
FORMAT: $1024.
SOFTREGEX: /^[a-z]+$/' ) }) - it('appends separate HARDREGEX/SOFTREGEX lines when a column has both', () => { + it('appends only the HARDREGEX line when a column has both rules, since HARDREGEX is the one applied', () => { const colInfo: DataFormat = { label: 'Some Character Column', type: 'char', @@ -60,7 +60,7 @@ describe('buildColInfoHtml', () => { expect( buildColInfoHtml('SOME_CHAR', colInfo, '/^[A-Z]+$/i', '/^.{5,10}$/') ).toBe( - 'NAME: SOME_CHAR
LABEL: Some Character Column
TYPE: char
LENGTH: 1024
FORMAT: $1024.
HARDREGEX: /^[A-Z]+$/i
SOFTREGEX: /^.{5,10}$/' + 'NAME: SOME_CHAR
LABEL: Some Character Column
TYPE: char
LENGTH: 1024
FORMAT: $1024.
HARDREGEX: /^[A-Z]+$/i' ) }) diff --git a/client/src/app/shared/utils/col-info-html.ts b/client/src/app/shared/utils/col-info-html.ts index af647c8..ba74eda 100644 --- a/client/src/app/shared/utils/col-info-html.ts +++ b/client/src/app/shared/utils/col-info-html.ts @@ -15,13 +15,15 @@ export function buildColInfoHtml( let html = `NAME: ${colName}
LABEL: ${colInfo.label}
TYPE: ${colInfo.type}
LENGTH: ${colInfo.length}
FORMAT: ${colInfo.format}` - // HARDREGEX/SOFTREGEX are only distinguished by name when a column has - // both - with just one rule, which one it is is already implied, so the - // generic REGEX label keeps the common case uncluttered. - if (hardRegexValue && softRegexValue) { - html += `
HARDREGEX: ${hardRegexValue}
SOFTREGEX: ${softRegexValue}` - } else if (hardRegexValue || softRegexValue) { - html += `
REGEX: ${hardRegexValue || softRegexValue}` + // Only ever one REGEX rule is applied per column: when both HARDREGEX + // and SOFTREGEX exist, HARDREGEX is evaluated first and blocks + // submission, so the soft rule only applies once hard passes (same + // precedence as makeRegexWarningRenderer / DcValidator.failsSoftRegex). + // Show only the rule that is applied. + if (hardRegexValue) { + html += `
HARDREGEX: ${hardRegexValue}` + } else if (softRegexValue) { + html += `
SOFTREGEX: ${softRegexValue}` } return html