dc/client/src/app/editor/utils/characterColumnFormula.integration.spec.ts

import Handsontable from 'handsontable'
import { HyperFormula } from 'hyperformula'
import { Col } from '../../shared/dc-validator/models/col.model'
import { isCharacterColumn } from '../../shared/dc-validator/utils/isCharacterColumn'
import { escapeCharacterColumnValue } from '../../shared/dc-validator/utils/escapeCharacterColumnValue'
import { unescapeFormula } from '../../shared/dc-validator/utils/unescapeFormula'
import { syncOverwrittenCellComment } from '../../shared/dc-validator/utils/syncOverwrittenCellComment'
import { resolveRevertedCellValue } from '../../shared/dc-validator/utils/resolveRevertedCellValue'
import {
  AutoEscapedCellMap,
  getRowKey,
  markAutoEscaped,
  isAutoEscaped,
  clearAutoEscaped
} from '../../shared/dc-validator/utils/autoEscapedCellTracker'
import { expandCellRanges, SimpleCell } from './expandCellRanges'

/**
 * Integration coverage for character-column-formula-plan.md, following the
 * same real-Handsontable-instance technique already used elsewhere in this
 * codebase (renderers.utils.spec.ts) - editor.component.ts itself has no
 * TestBed harness (too heavily coupled to be practical), so this mirrors
 * its actual hook implementations closely enough to prove the mechanics,
 * with Cypress covering the true end-to-end behaviour.
 */
describe('character-column-formula-plan integration', () => {
  const cols: Col[] = [
    { NAME: 'PRIMARY_KEY_FIELD', DDTYPE: 'N' } as Col,
    { NAME: 'A_COL', DDTYPE: 'N' } as Col,
    { NAME: 'B_COL', DDTYPE: 'N' } as Col,
    { NAME: 'FORMULA_HARD_COL', DDTYPE: 'C' } as Col,
    { NAME: 'PLAIN_TEXT_COL', DDTYPE: 'C' } as Col
  ]
  const headerPks = ['PRIMARY_KEY_FIELD']

  let container: HTMLDivElement

  beforeEach(() => {
    container = document.createElement('div')
    document.body.appendChild(container)
  })

  afterEach(() => {
    container.remove()
  })

  it('requirement 4: formulas evaluate live on a table with zero HARDFORMULA/SOFTFORMULA rules', () => {
    const data = [
      {
        PRIMARY_KEY_FIELD: 1,
        A_COL: 1,
        B_COL: 10,
        FORMULA_HARD_COL: '',
        PLAIN_TEXT_COL: '=1+1'
      }
    ]

    const hot = new Handsontable(container, {
      data,
      columns: cols.map((c) => ({ data: c.NAME })),
      formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()

    expect(hot.getDataAtRowProp(0, 'PLAIN_TEXT_COL')).toEqual(2)

    hot.destroy()
  })

  it('requirement 1: a backend =-led value in a character column is escaped and marked; a formula base column is skipped', () => {
    const formulaBaseCols = ['FORMULA_HARD_COL']
    const dataSource = [
      {
        PRIMARY_KEY_FIELD: 1,
        A_COL: 1,
        B_COL: 10,
        FORMULA_HARD_COL: '=B1*C1',
        PLAIN_TEXT_COL: '=100 + B_COL'
      }
    ]
    const headerColumns = cols.map((c) => c.NAME)
    const autoEscapedCells: AutoEscapedCellMap = new Map()

    for (const row of dataSource) {
      const rowKey = getRowKey(row, headerPks)
      for (const colName of headerColumns) {
        if (formulaBaseCols.includes(colName)) continue
        if (!isCharacterColumn(cols, colName)) continue

        const { value, wasEscaped } = escapeCharacterColumnValue(
          (row as any)[colName]
        )
        ;(row as any)[colName] = value
        if (wasEscaped) markAutoEscaped(autoEscapedCells, rowKey, colName)
      }
    }

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual("'=100 + B_COL")
    expect(dataSource[0].FORMULA_HARD_COL).toEqual('=B1*C1')

    const rowKey = getRowKey(dataSource[0], headerPks)
    expect(isAutoEscaped(autoEscapedCells, rowKey, 'PLAIN_TEXT_COL')).toEqual(
      true
    )
    expect(isAutoEscaped(autoEscapedCells, rowKey, 'FORMULA_HARD_COL')).toEqual(
      false
    )
  })

  it('requirement 3: a user-typed =-led value in a character column is never auto-escaped, and evaluates live', () => {
    const dataSource = [
      { PRIMARY_KEY_FIELD: 1, A_COL: 1, B_COL: 10, PLAIN_TEXT_COL: 'note-1' }
    ]
    const autoEscapedCells: AutoEscapedCellMap = new Map()

    const hot = new Handsontable(container, {
      data: dataSource,
      columns: [
        { data: 'PRIMARY_KEY_FIELD' },
        { data: 'A_COL' },
        { data: 'B_COL' },
        { data: 'PLAIN_TEXT_COL' }
      ],
      formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
      licenseKey: 'non-commercial-and-evaluation'
    })

    // Mirrors the new beforeChange hook - no escaping call at all, just
    // clearing a stale marker if this cell had one.
    hot.addHook('beforeChange', (changes: any[]) => {
      if (!changes) return
      for (const change of changes) {
        if (!change) continue
        const [changeRow, prop] = change
        const colName =
          typeof prop === 'string' ? prop : (hot.colToProp(prop) as string)
        const row = dataSource[hot.toPhysicalRow(changeRow)]
        if (row) {
          clearAutoEscaped(autoEscapedCells, getRowKey(row, headerPks), colName)
        }
      }
    })

    hot.setDataAtRowProp(0, 'PLAIN_TEXT_COL', '=1+1')
    hot.render()

    expect(hot.getDataAtRowProp(0, 'PLAIN_TEXT_COL')).toEqual(2)
    expect(
      isAutoEscaped(
        autoEscapedCells,
        getRowKey(dataSource[0], headerPks),
        'PLAIN_TEXT_COL'
      )
    ).toEqual(false)

    hot.destroy()
  })

  describe('requirements 2/5/6: submit-time marker-gated strip', () => {
    it('strips the marker from an untouched auto-escaped cell (round-trips = -> = )', () => {
      const dataSource = [
        {
          PRIMARY_KEY_FIELD: 1,
          PLAIN_TEXT_COL: "'=100 + 200"
        }
      ]
      const autoEscapedCells: AutoEscapedCellMap = new Map()
      markAutoEscaped(
        autoEscapedCells,
        getRowKey(dataSource[0], headerPks),
        'PLAIN_TEXT_COL'
      )

      submitStrip(dataSource, ['PLAIN_TEXT_COL'], autoEscapedCells, headerPks)

      expect(dataSource[0].PLAIN_TEXT_COL).toEqual('=100 + 200')
    })

    it("leaves a genuinely-literal backend value untouched ( ' =something stays ' =something )", () => {
      const dataSource = [
        {
          PRIMARY_KEY_FIELD: 1,
          PLAIN_TEXT_COL: "'=already literal"
        }
      ]
      // Never marked - escapeCharacterColumnValue never escapes a value
      // that doesn't start with '=' in the first place.
      const autoEscapedCells: AutoEscapedCellMap = new Map()

      submitStrip(dataSource, ['PLAIN_TEXT_COL'], autoEscapedCells, headerPks)

      expect(dataSource[0].PLAIN_TEXT_COL).toEqual("'=already literal")
    })

    it("leaves a user-edited cell untouched even if the marker was cleared and the new value starts with ' =", () => {
      const dataSource = [
        {
          PRIMARY_KEY_FIELD: 1,
          PLAIN_TEXT_COL: "'=user typed this"
        }
      ]
      const autoEscapedCells: AutoEscapedCellMap = new Map()
      // Simulates: cell was auto-escaped on load, then the user edited it -
      // beforeChange clears the marker (see the user-edit test above).
      markAutoEscaped(
        autoEscapedCells,
        getRowKey(dataSource[0], headerPks),
        'PLAIN_TEXT_COL'
      )
      clearAutoEscaped(
        autoEscapedCells,
        getRowKey(dataSource[0], headerPks),
        'PLAIN_TEXT_COL'
      )

      submitStrip(dataSource, ['PLAIN_TEXT_COL'], autoEscapedCells, headerPks)

      expect(dataSource[0].PLAIN_TEXT_COL).toEqual("'=user typed this")
    })
  })

  describe('requirement 7: "Apply as formula"', () => {
    // A minimal fake of the handful of Handsontable Core methods hidden()/
    // callback() actually use, backed directly by the plain dataSource
    // array/column list - no real Handsontable instance, so no DOM/render
    // lifecycle to race against. What's under test here is the selection
    // -> auto-escaped-cell matching logic itself; live formula evaluation
    // via a real HyperFormula engine is already covered by the
    // "requirement 4"/"requirement 3" tests above.
    const makeFakeHot = (
      dataSource: any[],
      colNames: string[],
      selectedRanges: { from: SimpleCell; to: SimpleCell }[]
    ) => ({
      getSelectedRange: () => selectedRanges,
      countRows: () => dataSource.length,
      countCols: () => colNames.length,
      toPhysicalRow: (row: number) => row,
      colToProp: (col: number) => colNames[col],
      getDataAtRowProp: (row: number, prop: string) => dataSource[row][prop],
      setDataAtRowProp: (row: number, prop: string, value: unknown) => {
        dataSource[row][prop] = value
      }
    })

    const buildHiddenAndCallback = (
      dataSource: any[],
      autoEscapedCells: AutoEscapedCellMap,
      hot: ReturnType<typeof makeFakeHot>
    ) => {
      const hidden = (): boolean => {
        const ranges = hot.getSelectedRange()
        if (!ranges || ranges.length === 0) return true

        const cells = expandCellRanges(
          ranges as any,
          hot.countRows(),
          hot.countCols()
        )
        return !cells.some(({ row, col }) => {
          const dataRow = dataSource[hot.toPhysicalRow(row)]
          if (!dataRow) return false
          const colName = hot.colToProp(col) as string
          return isAutoEscaped(
            autoEscapedCells,
            getRowKey(dataRow, headerPks),
            colName
          )
        })
      }

      const callback = (): void => {
        const ranges = hot.getSelectedRange()
        if (!ranges) return
        const cells = expandCellRanges(
          ranges as any,
          hot.countRows(),
          hot.countCols()
        )
        for (const { row, col } of cells) {
          const dataRow = dataSource[hot.toPhysicalRow(row)]
          if (!dataRow) continue
          const colName = hot.colToProp(col) as string
          const rowKey = getRowKey(dataRow, headerPks)
          if (!isAutoEscaped(autoEscapedCells, rowKey, colName)) continue

          hot.setDataAtRowProp(
            row,
            colName,
            unescapeFormula(hot.getDataAtRowProp(row, colName))
          )
          clearAutoEscaped(autoEscapedCells, rowKey, colName)
        }
      }

      return { hidden, callback }
    }

    it('is hidden when the selection contains no auto-escaped cell', () => {
      const dataSource = [
        { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: "'=already literal" }
      ]
      const autoEscapedCells: AutoEscapedCellMap = new Map()
      const colNames = ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL']

      const hot = makeFakeHot(dataSource, colNames, [
        { from: { row: 0, col: 1 }, to: { row: 0, col: 1 } }
      ])

      const { hidden } = buildHiddenAndCallback(
        dataSource,
        autoEscapedCells,
        hot
      )
      expect(hidden()).toEqual(true)
    })

    it('is shown, and only strips the auto-inserted marker, when the selection contains an auto-escaped cell alongside a genuine literal one', () => {
      const dataSource = [
        { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: "'=100 + 200" },
        { PRIMARY_KEY_FIELD: 2, PLAIN_TEXT_COL: "'=already literal" }
      ]
      const autoEscapedCells: AutoEscapedCellMap = new Map()
      markAutoEscaped(
        autoEscapedCells,
        getRowKey(dataSource[0], headerPks),
        'PLAIN_TEXT_COL'
      )
      const colNames = ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL']

      const hot = makeFakeHot(dataSource, colNames, [
        { from: { row: 0, col: 1 }, to: { row: 1, col: 1 } } // both rows, PLAIN_TEXT_COL column
      ])

      const { hidden, callback } = buildHiddenAndCallback(
        dataSource,
        autoEscapedCells,
        hot
      )
      expect(hidden()).toEqual(false)

      callback()

      // Marker stripped - now the plain formula string a live engine would
      // evaluate to 300 (see "requirement 4"/"requirement 3" above for
      // proof that a real engine does exactly that).
      expect(dataSource[0].PLAIN_TEXT_COL).toEqual('=100 + 200')
      expect(dataSource[1].PLAIN_TEXT_COL).toEqual("'=already literal")
      expect(
        isAutoEscaped(
          autoEscapedCells,
          getRowKey(dataSource[0], headerPks),
          'PLAIN_TEXT_COL'
        )
      ).toEqual(false)
    })
  })
})

/**
 * Handsontable's own columnSorting integration caches a reference to the
 * formulas plugin's engine - disabling and re-enabling that plugin
 * invalidates the cached reference, so sorting afterward crashes with
 * "Cannot read properties of undefined (reading 'getSheetWidth')".
 * initSetup must never call disablePlugin()/enablePlugin() on the formulas
 * plugin around a table load - only the maxRows fix below is safe.
 */
describe('formulas plugin must not be disabled/re-enabled around a table load', () => {
  it('column sorting keeps working after loading data with formulas already enabled', () => {
    const container = document.createElement('div')
    document.body.appendChild(container)
    const formulas = { engine: HyperFormula, licenseKey: 'gpl-v3' }
    const colNames = ['PK', 'A', 'B']
    const data = Array.from({ length: 10 }, (_, i) => ({
      PK: 10 - i,
      A: i,
      B: i
    }))

    const hot = new Handsontable(container, {
      data,
      columns: colNames.map((c) => ({ data: c })),
      formulas,
      columnSorting: true,
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()

    // Mirrors a second table load on the same instance - data replaced,
    // formulas passed again, but the plugin is never disabled first.
    hot.updateSettings({ data, formulas })
    hot.render()

    let error: any = null
    try {
      const sortPlugin: any = hot.getPlugin('columnSorting')
      sortPlugin.sort({ column: 0, sortOrder: 'asc' })
      hot.render()
    } catch (e) {
      error = e
    }

    expect(error).toBeNull()

    hot.destroy()
  })
})

/**
 * Handsontable's own `maxRows` grid setting (a UI/licensing cap on how many
 * NEW rows can be ADDED - editor_rows_allowed) silently becomes
 * HyperFormula's OWN sheet-size limit too, via the formulas plugin's
 * getEngineSettingsOverrides (maps hotSettings.maxRows straight into the
 * engine config). editor_rows_allowed can be smaller than a table's
 * ALREADY-EXISTING row count, and formulas are on for every table now -
 * capping maxRows at editor_rows_allowed alone makes HyperFormula reject
 * any table bigger than that cap outright. See initSetup's own
 * Math.max(dataSource.length, editor_rows_allowed) fix.
 *
 * Handsontable 18.1 fixed the engine coupling upstream (#10672): the grid's
 * maxRows/maxCols no longer limit the HyperFormula engine, so a capped
 * maxRows no longer rejects a larger existing dataset. DC's own guard is
 * still in place; the first test below now pins the fixed behaviour.
 */
describe('maxRows must never be smaller than the actual loaded row count', () => {
  it('no longer throws when maxRows is capped below the real row count (HOT 18.1 #10672)', () => {
    const container = document.createElement('div')
    document.body.appendChild(container)
    const colNames = ['PK', 'A', 'B']
    const data = Array.from({ length: 100 }, (_, i) => ({ PK: i, A: i, B: i }))

    let error: any = null
    try {
      const hot = new Handsontable(container, {
        data,
        columns: colNames.map((c) => ({ data: c })),
        formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
        maxRows: 15, // editor_rows_allowed-style small licensing cap
        licenseKey: 'non-commercial-and-evaluation'
      })
      hot.render()
      hot.destroy()
    } catch (e) {
      error = e
    }

    expect(error).toBeNull()
  })

  it('does not throw when maxRows is never let below the actual row count (the fix)', () => {
    const container = document.createElement('div')
    document.body.appendChild(container)
    const colNames = ['PK', 'A', 'B']
    const data = Array.from({ length: 100 }, (_, i) => ({ PK: i, A: i, B: i }))
    const editorRowsAllowed = 15

    let error: any = null
    try {
      const hot = new Handsontable(container, {
        data,
        columns: colNames.map((c) => ({ data: c })),
        formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
        maxRows: Math.max(data.length, editorRowsAllowed),
        licenseKey: 'non-commercial-and-evaluation'
      })
      hot.render()
      hot.destroy()
    } catch (e) {
      error = e
    }

    expect(error).toBeNull()
  })
})

/**
 * dataSourceRaw drives the Revert feature (see editor.component.ts's own
 * syncOverwrittenCellComment/markOverwrittenCells). It's captured before
 * the escape loop runs, so without also escaping the matching
 * dataSourceRaw cell there, it would keep the true unescaped backend
 * string - meaning Revert (after "Apply as formula" turns a cell into a
 * live formula) would restore raw `=...` text, which evaluates live again
 * instead of going back to the escaped text the user actually started
 * from.
 */
describe('Revert after "Apply as formula" restores the escaped text, not the raw backend value', () => {
  // Runs the "Apply as formula" -> afterChange comment sync -> Revert
  // sequence, parameterized on whether dataSourceRaw's cell got escaped
  // alongside dataSource's (the fix) or was left as the true unescaped
  // backend string (the bug).
  const runRevertAfterApplyAsFormula = (escapeDataSourceRaw: boolean) => {
    const dataSource = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: '=100 + 200' }]
    const dataSourceRaw = JSON.parse(JSON.stringify(dataSource))

    const { value: escapedValue } = escapeCharacterColumnValue(
      dataSource[0].PLAIN_TEXT_COL
    )
    dataSource[0].PLAIN_TEXT_COL = escapedValue as string
    if (escapeDataSourceRaw) {
      dataSourceRaw[0].PLAIN_TEXT_COL = escapedValue as string
    }

    // "Apply as formula": strips the marker - the cell now holds a real
    // formula, which a live engine would evaluate to 300.
    dataSource[0].PLAIN_TEXT_COL = unescapeFormula(
      dataSource[0].PLAIN_TEXT_COL
    ) as string
    const liveEvaluatedValue = 300

    // afterChange's own live overwritten-comment sync.
    syncOverwrittenCellComment(
      liveEvaluatedValue,
      dataSourceRaw[0].PLAIN_TEXT_COL,
      false
    )
    const comment = `Original value: ${dataSourceRaw[0].PLAIN_TEXT_COL}`

    // Revert: reads the comment, restores it verbatim (PLAIN_TEXT_COL
    // isn't numeric).
    const rawValueText = comment.slice('Original value: '.length)
    return resolveRevertedCellValue(rawValueText, false)
  }

  it('reverts to the raw, live-evaluating backend string when dataSourceRaw was never escaped (the bug)', () => {
    expect(runRevertAfterApplyAsFormula(false)).toEqual('=100 + 200')
  })

  it('reverts to the escaped form when dataSourceRaw is escaped alongside dataSource (the fix)', () => {
    expect(runRevertAfterApplyAsFormula(true)).toEqual("'=100 + 200")
  })
})

/**
 * Revert restoring the escaped form (the fix above) is only half the
 * story - beforeChange also needs to re-mark that cell as auto-escaped,
 * or "Apply as formula" stays hidden for it afterward even though it's
 * back to holding exactly the kind of value that item is for. Mirrors
 * the real beforeChange hook: revert_cells tags its own setDataAtRowProp
 * call with source 'revert', which is the only thing that triggers
 * re-marking - a value that merely happens to start with `'=` because
 * the user typed it themselves must never be re-marked (it wasn't the
 * app that put that quote there).
 */
describe('beforeChange re-marks a cell revert_cells restores to its escaped form', () => {
  const runBeforeChange = (newValue: string, source: string | undefined) => {
    const autoEscapedCells: AutoEscapedCellMap = new Map()
    const row = { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'placeholder' }
    const rowKey = getRowKey(row, ['PRIMARY_KEY_FIELD'])

    // Mirrors beforeChange's own branch.
    if (
      source === 'revert' &&
      typeof newValue === 'string' &&
      newValue.startsWith("'=")
    ) {
      markAutoEscaped(autoEscapedCells, rowKey, 'PLAIN_TEXT_COL')
    } else {
      clearAutoEscaped(autoEscapedCells, rowKey, 'PLAIN_TEXT_COL')
    }

    return isAutoEscaped(autoEscapedCells, rowKey, 'PLAIN_TEXT_COL')
  }

  it('re-marks a reverted escaped value so "Apply as formula" is offered again', () => {
    expect(runBeforeChange("'=100 + 200", 'revert')).toEqual(true)
  })

  it('does not re-mark a revert that restores a plain (non-escaped) value', () => {
    expect(runBeforeChange('note-1', 'revert')).toEqual(false)
  })

  it('does not mark a user-typed value that happens to start with the marker shape', () => {
    expect(runBeforeChange("'=user typed this", undefined)).toEqual(false)
  })
})

/**
 * saveTable() must resolve a genuine live formula's computed value before
 * submission, the same way it already does for HARDFORMULA/SOFTFORMULA
 * columns - otherwise the raw formula text itself (e.g. "=1+1") reaches
 * the backend instead of what the grid actually displays (2). Mirrors
 * saveTable's own combined strip/resolve loop against a real
 * Handsontable + HyperFormula instance, since proving a formula's
 * computed value requires a real engine.
 */
describe('submission resolves a genuine live formula to its computed value, not the raw text', () => {
  const headerPks = ['PRIMARY_KEY_FIELD']

  const runSubmitPrep = (
    dataSource: any[],
    colNames: string[],
    autoEscapedCells: AutoEscapedCellMap
  ): void => {
    const hot = new Handsontable(document.createElement('div'), {
      data: dataSource,
      columns: colNames.map((c) => ({ data: c })),
      formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()

    // Mirrors saveTable's own combined loop.
    dataSource.forEach((row, physicalRowIndex) => {
      const visualRowIndex = hot.toVisualRow(physicalRowIndex)
      const rowKey = getRowKey(row, headerPks)
      for (const colName of colNames) {
        if (isAutoEscaped(autoEscapedCells, rowKey, colName)) {
          row[colName] = unescapeFormula(row[colName])
        } else if (
          typeof row[colName] === 'string' &&
          row[colName].startsWith('=')
        ) {
          row[colName] = hot.getDataAtRowProp(visualRowIndex, colName)
        }
      }
    })

    hot.destroy()
  }

  it('submits the computed value for a user-typed formula, not the raw formula text', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: '=1+1' }]

    runSubmitPrep(
      dataSource,
      ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'],
      new Map()
    )

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual(2)
  })

  it('submits the computed value for a cell "Apply as formula" was used on', () => {
    // "Apply as formula" leaves the cell exactly like a user-typed formula
    // - no marker, raw dataSource text starting with `=`.
    const dataSource: any[] = [
      { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: '=100 + 200' }
    ]

    runSubmitPrep(
      dataSource,
      ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'],
      new Map()
    )

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual(300)
  })

  it('still strips the marker (not a computed value) for an untouched auto-escaped cell', () => {
    const dataSource = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: "'=100 + 200" }]
    const autoEscapedCells: AutoEscapedCellMap = new Map()
    markAutoEscaped(
      autoEscapedCells,
      getRowKey(dataSource[0], headerPks),
      'PLAIN_TEXT_COL'
    )

    runSubmitPrep(
      dataSource,
      ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'],
      autoEscapedCells
    )

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual('=100 + 200')
  })

  it('leaves a genuine literal value untouched', () => {
    const dataSource = [
      { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: "'=already literal" }
    ]

    runSubmitPrep(
      dataSource,
      ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'],
      new Map()
    )

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual("'=already literal")
  })

  // coerceNumericRow (beforePaste/beforeAutofill) deliberately leaves a
  // non-numeric, formula-looking pasted/autofilled value alone rather
  // than forcing it to NaN - so a *numeric* column (DDTYPE N, not just a
  // character one) can end up holding a live formula too. Submission must
  // resolve its computed value the same way, or the raw `=2+2` text
  // reaches the backend - which a numeric SAS field can't parse (it gets
  // misread as an invalid "special missing value" and the submission
  // fails outright).
  it('submits the computed value for a live formula in a numeric (non-character) column too', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, B_COL: '=2+2' }]

    runSubmitPrep(dataSource, ['PRIMARY_KEY_FIELD', 'B_COL'], new Map())

    expect(dataSource[0].B_COL).toEqual(4)
  })

  // HARDFORMULA/SOFTFORMULA rules aren't restricted to character columns
  // either - applyFormulaRules seeds the same live `=...` string into a
  // numeric column regardless of RULE_TYPE, and this resolve step (the
  // same one above) doesn't distinguish HARDFORMULA from SOFTFORMULA or
  // ad-hoc from rule-seeded - it just resolves whatever `=`-led text it
  // finds. A non-PK column is never used for row-matching, so this is
  // already the correct, complete fix - unlike a PK-as-formula-column
  // (see the freeze-at-load describe block below).
  it('submits the computed value for a HARDFORMULA-seeded numeric (non-PK) column', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, B_COL: '=3+3' }]

    runSubmitPrep(dataSource, ['PRIMARY_KEY_FIELD', 'B_COL'], new Map())

    expect(dataSource[0].B_COL).toEqual(6)
  })

  it('submits the computed value for a SOFTFORMULA-seeded numeric (non-PK) column', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, B_COL: '=4+4' }]

    runSubmitPrep(dataSource, ['PRIMARY_KEY_FIELD', 'B_COL'], new Map())

    expect(dataSource[0].B_COL).toEqual(8)
  })
})

/**
 * A primary key identifies its own row for the rest of the edit session -
 * getRowKey, dataModified (see saveTable) and classifyRow all match a row
 * up by comparing its current PK value against an earlier snapshot. A live
 * formula left sitting in a PK cell keeps recalculating right up until
 * submission resolves it, so the PK that gets matched against an earlier
 * snapshot can differ from the PK the row was snapshotted under -
 * silently dropping the row from what's submitted. Unlike any other
 * column (which stays live and is resolved only once, at submit time), a
 * PK cell resolves its formula to a plain computed value the moment that
 * edit completes - typed, pasted, autofilled, or "Apply as formula"'d -
 * exactly like editor.component.ts's own dedicated afterChange hook.
 * Mirrors that hook's resolve step against a real
 * Handsontable + HyperFormula instance, since proving a formula's
 * computed value requires a real engine.
 */
describe('a primary key resolves a live formula to its computed value as soon as editing completes', () => {
  const headerPks = ['PRIMARY_KEY_FIELD']

  // Mirrors the afterChange hook's own resolve step, minus the setTimeout
  // deferral (an implementation detail there purely to avoid re-entering
  // change processing synchronously from inside afterChange itself - the
  // resolve logic it defers to is exactly this).
  const resolvePkFormulaIfNeeded = (
    hot: Handsontable,
    row: number,
    colName: string,
    newValue: any
  ): void => {
    if (typeof newValue !== 'string' || !newValue.startsWith('=')) return
    if (!headerPks.includes(colName)) return

    const computed = hot.getDataAtRowProp(row, colName)
    hot.setDataAtRowProp(row, colName, computed, 'resolvePkFormula')
  }

  const setup = (dataSource: any[], colNames: string[]): Handsontable => {
    const hot = new Handsontable(document.createElement('div'), {
      data: dataSource,
      columns: colNames.map((c) => ({ data: c })),
      formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()
    return hot
  }

  it('resolves a formula typed/pasted/autofilled into a PK cell to its computed value once editing completes', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'x' }]
    const hot = setup(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    hot.setDataAtRowProp(0, 'PRIMARY_KEY_FIELD', '=2+3')
    resolvePkFormulaIfNeeded(hot, 0, 'PRIMARY_KEY_FIELD', '=2+3')

    expect(dataSource[0].PRIMARY_KEY_FIELD).toEqual(5)
    hot.destroy()
  })

  // The resolve step never checks the PK's own DDTYPE - only headerPks
  // membership - so a character-typed PK whose formula resolves to text
  // (not a number) is frozen exactly the same way as a numeric one.
  it('resolves a formula into a character-typed PK cell to its computed text value once editing completes', () => {
    const dataSource: any[] = [
      { PRIMARY_KEY_FIELD: 'PK-1', PLAIN_TEXT_COL: 'x' }
    ]
    const hot = setup(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    hot.setDataAtRowProp(0, 'PRIMARY_KEY_FIELD', '="PK-" & 2')
    resolvePkFormulaIfNeeded(hot, 0, 'PRIMARY_KEY_FIELD', '="PK-" & 2')

    expect(dataSource[0].PRIMARY_KEY_FIELD).toEqual('PK-2')
    hot.destroy()
  })

  it('leaves a non-PK column formula alone - stays live, resolved only at submission like before', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'x' }]
    const hot = setup(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    hot.setDataAtRowProp(0, 'PLAIN_TEXT_COL', '=2+3')
    resolvePkFormulaIfNeeded(hot, 0, 'PLAIN_TEXT_COL', '=2+3')

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual('=2+3')
    hot.destroy()
  })

  it('leaves a genuinely numeric edit into a PK column untouched - not treated as a formula', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'x' }]
    const hot = setup(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    hot.setDataAtRowProp(0, 'PRIMARY_KEY_FIELD', 5)
    resolvePkFormulaIfNeeded(hot, 0, 'PRIMARY_KEY_FIELD', 5)

    expect(dataSource[0].PRIMARY_KEY_FIELD).toEqual(5)
    hot.destroy()
  })

  it('a PK resolved eagerly on entry stays matchable against an earlier dataModified snapshot at submit time', () => {
    // The PK is already a plain computed value by the time anything
    // snapshots it, so dataModified's PK-matching never sees it change
    // out from under itself.
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'x' }]
    const hot = setup(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    hot.setDataAtRowProp(0, 'PRIMARY_KEY_FIELD', '=2+3')
    resolvePkFormulaIfNeeded(hot, 0, 'PRIMARY_KEY_FIELD', '=2+3')

    const dataModified: any[] = [
      { PRIMARY_KEY_FIELD: dataSource[0].PRIMARY_KEY_FIELD }
    ]
    const stillMatches = dataModified.some((row) =>
      headerPks.every((pk) => row[pk] === dataSource[0][pk])
    )

    expect(stillMatches).toEqual(true)
    hot.destroy()
  })
})

const submitStrip = (
  dataSource: any[],
  headerColumns: string[],
  autoEscapedCells: AutoEscapedCellMap,
  headerPks: string[]
): void => {
  for (const row of dataSource) {
    const rowKey = getRowKey(row, headerPks)
    for (const colName of headerColumns) {
      if (!isAutoEscaped(autoEscapedCells, rowKey, colName)) continue
      row[colName] = unescapeFormula(row[colName])
    }
  }
}

/**
 * Nothing gates a HARDFORMULA/SOFTFORMULA rule's BASE_COL by column type or
 * PK-ness, so a primary key can itself be one - applyFormulaRules seeds it
 * with a live `=...` formula string at load, exactly like any other
 * formula column. Left alone, that reproduces the exact bug the
 * user-entered-PK-formula fix above solves, just triggered by the rule
 * seeding instead of a user edit: saveTable's formulaRules loop only
 * resolves it once, at submit, so anything that changed a column the PK's
 * formula depends on in between would silently shift dataModified's
 * PK-matching out from under itself - and, unlike a user-entered formula,
 * a rule-seeded one is ALWAYS present in dataSource, from the very first
 * render, so it's not a rare edge case at all. resolvePkFormulaSeedValues
 * (editor.component.ts, called once right after the initial
 * hot.updateSettings()) freezes it to its computed value immediately,
 * mirrored here against a real Handsontable + HyperFormula instance -
 * writing the same frozen value onto both dataSource and
 * dataSourceUnchanged (deliberately overriding whatever
 * overlayFormulaRawValuesOnUnchanged put there), since a PK must agree
 * with itself on both sides for classifyRow's own PK-based lookup to ever
 * find its row again.
 */
describe('a primary key configured as a HARDFORMULA/SOFTFORMULA column freezes to its computed value at load', () => {
  const resolvePkFormulaSeedValues = (
    hot: Handsontable,
    dataSource: any[],
    dataSourceUnchanged: any[],
    pkFormulaCols: string[]
  ): void => {
    dataSource.forEach((_row, rowIndex) => {
      for (const pkCol of pkFormulaCols) {
        const computed = hot.getDataAtRowProp(rowIndex, pkCol)
        hot.setDataAtRowProp(rowIndex, pkCol, computed, 'resolvePkFormula')

        const unchangedRow = dataSourceUnchanged[rowIndex]
        if (unchangedRow) unchangedRow[pkCol] = computed
      }
    })
  }

  it('freezes a HARDFORMULA-driven numeric PK to its computed value on both dataSource and dataSourceUnchanged', () => {
    // applyFormulaRules seeds dataSource and dataSourceUnchanged
    // identically at load - before anything has diverged them yet.
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: '=2+3', A_COL: 1 }]
    const dataSourceUnchanged: any[] = [{ PRIMARY_KEY_FIELD: '=2+3', A_COL: 1 }]

    const hot = new Handsontable(document.createElement('div'), {
      data: dataSource,
      columns: ['PRIMARY_KEY_FIELD', 'A_COL'].map((c) => ({ data: c })),
      formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()

    resolvePkFormulaSeedValues(hot, dataSource, dataSourceUnchanged, [
      'PRIMARY_KEY_FIELD'
    ])

    expect(dataSource[0].PRIMARY_KEY_FIELD).toEqual(5)
    expect(dataSourceUnchanged[0].PRIMARY_KEY_FIELD).toEqual(5)

    // classifyRow's own PK-based lookup - both sides now agree.
    const found = dataSourceUnchanged.find((row) =>
      ['PRIMARY_KEY_FIELD'].every((pk) => row[pk] === dataSource[0][pk])
    )
    expect(found).toBeTruthy()

    hot.destroy()
  })

  it('freezes a SOFTFORMULA-driven character PK to its computed value on both dataSource and dataSourceUnchanged', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: '="PK-" & 2', A_COL: 1 }]
    const dataSourceUnchanged: any[] = [
      { PRIMARY_KEY_FIELD: '="PK-" & 2', A_COL: 1 }
    ]

    const hot = new Handsontable(document.createElement('div'), {
      data: dataSource,
      columns: ['PRIMARY_KEY_FIELD', 'A_COL'].map((c) => ({ data: c })),
      formulas: { engine: HyperFormula, licenseKey: 'gpl-v3' },
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()

    resolvePkFormulaSeedValues(hot, dataSource, dataSourceUnchanged, [
      'PRIMARY_KEY_FIELD'
    ])

    expect(dataSource[0].PRIMARY_KEY_FIELD).toEqual('PK-2')
    expect(dataSourceUnchanged[0].PRIMARY_KEY_FIELD).toEqual('PK-2')

    const found = dataSourceUnchanged.find((row) =>
      ['PRIMARY_KEY_FIELD'].every((pk) => row[pk] === dataSource[0][pk])
    )
    expect(found).toBeTruthy()

    hot.destroy()
  })
})

/**
 * A row whose backend source never included a given character column at
 * all (a sparse row - e.g. a table where only some rows use a particular
 * optional column) must stay that way after the escape loop runs, not
 * gain that key as a new own property set to undefined.
 * escapeCharacterColumnValue(undefined) legitimately returns
 * { value: undefined, wasEscaped: false } - there's nothing to escape -
 * but writing that back via row[colName] = value unconditionally
 * materializes a property the row never had. The SASjs adapter's own
 * request validation rejects any row holding an explicit `undefined`
 * value ("Can't assign undefined to <COL>"), so a submit fails for a
 * cell that was never touched at all. Mirrors initSetup/
 * getPendingExcelPreview's own escape loop.
 */
describe('the escape loop never materializes a missing column key as an explicit undefined value', () => {
  const runEscapeLoop = (dataSource: any[], headerColumns: string[]): void => {
    for (const row of dataSource) {
      for (const colName of headerColumns) {
        const { value, wasEscaped } = escapeCharacterColumnValue(row[colName])
        if (!wasEscaped) continue

        row[colName] = value
      }
    }
  }

  it('leaves a genuinely-missing column key missing, not undefined, when nothing needs escaping', () => {
    const dataSource: any[] = [
      { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'note-1' }
    ]
    // OTHER_COL is never a key on this row at all - this table's backend
    // source simply never populated it for this particular row.

    runEscapeLoop(dataSource, [
      'PRIMARY_KEY_FIELD',
      'PLAIN_TEXT_COL',
      'OTHER_COL'
    ])

    expect(
      Object.prototype.hasOwnProperty.call(dataSource[0], 'OTHER_COL')
    ).toEqual(false)
  })

  it('still escapes and writes back a genuine =-led value', () => {
    const dataSource: any[] = [{ PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: '=1+1' }]

    runEscapeLoop(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual("'=1+1")
  })

  it('leaves an existing plain (non-formula) value untouched, still present', () => {
    const dataSource: any[] = [
      { PRIMARY_KEY_FIELD: 1, PLAIN_TEXT_COL: 'note-1' }
    ]

    runEscapeLoop(dataSource, ['PRIMARY_KEY_FIELD', 'PLAIN_TEXT_COL'])

    expect(dataSource[0].PLAIN_TEXT_COL).toEqual('note-1')
  })
})

/**
 * Now that the escape loop above leaves a genuinely-missing column key
 * missing (instead of materializing it as undefined), a sparse row - one
 * whose backend source never populated some earlier column - has fewer
 * real keys than the grid has columns. dynamicCellValidation
 * (editor.component.ts) used to resolve the clicked column's name via
 * `Object.keys(clickedRow)[column]`, a positional guess that only worked
 * by coincidence when every row had the full set of keys. For a sparse
 * row it drifts out of alignment (or runs out of bounds entirely) once
 * the clicked column index passes the gap, resolving to `undefined` -
 * which then got sent as a literal `variable_nm: undefined` in a
 * getdynamiccolvals request, tripping the same SASjs adapter validation
 * ("Can't assign undefined to variable_nm"). hot.colToProp(column) reads
 * the column's actual configured data key instead, unaffected by the
 * row's own key count/order.
 */
describe("dynamicCellValidation resolves the clicked column by its Handsontable mapping, not by counting the row object's own keys", () => {
  it('demonstrates the failure mode: Object.keys(row)[column] runs out of bounds for a sparse row', () => {
    const hot = new Handsontable(document.createElement('div'), {
      data: [{ PRIMARY_KEY_FIELD: 1, LAST_COL: 'x' }], // MIDDLE_COL missing entirely
      columns: [
        { data: 'PRIMARY_KEY_FIELD' },
        { data: 'MIDDLE_COL' },
        { data: 'LAST_COL' }
      ],
      licenseKey: 'non-commercial-and-evaluation'
    })
    hot.render()

    const lastColIndex = 2
    const clickedRow = { PRIMARY_KEY_FIELD: 1, LAST_COL: 'x' }

    expect(Object.keys(clickedRow)[lastColIndex]).toBeUndefined()
    expect(hot.colToProp(lastColIndex)).toEqual('LAST_COL')

    hot.destroy()
  })
})