const nodePath = require('path')

let appLoc = nodePath.join(..._program.split('services')[0].split('/'))
const sasjsRoot = nodePath.resolve(weboutPath, '..', '..', '..')
const driveRoot = nodePath.resolve(sasjsRoot, 'drive')
const dcLibref = 'DC_JSLIB'

// Load shared DC mock utilities
eval(fs.readFileSync(nodePath.resolve(driveRoot, 'files', appLoc, 'services', 'dcMockUtils.js'), 'utf8'))

// Mirrors SAS cats() for the values a dropdown source can hold: a special
// missing stored in its period form (".a") becomes the bare uppercase letter
// ("A") that cats() returns, so the mock's dropdown list matches the real one.
function cats(value) {
    if (typeof value === 'string') {
        const m = /^\.(_|[a-z])$/i.exec(value.trim())
        if (m) return m[1].toUpperCase()
        return value.trim()
    }
    return String(value)
}

/**
 * SAS sort position of a special missing, or null when the value is ordinary.
 * `._` sorts first, then `.a` to `.z` - all of them below every number.
 */
function missingRank(value) {
    const m = /^\.(_|[a-z])$/i.exec(String(value).trim())
    if (!m) return null
    return m[1] === '_' ? 0 : m[1].toUpperCase().charCodeAt(0) - 64
}

// ─── Parse input ──────────────────────────────────────────────────────────────

const _sctRow = fetchTable('SASControlTable')[0] || {}
let requestedLibref = dcLibref
let requestedTable = ''
if (_sctRow.LIBDS) {
    const parts = _sctRow.LIBDS.split('.')
    if (parts.length >= 2) { requestedLibref = parts[0]; requestedTable = parts[1] }
}
if (!requestedTable) {
    if (_sctRow.DSN) requestedTable = _sctRow.DSN.trim()
    if (_sctRow.LIBREF) requestedLibref = _sctRow.LIBREF.trim()
}

// FILTER_RK (matching getdata.sas: if filter_rk le 0 then filter_rk=-1)
const filterRk = Number(_sctRow.FILTER_RK) > 0 ? Number(_sctRow.FILTER_RK) : -1

// ─── Load table data and schema ──────────────────────────────────────────────
// A table can live in any libref (DC_JSLIB for control tables, TESTDATA for
// demo user tables), so resolve the data directory from the libref.

const dataDir = libDataDir(requestedLibref)
const loadTableData = makeTableLoader(dataDir)

let tableData = loadTableData(requestedTable)
if (!tableData) {
    // Fall back to searching every libref if the table isn't in the requested
    // libref (e.g. a LIBDS that doesn't match the actual data folder)
    const found = loadTableAnyLib(requestedTable)
    if (found) {
        tableData = found.data
        requestedLibref = found.libref
    }
}

// Load table registration from MPE_TABLES (needed for PK detection, temporal
// column exclusion, and sasparams). MPE_TABLES always lives in the control
// library (DC_JSLIB / mpelib), not in user data libraries.
const mpeDataDir = libDataDir(dcLibref)
const mpeLoadTableData = makeTableLoader(mpeDataDir)

let tableReg = null
const mpeTablesData = mpeLoadTableData('MPE_TABLES')
if (mpeTablesData && mpeTablesData.rows) {
    tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
}

// The variables exposed to the editor as the primary key, matching getdata.sas:
// pk = coalesce(rk_underlying, buskey). A table registered with rk_underlying
// is keyed on its business key in the editor - the RK itself is assigned by the
// backend loader, so it is neither shown nor sent by the user.
const rkUnderlying = tableReg && tableReg.rk_underlying ? tableReg.rk_underlying.trim() : ''
const buskey = tableReg && tableReg.buskey ? tableReg.buskey.trim() : ''
const pkFields = rkUnderlying || buskey
const pkColumns = pkFields ? pkFields.split(/\s+/) : []

// Order columns with the PK variables first, in PK order, then the remaining
// columns in schema order. getdata.sas does the same ("order delete var and pk
// fields at start of table") and the client reads the first PKCNT columns
// positionally when it detects duplicate keys.
function pkFirst(columns, keys) {
    const rank = new Map(keys.map((name, i) => [name.toUpperCase(), i]))
    const keyColumns = columns
        .filter(col => rank.has(col.name.toUpperCase()))
        .sort((a, b) => rank.get(a.name.toUpperCase()) - rank.get(b.name.toUpperCase()))
    return keyColumns.concat(columns.filter(col => !rank.has(col.name.toUpperCase())))
}

// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
// Applied before the filter, so a filter can reference them.  The derived
// columns are added to the schema too, which is what puts them in COLS and in
// the grid; they are held read-only by a READONLY rule below.

const derived = mpeDerived({
    libref: requestedLibref,
    table: requestedTable,
    dataDir: mpeDataDir,
    columns: tableData ? tableData.columns : []
})
if (tableData && derived.length > 0) {
    tableData = mpeDeriveRows(tableData, derived)
}

// ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
// Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
// NOTNULL constraints. MPE_VALIDATIONS lives in the control library.

const validationsData = mpeLoadTableData('MPE_VALIDATIONS')
let dqrules = []
let dqdata = []
if (validationsData && validationsData.rows) {
    const pkCols = new Set(pkColumns.map(c => c.toUpperCase()))
    dqrules = validationsData.rows
        .filter(r => r.base_lib === requestedLibref && r.base_ds === requestedTable)
        .map(r => ({
            BASE_COL: r.base_col,
            RULE_TYPE: r.rule_type,
            RULE_VALUE: r.rule_value,
            X: pkCols.has(r.base_col.toUpperCase()) ? 0 : 1
        }))
}

// ─── Apply stored filter (mpe_filtermaster) ──────────────────────────────────
// Mirrors getdata.sas: %mpe_filtermaster(EDIT,&orig_libds,filter_rk=&filter_rk)

const mpeDataDir2 = libDataDir(dcLibref)
const {
    query: filterQuery,
    predicate: filterPredicate,
    filterText: filterText
} = mpeFilterMaster({
    mode: 'EDIT',
    libds: (requestedLibref + '.' + requestedTable).toUpperCase(),
    filterRk: filterRk,
    dataDir: mpeDataDir2,
    columns: tableData ? tableData.columns : [],
    tableReg: tableReg
})

// Synthesise NOTNULL rules from the table schema, the same way getdata.sas
// merges dictionary.columns (notnull='yes') into dqrules.
if (tableData && tableData.columns) {
    const existingNotNull = new Set(
        dqrules.filter(r => r.RULE_TYPE === 'NOTNULL').map(r => r.BASE_COL.toUpperCase())
    )
    for (const rule of notNullRules(tableData.columns)) {
        if (!existingNotNull.has(rule.BASE_COL)) dqrules.push(rule)
    }
}

// A derived variable is computed at read time, so the grid must not accept a
// value for it.  COLS carries READ for the same reason (a derived column has no
// CLS rule row), but the client only applies COLS.CLS_RULE when CLS_FLAG is
// set, which column-level security drives - so the rule is declared here too,
// as getdata.sas does.  dqrules is the request-local array the filter/map above
// built, so appending to it leaves the loaded MPE_VALIDATIONS rows untouched.
for (const v of derived) {
    dqrules.push({ BASE_COL: v.name, RULE_TYPE: 'READONLY', RULE_VALUE: '' })
}

// ─── DQ dropdown data (from MPE_SELECTBOX + SOFTSELECT from table data) ──────
// MPE_SELECTBOX lives in the control library.

const selectboxData = mpeLoadTableData('MPE_SELECTBOX')
if (selectboxData && selectboxData.rows) {
    dqdata = selectboxData.rows
        .filter(r => r.select_lib === requestedLibref && r.select_ds === requestedTable)
        .map(r => ({
            BASE_COL: r.base_column,
            RULE_VALUE: r.base_column,
            RULE_DATA: r.selectbox_value,
            SELECTBOX_ORDER: r.selectbox_order
        }))
}

// For SOFTSELECT/HARDSELECT rules that reference the table's own data,
// generate dropdown values from the actual table rows
if (tableData && tableData.rows && tableData.columns) {
    const selectRules = dqrules.filter(r =>
        (r.RULE_TYPE === 'SOFTSELECT' || r.RULE_TYPE === 'HARDSELECT') && r.RULE_VALUE.includes('.'))
    for (const rule of selectRules) {
        const parts = rule.RULE_VALUE.split('.')
        const colName = parts[parts.length - 1]
        const lcName = colName.toLowerCase()
        const seen = new Set()
        const values = []
        for (const row of tableData.rows) {
            let val = row[colName]
            if (val === undefined) val = row[lcName]
            if (val === undefined) {
                const key = Object.keys(row).find(k => k.toLowerCase() === lcName)
                if (key) val = row[key]
            }
            if (val !== undefined && val !== null) {
                const strVal = cats(val)
                if (!seen.has(strVal)) {
                    seen.add(strVal)
                    values.push({ raw: val, str: strVal })
                }
            }
        }
        // getdata.sas orders the source by the column itself, and a special
        // missing sorts below every number - so the list reads `._`, `.a`-`.z`,
        // then the numbers ascending.
        values.sort((a, b) => {
            const ra = missingRank(a.raw)
            const rb = missingRank(b.raw)
            if (ra !== null && rb !== null) return ra - rb
            if (ra !== null) return -1
            if (rb !== null) return 1
            const na = Number(a.str)
            const nb = Number(b.str)
            if (!isNaN(na) && !isNaN(nb)) return na - nb
            return a.str < b.str ? -1 : a.str > b.str ? 1 : 0
        })
        values.forEach((v, i) => {
            dqdata.push({ BASE_COL: rule.BASE_COL, RULE_VALUE: rule.RULE_VALUE, RULE_DATA: v.str, SELECTBOX_ORDER: i + 1 })
        })
    }
}

// ─── Build response ──────────────────────────────────────────────────────────

let response
// Set when the row or cell limit cut the selection down - mirrors the
// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.

if (tableData && tableData.columns && tableData.rows) {
    // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
    let excludeCols = new Set()
    if (tableReg && (tableReg.loadtype === 'TXTEMPORAL' || tableReg.loadtype === 'BITEMPORAL')) {
        if (tableReg.var_txfrom) excludeCols.add(tableReg.var_txfrom.toUpperCase())
        if (tableReg.var_txto) excludeCols.add(tableReg.var_txto.toUpperCase())
    }
    if (tableReg && tableReg.loadtype === 'BITEMPORAL') {
        if (tableReg.var_busfrom) excludeCols.add(tableReg.var_busfrom.toUpperCase())
        if (tableReg.var_busto) excludeCols.add(tableReg.var_busto.toUpperCase())
    }
    // getdata.sas drops the RK field from the editor view when the table is
    // registered with rk_underlying (drop_rk=&buskey), and exposes the business
    // key instead.
    if (rkUnderlying && buskey) {
        buskey.split(/\s+/).forEach(c => excludeCols.add(c.toUpperCase()))
    }

    let visibleColumns = pkFirst(
        tableData.columns.filter(c => !excludeCols.has(c.name.toUpperCase())),
        pkColumns
    )
    let visibleRows = tableData.rows
        .filter(r => filterPredicate(r))
        .map(r => {
        const copy = {}
        for (const key of Object.keys(r)) {
            if (!excludeCols.has(key.toUpperCase())) copy[key] = r[key]
        }
        return copy
    })

    // ─── PRE_EDIT_HOOK ───────────────────────────────────────────────────────
    // Mirrors getdata.sas: %mpe_runhook(PRE_EDIT_HOOK) runs after the filter has
    // been applied and the rows sorted, with the data in work.OUT, and may
    // replace it.  The mock eval()s the hook's .js counterpart in this scope, so
    // the hook may reassign visibleRows / visibleColumns - e.g. to show the live
    // rows of another table behind an empty mirror.
    if (tableReg && tableReg.pre_edit_hook) {
        const hookSrc = mockHookSource(tableReg.pre_edit_hook)
        if (hookSrc) {
            try {
                eval(hookSrc)
            } catch (err) {
                console.log('Error running pre_edit_hook ' + tableReg.pre_edit_hook + ': ' + err.message)
            }
        }
    }

    // ─── Row and cell limits ─────────────────────────────────────────────────
    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
    // cells (rows x columns), whichever is reached first - the selection is
    // cut to min(rows, cells/columns) and flagged in sasparams.
    // Both options are read from MPE_CONFIG, the way the real service picks
    // them up from the settings program it generates - so editing the option
    // in DC changes what the mock serves, and an option that is absent (or
    // switched off, var_active=0) falls back to the shipped default.  The
    // check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
    // does - a hook may replace the data with a larger table.
    // MPE_CONFIG lives in the control library, not in the table's own library,
    // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
    // uses.  Passing the requested table's dataDir made the read fail for every
    // non-control libref, and the catch then fell back to the shipped defaults.
    const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
    const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
    const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
    // getdata.sas counts the columns of work.out, i.e. the table's own
    // columns - the delete column is client-owned and not part of it.
    const editColCount = visibleColumns.length
    let maxRows = maxObsWebEdit
    let limitVar = 'DC_MAXOBS_WEBEDIT'
    if (editColCount > 0) {
        const cellRows = Math.floor(maxCellsWebEdit / editColCount)
        if (cellRows < maxRows) {
            maxRows = cellRows
            limitVar = 'DC_MAXCELLS_WEBEDIT'
        }
    }
    if (maxRows < 1) maxRows = 1
    let truncated = 0
    let truncMsg = ''
    if (visibleRows.length > maxRows) {
        truncated = 1
        // Same wording as the macro getdata.sas sets, so the spec can assert
        // on the text the real service would send.
        truncMsg =
            'Only the first ' +
            maxRows +
            ' rows are loaded - the ' +
            limitVar +
            ' limit has been reached.  Filter the table to work with the rest.'
        visibleRows = visibleRows.slice(0, maxRows)
    }

    // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
    const colLookup = {}
    for (const col of visibleColumns) {
        colLookup[col.name.toLowerCase()] = { name: col.name, ddtype: getDdType(col) }
    }
    const augmentedRows = visibleRows.map(r => {
        const normalised = {}
        for (const key of Object.keys(r)) {
            const info = colLookup[key.toLowerCase()]
            const targetKey = info ? info.name : key
            normalised[targetKey] = info ? formatCellValue(r[key], info.ddtype) : r[key]
        }
        return { ...normalised }
    })

    // $sasdata vars: temporal columns are char ($200.), others keep their SAS type
    const vars = {}
    for (const col of visibleColumns) {
        vars[col.name] = sasVarsEntry(col)
    }

    const colHeaders = visibleColumns.map(c => c.name)

    // Determine DTVARS, DTTMVARS, TMVARS from column formats
    let dtVars = [], dttmVars = [], tmVars = []
    for (const col of visibleColumns) {
        const ddtype = getDdType(col)
        if (ddtype === 'DATETIME') dttmVars.push(col.name)
        else if (ddtype === 'DATE') dtVars.push(col.name)
        else if (ddtype === 'TIME') tmVars.push(col.name)
    }

    // Build cols array.  Every column of the table, as getdata.sas emits them:
    // the temporal columns are dropped from the data (the rows below) but not
    // from the cols payload, so a filter can still be built on them.  mpGetcols
    // is the JS counterpart of %mp_getcols.sas - the macro getdata.sas itself
    // calls to type these columns - so the editor sees the same DDTYPE the
    // service would give it.
    const cols = mpGetcols(tableData.columns).map((c, i) => {
        const raw = tableData.columns[i]
        const ddtype = c.DDTYPE === 'CHARACTER' ? 'C' : c.DDTYPE === 'NUMERIC' ? 'N' : c.DDTYPE
        let coltype
        if (ddtype === 'DATE') coltype = `{"data":"${raw.name}","type":"date"}`
        else if (ddtype === 'DATETIME') coltype = `{"data":"${raw.name}","type":"datetime"}`
        else if (ddtype === 'TIME') coltype = `{"data":"${raw.name}","type":"time"}`
        else if (ddtype === 'N') coltype = `{"data":"${raw.name}","type":"numeric","format":"0"}`
        else coltype = `{"data":"${raw.name}"}`
        return {
            NAME: c.NAME,
            LABEL: c.LABEL,
            FMTNAME: c.FMTNAME || ' ',
            DDTYPE: ddtype,
            CLS_RULE: 'READ',
            MEMLABEL: ' ',
            DESC: ' ',
            LONGDESC: ' ',
            COLTYPE: coltype
        }
    }).sort((a, b) => a.NAME.localeCompare(b.NAME))

    // Build maxvarlengths
    const maxvarlengths = [
        ...visibleColumns.map(col => {
            let maxLen = 0
            const lcName = col.name.toLowerCase()
            for (const row of augmentedRows) {
                let val = row[col.name]
                if (val === undefined) val = row[lcName]
                if (val !== undefined && val !== null) {
                    const len = String(val).length
                    if (len > maxLen) maxLen = len
                }
            }
            return { NAME: col.name.toLowerCase(), MAXLEN: maxLen }
        })
    ]

    response = {
        approvers: [{ PERSONNAME: 'sasdemo', EMAIL: 'sasdemo', USERID: 'sasdemo' }],
        cols,
        dqdata, dqrules,
        dsmeta: [
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: String(tableData.rows.length) },
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' },
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Variables', VALUE: String(tableData.columns.length) },
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Engine', VALUE: 'V9' }
        ],
        maxvarlengths,
        query: filterQuery,
        sasdata: augmentedRows,
        $sasdata: { vars },
        sasparams: [{
            COLHEADERS: colHeaders.join(','),
            FILTER_TEXT: filterText === '' ? ' ' : filterText,
            PKCNT: pkColumns.length,
            PK: pkFields,
            DTVARS: dtVars.length ? ' ' + dtVars.join(' ') : '',
            DTTMVARS: dttmVars.length ? ' ' + dttmVars.join(' ') : '',
            TMVARS: tmVars.length ? ' ' + tmVars.join(' ') : '',
            LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
            RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
            CLS_FLAG: 0,
            ISMAP: 0,
            TRUNCATED: truncated,
            TRUNC_MSG: truncMsg,
            // names the derived columns for the grid, as getdata.sas does - the
            // READONLY rule above is why they cannot be edited, this is how the
            // EDIT screen knows they are computed rather than locked
            DERIVEDVARS: derived.map(d => d.name).join(' ')
        }],
        versions: [],
        xl_rules: []
    }
} else {
    response = {
        approvers: [], cols: [], dqdata: [], dqrules: [],
        dsmeta: [
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Data Set Name', VALUE: requestedLibref + '.' + requestedTable },
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Observations', VALUE: '0' },
            { ODS_TABLE: 'ATTRIBUTES', NAME: 'Member Type', VALUE: 'DATA' }
        ],
        maxvarlengths: [], query: [], sasdata: [],
        $sasdata: { vars: {} },
        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '', DERIVEDVARS: '' }],
        versions: [], xl_rules: []
    }
}

webOutOpen()
webOutObj(response.approvers, 'approvers')
webOutObj(response.cols, 'cols')
webOutObj(response.dqdata, 'dqdata')
webOutObj(response.dqrules, 'dqrules')
webOutObj(response.dsmeta, 'dsmeta')
webOutObj(response.maxvarlengths, 'maxvarlengths')
webOutObj(response.query, 'query')
webOutObj(response.sasdata, 'sasdata', response.$sasdata)
webOutObj(response.sasparams, 'sasparams')
webOutObj(response.versions, 'versions')
webOutObj(response.xl_rules, 'xl_rules')
webOutClose()