import {
  AfterViewInit,
  ChangeDetectorRef,
  Component,
  ElementRef,
  OnInit,
  QueryList,
  ViewChild,
  ViewChildren,
  ViewEncapsulation
} from '@angular/core'
import { ActivatedRoute, Router } from '@angular/router'
import Handsontable from 'handsontable'
import { Subject, Subscription } from 'rxjs'
import { SasStoreService } from '../services/sas-store.service'

import * as XLSX from '@sheet/crypto'

/**
 * Used in combination with buffer
 */
const iconv = require('iconv-lite')
/**
 * In combination with `iconv` is used for encoding json data captured with sheet js from excel file into a file again
 * Which will be send to backend
 */
const Buffer = require('buffer/').Buffer
type AOA = any[][]

import { HotTableRegisterer } from '@handsontable/angular'
import { UploadFile } from '@sasjs/adapter'
import { isSpecialMissing } from '@sasjs/utils/input/validators'
import CellRange from 'handsontable/3rdparty/walkontable/src/cell/range'
import { CellValidationSource } from '../models/CellValidationSource'
import { FileUploader } from '../models/FileUploader.class'
import { FilterGroup, FilterQuery } from '../models/FilterQuery'
import { HotTableInterface } from '../models/HotTable.interface'
import {
  $DataFormats,
  DSMeta,
  EditorsGetDataServiceResponse,
  Version
} from '../models/sas/editors-getdata.model'
import { DataFormat } from '../models/sas/common/DateFormat'
import SheetInfo from '../models/SheetInfo'
import { Approver, ExcelRule } from '../models/TableData'
import { QueryComponent } from '../query/query.component'
import { EventService } from '../services/event.service'
import { HelperService } from '../services/helper.service'
import { LoggerService } from '../services/logger.service'
import { SasService } from '../services/sas.service'
import { DcValidator } from '../shared/dc-validator/dc-validator'
import { Col } from '../shared/dc-validator/models/col.model'
import { DcValidation } from '../shared/dc-validator/models/dc-validation.model'
import { DQRule } from '../shared/dc-validator/models/dq-rules.model'
import { getHotDataSchema } from '../shared/dc-validator/utils/getHotDataSchema'
import { globals } from '../_globals'
import { UploadStaterComponent } from './components/upload-stater/upload-stater.component'
import { DynamicExtendedCellValidation } from './models/dynamicExtendedCellValidation'
import { EditRecordInputFocusedEvent } from './models/edit-record/edit-record-events'
import { EditorRestrictions } from './models/editor-restrictions.model'
import { dateFormat, dateToTime, dateToUtcTime } from './utils/date.utils'
import {
  excelDateToJSDate,
  getMissingHeaders,
  parseTableColumns
} from './utils/grid.utils'
import {
  errorRenderer,
  noSpinnerRenderer,
  spinnerRenderer
} from './utils/renderers.utils'
import { isStringDecimal, isStringNumber } from './utils/types.utils'
import { LicenceService } from '../services/licence.service'
import * as numbro from 'numbro'
import * as languages from 'numbro/dist/languages.min'

@Component({
  selector: 'app-editor',
  templateUrl: './editor.component.html',
  styleUrls: ['./editor.component.scss'],
  host: {
    class: 'content-container'
  },
  encapsulation: ViewEncapsulation.Emulated
})
export class EditorComponent implements OnInit, AfterViewInit {
  @ViewChildren('uploadStater')
  uploadStaterCompList: QueryList<UploadStaterComponent> = new QueryList()
  @ViewChildren('queryFilter')
  queryFilterCompList: QueryList<QueryComponent> = new QueryList()
  @ViewChildren('hotInstance')
  hotInstanceCompList: QueryList<Handsontable> = new QueryList()
  @ViewChildren('fileUploadInput')
  fileUploadInputCompList: QueryList<ElementRef> = new QueryList()

  public static cnt: number = 0
  public static nonPkCnt: number = 0
  public static lastCell: number = 0
  private _tableSub: Subscription | undefined
  public message: string = ''
  public $dataFormats: $DataFormats | null = null
  public submit: boolean | undefined
  public cols: Col[] = []

  @ViewChild('ht', { static: true }) ht!: ElementRef

  /** Feature restrictions
   *
   * What can be restricted
   * - Add Row button
   * - Insert rows above/below
   * - Add record button
   * - Edit record button
   *
   * Types of limitations ordered by priority of enforcement (Restrictions upper on the list cannot be un-restricted by lower types)
   * - Restrict edit record feature with config in startupservice (comes from appService)
   * - Restrict `edit record feature` and `add row` if demo and demo limits set as such - since demo is limited to less rows, and buttons which adds rows triggers error
   * - Restrict `add record feature` and `add row` based on configuration of `Column Level Security`
   */
  restrictions: EditorRestrictions = {}

  datasetInfo: boolean = false
  dsmeta: DSMeta[] = []
  versions: Version[] = []
  dsNote = ''

  viewboxes: boolean = false

  Infinity = Infinity

  public hotInstance!: Handsontable
  public dcValidator: DcValidator | undefined

  public hotTable: HotTableInterface = {
    data: [],
    colHeaders: [],
    hidden: true,
    columns: [],
    height: '100%',
    minSpareRows: 1,
    licenseKey: undefined,
    readOnly: true,
    copyPaste: {
      copyColumnHeaders: true,
      copyColumnHeadersOnly: true
    },
    settings: {
      contextMenu: {
        items: {
          edit_row: {
            name: 'Edit row',
            hidden() {
              const hot: Handsontable.Core = this

              const fullCellRange: CellRange[] | undefined =
                hot.getSelectedRange()

              if (!fullCellRange) return false

              const cellRange = fullCellRange[0]

              return cellRange.from.row !== cellRange.to.row
            },
            callback: (
              key: string,
              selection: any[],
              clickEvent: MouseEvent
            ) => {
              let firstSelection = selection[0]

              if (firstSelection.start.row === firstSelection.end.row) {
                this.editRecord(null, firstSelection.start.row)
              }
            }
          },
          row_above: {
            name: 'Insert Row above'
          },
          row_below: {
            name: 'Insert Row below'
          },
          remove_row: {
            name: 'Ignore row'
          },
          copy: {
            name: 'Copy without headers'
          },
          copy_with_column_headers: {
            name: 'Copy with headers'
          },
          copy_column_headers_only: {
            name: 'Copy headers only'
          },
          sp1: {
            name: '---------'
          },
          undo: {
            name: 'Undo'
          },
          redo: {
            name: 'Redo'
          }
        }
      }
    }
  }

  public hotCellsPropRow: number | null = null

  public filter: boolean = false
  public submitLoading: boolean = false
  public uploadLoading: boolean = false
  public rowsChanged: any = {
    rowsUpdated: 0,
    rowsDeleted: 0,
    rowsAdded: 0
  }
  public modifedRowsIndexes: number[] = []
  public queryErr: boolean = false
  public queryErrMessage: string | undefined
  public successEnable: boolean = false
  public libTab: string | undefined
  public queryFilter: any
  public _query: Subscription | undefined

  public whereString: string | undefined
  public clauses: any
  public nullVariables: boolean = false

  public tableId: string | undefined
  public pkFields: any = []

  public libds: string | undefined
  public filter_pk: string | undefined
  public table: any
  public filename: string = ''
  public selectedColumn: any
  public hotSelection: Array<number> | null | undefined
  public submitLimitNotice: boolean = false
  public badEdit: boolean = false
  public badEditCause: string | undefined
  public badEditTitle: string | undefined
  public tableTrue: boolean | undefined
  public saveLoading: boolean = false
  public approvers: string[] = []
  public approver: any
  public readOnlyFields!: number
  public errValidation: boolean = false
  public dataObj: any
  public disableSubmit: boolean | undefined
  public pkNull: boolean = false
  public noPkNull: boolean = false
  public tableData: Array<any> = []
  public queryText: string = ''
  public queryTextSaved: string = ''
  public showApprovers: boolean = false
  public pkDups: boolean = false
  public validationDone: number = 0
  public duplicatePkIndexes: any = []
  public columnHeader: string[] = []
  public specInfo: { col: string; len: number; type: number }[] = []
  public tooLong: boolean = false
  public exceedCells: {
    col: string
    len: number
    val: string
  }[] = []
  public uploader: FileUploader = new FileUploader()
  public uploadUrl: string = ''
  public excelFileReady: boolean = false
  public uploadPreview: boolean = false
  public excelFileParsing: boolean = false
  public excelUploadState: string | null = null
  public data: AOA = []
  public headerArray: string[] = []
  public hotDataSchema: any = {}
  public headerShow: string[] = []
  public headerVisible: boolean = false
  public hasBaseDropZoneOver: boolean = false
  public hasAnotherDropZoneOver: boolean = false
  public headerPks: string[] = []
  public columnLevelSecurityFlag: boolean = false
  public dateTimeHeaders: string[] = []
  public timeHeaders: string[] = []
  public dateHeaders: string[] = []

  public xlRules: ExcelRule[] = []
  public encoding: string = 'UTF-8'

  // header column names
  headerColumns: Array<any> = []
  cellValidation: DcValidation[] = []
  // hot table data source
  dataSource!: any[]
  prevDataSource!: any[]
  dataSourceUnchanged!: any[]
  dataSourceBeforeSubmit!: any[]
  dataModified!: any[]

  public filePasswordSubject: Subject<string | undefined> = new Subject()
  public fileUnlockError: boolean = false

  public filePasswordModal: boolean = false
  public showUploadModal: boolean = false
  public discardSourceFile: boolean = false
  public manualFileEditModal: boolean = false

  public recordAction: string | null = null

  public currentEditRecord: any
  public currentEditRecordValidator: DcValidator | undefined
  public currentEditRecordLoadings: number[] = []
  public currentEditRecordErrors: number[] = []
  public currentEditRecordIndex: number = -1

  public generateEditRecordUrlLoading: boolean = false
  public generatedRecordUrl: string | null = null
  public addRecordUrl: string | null = null
  public recordNewOrPkModified: boolean = false
  public addRecordLoading: boolean = false
  public singleRowSelected: boolean = false
  public addingNewRow: boolean = false
  public getdataError: boolean = false
  public zeroFilterRows: boolean = false

  public tableFileDragOver: boolean = false

  /**
   * Hash/values table used for dynamic cell validation
   */
  public cellValidationSource: CellValidationSource[] = []
  public validationTableLimit: number = 20
  public extendedCellValidationFields: {
    DISPLAY_INDEX: number
    EXTRA_COL_NAME: number
    DISPLAY_VALUE: number
    DISPLAY_TYPE: number
    RAW_VALUE_NUM: number
    RAW_VALUE_CHAR: number
    FORCE_FLAG: number
  } = {
    DISPLAY_INDEX: 0,
    EXTRA_COL_NAME: 1,
    DISPLAY_VALUE: 2,
    DISPLAY_TYPE: 3,
    RAW_VALUE_NUM: 4,
    RAW_VALUE_CHAR: 5,
    FORCE_FLAG: 6
  }

  public cellValidationFields: {
    DISPLAY_INDEX: number
    DISPLAY_VALUE: number
    RAW_VALUE: number
  } = { DISPLAY_INDEX: 0, DISPLAY_VALUE: 1, RAW_VALUE: 2 }

  public disabledBasicDynamicCellValidationMap: {
    row: number
    col: number
    active: boolean
  }[] = []

  public licenceState = this.licenceService.licenceState

  constructor(
    private licenceService: LicenceService,
    private eventService: EventService,
    private loggerService: LoggerService,
    private sasStoreService: SasStoreService,
    private helperService: HelperService,
    private router: Router,
    private route: ActivatedRoute,
    private sasService: SasService,
    private cdf: ChangeDetectorRef,
    private hotRegisterer: HotTableRegisterer
  ) {
    const lang = languages[window.navigator.language]
    if (lang)
      numbro.default.registerLanguage(languages[window.navigator.language])

    this.hotRegisterer = new HotTableRegisterer()

    this.parseRestrictions()
    this.setRestrictions()
  }

  /**
   * Prepare feature restrictions based on licence key
   */
  private parseRestrictions() {
    this.restrictions.restrictAddRecord =
      this.licenceState.value.addRecord === false
    this.restrictions.restrictEditRecord =
      this.licenceState.value.editRecord === false
    this.restrictions.restrictFileUpload =
      this.licenceState.value.fileUpload === false
  }

  /**
   * Applying prepared restrictions
   * @param overrideRestrictions can be used to apply and override specific restrictions
   */
  private setRestrictions(overrideRestrictions?: EditorRestrictions) {
    if (overrideRestrictions) {
      this.restrictions = {
        ...this.restrictions,
        ...overrideRestrictions
      }
    }

    if (this.restrictions.removeEditRecordButton) {
      delete (this.hotTable?.settings?.contextMenu as any).items.edit_row
    }

    if (this.restrictions.restrictAddRow) {
      delete (this.hotTable?.settings?.contextMenu as any).items.row_above
      delete (this.hotTable?.settings?.contextMenu as any).items.row_below
      delete (this.hotTable?.settings?.contextMenu as any).items.remove_row
    }
  }

  /**
   * Disabling add row button based on wether rows limit is present
   */
  private checkRowLimit() {
    if (this.columnLevelSecurityFlag) return

    if (this.licenceState.value.editor_rows_allowed !== Infinity) {
      if (
        this.dataSource?.length >= this.licenceState.value.editor_rows_allowed
      ) {
        this.restrictions.restrictAddRow = true
      } else {
        this.restrictions.restrictAddRow = false
      }
    }
  }

  /**
   * Resetting filter variables
   */
  public resetFilter() {
    if (this.queryFilterCompList.first) {
      this.queryFilterCompList.first.resetFilter()
    }
  }

  /**
   * Openning file upload modal
   * If feature is locked, `feature locked` modal will be shown
   */
  public onShowUploadModal() {
    if (this.restrictions.restrictFileUpload) {
      this.eventService.showDemoLimitModal('File Upload')
      return
    }

    if (this.columnLevelSecurityFlag) {
      this.eventService.showInfoModal(
        'Information',
        'Upload feature is disabled while Column Level Security rules are active'
      )
      return
    }

    if (!this.uploadPreview) this.showUploadModal = true
  }

  /**
   * Called by FileDropDirective
   * @param e true if file is dragged over the drop zone
   */
  public fileOverBase(e: boolean): void {
    this.hasBaseDropZoneOver = e
  }

  /**
   * Function that updates the !ref range value provided in official docs.
   * @param ws worksheet to be updated
   */
  private update_sheet_range(ws: XLSX.WorkSheet) {
    var range = { s: { r: Infinity, c: Infinity }, e: { r: 0, c: 0 } }
    Object.keys(ws)
      .filter(function (x) {
        return x.charAt(0) != '!'
      })
      .map(XLSX.utils.decode_cell)
      .forEach(function (x: any) {
        range.s.c = Math.min(range.s.c, x.c)
        range.s.r = Math.min(range.s.r, x.r)
        range.e.c = Math.max(range.e.c, x.c)
        range.e.r = Math.max(range.e.r, x.r)
      })

    ws['!ref'] = XLSX.utils.encode_range(range)
  }

  /**
   * Function that gives the sheet name which contains data and range of data in that sheet, if some headers are missing then also gives the info about those missing headers
   * @param wb Excel workbook
   * @returns {object: SheetInfo} an object which contains necessary information about workbook that which sheet contains required data and what's the range
   */
  public getRangeAndSheet(wb: XLSX.WorkBook): SheetInfo {
    let rangeStartRow: number = 0
    let rangeStartCol: number = 0
    let startRow: number = -1
    let endRow: number = -1
    let sheetName: string = ''
    let isComplete = false
    let missingHeaders: string[] = []
    let csvArrayHeaders: string[] = [
      '_____DELETE__THIS__RECORD_____',
      ...this.headerArray
    ]
    let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
    let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
      (map: any, obj: string) => {
        map[obj] = -1
        return map
      },
      {}
    )

    wb.SheetNames.forEach((element: string) => {
      // Checking for required data in each sheet in workbook/
      if (isComplete) {
        return
      }

      missingHeaders = []
      sheetName = element
      const ws: XLSX.WorkSheet = wb.Sheets[sheetName]

      this.data = <AOA>XLSX.utils.sheet_to_json(ws, {
        header: 1,
        blankrows: false,
        defval: ''
      })

      if (this.data.length <= 1) {
        return
      }

      let tempArr: string[] = []
      this.headerArray.forEach(() => tempArr.push(''))
      this.data.push(tempArr)

      let foundHeaders = false

      this.data.forEach((row: any, index: number) => {
        if (isComplete) {
          return
        }

        if (foundHeaders) {
          let isDataEnd = true
          let isPkNull = false

          csvArrayHeadersLower.forEach((x) => {
            const col = csvArrayHeadersMap[x]

            if (row[col] !== '' && row[col] !== undefined) {
              isDataEnd = false
            } else {
              if (this.headerPks.indexOf(x.toUpperCase()) !== -1) {
                isPkNull = true
              }
            }
          })

          if (isDataEnd || isPkNull) {
            endRow = index
            isComplete = true
          } else {
            if (startRow === -1) {
              startRow = index
            }
          }
        } else {
          const rowLowerCase: string[] = row.map((x: any) =>
            x.toString().toLowerCase()
          )

          // If in file there is no delete column, remove it from search of missing.
          // This way delete column will be optional to provide in file
          if (!rowLowerCase.includes('_____delete__this__record_____')) {
            const deleteIndex = csvArrayHeadersLower.indexOf(
              '_____delete__this__record_____'
            )

            if (deleteIndex > -1) csvArrayHeadersLower.splice(deleteIndex, 1)
          }

          foundHeaders = true

          csvArrayHeadersLower.forEach((x) => {
            if (rowLowerCase.indexOf(x) === -1) {
              foundHeaders = false
            }
          })

          let result = []

          result = this.findValidHeaders(
            rowLowerCase,
            csvArrayHeadersLower,
            index,
            sheetName
          )

          if (result[0] === false) {
            foundHeaders = false

            if (result[1].length > 0) {
              result[1].forEach((data: string) => {
                missingHeaders.push(data)
              })
            }
          } else {
            csvArrayHeadersMap = result[1]
          }
        }
      })

      if (isComplete) {
        this.update_sheet_range(ws)
        const worksheetSel = ws['!ref']

        if (worksheetSel) {
          const range = XLSX.utils.decode_range(ws['!ref'] || '')
          rangeStartRow = range.s.r
          rangeStartCol = range.s.c
        }
      }
    })

    // If start row is still -1 that means first row of found range is empty
    if (startRow === -1) isComplete = false

    const returnObj: SheetInfo = {
      foundData: isComplete,
      sheetName,
      startRow,
      endRow,
      csvArrayHeadersMap,
      missingHeaders,
      rangeStartRow,
      rangeStartCol
    }

    return returnObj
  }

  /**
   * When excel is password protected we will display the password promppt for user to type password in.
   * @returns Password user input or undefined if discarded by user
   */
  public promptExcelPassword(): Promise<string | undefined> {
    return new Promise((resolve, reject) => {
      this.filePasswordModal = true

      setTimeout(() => {
        const filePasswordInputElement: any =
          document.querySelector('#filePasswordInput')
        if (filePasswordInputElement) {
          filePasswordInputElement.focus()
          filePasswordInputElement.value = ''
        }
      }, 100)

      this.filePasswordSubject.subscribe((password: string | undefined) => {
        this.fileUnlockError = false

        if (password) {
          resolve(password)
        } else {
          resolve(undefined)
        }
      })
    })
  }

  /**
   * Parses attached file, to be uploaded
   * If attached file is CSV it will be send to backend straight away
   * If attached file is EXCEL it will be displayed in the table, in preview mode
   * @param event file drop event
   * @param dropped whether it's dropped or added by browse button
   */
  public getFileDesc(event: any, dropped: boolean = false) {
    this.excelUploadState = 'Loading'
    this.excelFileParsing = true

    let file
    if (dropped) {
      file = event[0]
    } else {
      file = event.target.files[0]
    }

    this.excelFileReady = false

    this.filename = ''
    let filename = file.name
    this.filename = filename

    this.appendUploadState(`Loading ${filename} into the browser`)

    let foundData = {
      sheet: ''
    }

    let fileType = filename.slice(
      filename.lastIndexOf('.') + 1,
      filename.lastIndexOf('.') + 4
    )

    if (fileType.toLowerCase() === 'xls') {
      let reader: FileReader = new FileReader()
      const self = this
      reader.onload = async (theFile: any) => {
        /* read workbook */
        const bstr = this.toBstr(theFile.target.result)
        let wb: XLSX.WorkBook | undefined = undefined
        let fileUnlocking: boolean = false

        const xlsxOptions: XLSX.ParsingOptions = {
          type: 'binary',
          cellDates: false,
          cellFormula: true,
          cellStyles: true,
          cellNF: false,
          cellText: false
        }

        try {
          wb = XLSX.read(bstr, {
            ...xlsxOptions
          })
        } catch (err: any) {
          if (err.message.toLowerCase().includes('password')) {
            fileUnlocking = true

            while (fileUnlocking) {
              const password = await this.promptExcelPassword()

              if (password) {
                try {
                  wb = XLSX.read(bstr, {
                    ...xlsxOptions,
                    password: password
                  })

                  fileUnlocking = false
                  this.fileUnlockError = false
                } catch (err: any) {
                  this.fileUnlockError = true

                  if (!err.message.toLowerCase().includes('password')) {
                    fileUnlocking = false
                  }
                }
              } else {
                fileUnlocking = false
              }
            }
          } else {
            this.eventService.showAbortModal(
              null,
              err,
              undefined,
              'Error reading file'
            )
          }
        }

        if (!wb) {
          this.excelFileParsing = false
          this.showUploadModal = false
          return
        }

        /* save data */
        let isComplete: boolean = false
        let missingHeaders: string[] = []

        const csvArrayHeaders: string[] = [
          '_____DELETE__THIS__RECORD_____',
          ...this.headerArray
        ]
        let csvArrayHeadersLower = csvArrayHeaders.map((x) => x.toLowerCase())
        let csvArrayHeadersMap = csvArrayHeadersLower.reduce(
          (map: any, obj: string) => {
            map[obj] = -1
            return map
          },
          {}
        )

        let csvArrayData: any[] = []
        const rangeSheetRes: SheetInfo = this.getRangeAndSheet(wb)
        missingHeaders = rangeSheetRes.missingHeaders

        if (rangeSheetRes.foundData) {
          isComplete = true
          csvArrayHeadersMap = rangeSheetRes.csvArrayHeadersMap
          const ws: XLSX.WorkSheet = wb.Sheets[rangeSheetRes.sheetName]

          this.appendUploadState(
            `Table found on sheet ${rangeSheetRes.sheetName} on row ${rangeSheetRes.startRow}`
          )

          let startAddress = ''
          let endAddress = ''

          for (
            let row = rangeSheetRes.startRow;
            row < rangeSheetRes.endRow;
            ++row
          ) {
            const arr: any[] = []

            csvArrayHeadersLower.forEach((x) => {
              const col = csvArrayHeadersMap[x]
              const addr = XLSX.utils.encode_cell({
                r: rangeSheetRes.rangeStartRow + row,
                c: rangeSheetRes.rangeStartCol + col
              })

              if (startAddress === '') startAddress = addr
              endAddress = addr

              let cell

              if (!ws[addr]) {
                cell = { v: '' }
              } else {
                cell = ws[addr]
              }
              arr.push(cell)
            })

            // If we found at least one non empty value it means it is not empty row
            // othervise, it is empty row
            let arrNonEmptyValue = arr.find((x) => x.v !== '')

            if (arrNonEmptyValue) csvArrayData.push(arr)
          }

          this.eventService.showInfoModal(
            'Table Found',
            `Sheet: ${rangeSheetRes.sheetName}\nRange: ${startAddress}:${endAddress}`
          )
        } else {
          missingHeaders = rangeSheetRes.missingHeaders
        }

        if (missingHeaders.length > 0) {
          missingHeaders.sort(function compareSecondColumn(a, b) {
            if (a[1] === b[1]) {
              return 0
            } else {
              return a[1] > b[1] ? -1 : 1
            }
          })
          let abortMsg = missingHeaders
            .map((x) => x[0])
            .slice(0, 5)
            .join('\n')

          this.eventService.showAbortModal(null, abortMsg)

          setTimeout(() => {
            this.filename = ''
          })

          this.excelFileParsing = false
          this.uploader.queue.pop()
          return
        }

        // If first row is empty, that means no data has been found
        if (csvArrayData.length === 0 || csvArrayData[0].length === 0) {
          let abortMsg = 'No relevant data found in File !'
          this.eventService.showAbortModal(null, abortMsg)

          setTimeout(() => {
            this.filename = ''
          })

          this.excelFileParsing = false
          this.uploader.queue.pop()
          return
        }

        if (
          this.dateTimeHeaders.length > 0 ||
          this.dateHeaders.length > 0 ||
          this.timeHeaders.length > 0
        ) {
          csvArrayData = this.updateDateTimeCols(csvArrayHeaders, csvArrayData)
        }

        if (this.xlRules.length > 0) {
          csvArrayData = this.updateXLRuleCols(csvArrayHeaders, csvArrayData)
        }

        if (!isComplete) {
          if (missingHeaders.length === 0) {
            let abortMsg = 'No relevant data found in File !'
            this.eventService.showAbortModal(null, abortMsg)

            setTimeout(() => {
              this.filename = ''
            })
          } else {
            missingHeaders.sort(function compareSecondColumn(a, b) {
              if (a[1] === b[1]) {
                return 0
              } else {
                return a[1] > b[1] ? -1 : 1
              }
            })
            let abortMsg = missingHeaders
              .map((x) => x[0])
              .slice(0, 5)
              .join('\n')

            this.eventService.showAbortModal(null, abortMsg)
          }
          this.excelFileParsing = false
          this.uploader.queue.pop()
          return
        } else {
          this.headerShow = csvArrayHeaders
          csvArrayData = csvArrayData.map((row: any) =>
            row.map((col: any) => (col.t === 'n' ? col.v : col.w))
          )

          csvArrayData = csvArrayData.map((row: any) => {
            return row.map((col: any, index: number) => {
              if (!col && col !== 0) col = ''

              /**
               * Keeping this for the reference
               * Code below used to convert JSON to CSV
               * now the XLSX is converting to CSV
               */
              // if (isNaN(col)) {
              //   // Match and replace the double quotes, ignore the first and last char
              //   // in case they are double quotes already
              //   col = col.replace(/(?<!^)"(?!$)/g, '""')

              //   if (col.search(/,/g) > -1 ||
              //       col.search(/\r|\n/g) > -1
              //   ) {
              //     // Missing quotes at the end
              //     if (col.search(/"$/g) < 0) {
              //       col = col + '"' // So we add them
              //     }

              //     // Missing quotes at the start
              //     if (col.search(/^"/g) < 0) {
              //       col = '"' + col // So we add them
              //     }
              //   }
              // }

              const colName = this.headerShow[index]
              const colRule = this.dcValidator?.getRule(colName)

              if (colRule?.type === 'numeric') {
                if (isSpecialMissing(col) && !col.includes('.')) col = '.' + col
              }

              return col
            })
          })

          this.data = csvArrayData

          // Apply licence rows limitation if exists, it is only affecting data
          // which will be send to SAS
          const strippedCsvArrayData = csvArrayData.slice(
            0,
            this.licenceState.value.submit_rows_limit
          )
          // To submit to sas service, we need clean version of CSV of file
          // attached. XLSX will do the parsing and heavy lifting
          // First we create worksheet of json (data we extracted)
          let ws = XLSX.utils.json_to_sheet(strippedCsvArrayData, {
            skipHeader: true
          })
          // create CSV to be uploaded from worksheet
          let csvContentClean = XLSX.utils.sheet_to_csv(ws)
          // Prepend headers
          csvContentClean = csvArrayHeaders.join(',') + '\n' + csvContentClean

          if (this.encoding === 'WLATIN1') {
            let encoded = iconv.decode(Buffer.from(csvContentClean), 'CP-1252')
            let blob = new Blob([encoded], { type: 'application/csv' })
            let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
            this.uploader.addToQueue([newCSVFile])
          } else {
            let blob = new Blob([csvContentClean], { type: 'application/csv' })
            let newCSVFile: File = this.blobToFile(blob, this.filename + '.csv')
            this.uploader.addToQueue([newCSVFile])
          }

          this.excelFileReady = true
        }

        if (this.data.length === 0) {
          this.showUploadModal = false
          this.uploadPreview = false
          this.excelFileParsing = false

          this.eventService.showAbortModal(
            null,
            `Table in the file is empty. Data found on sheet: ${foundData.sheet}`
          )

          return
        }

        this.excelFileReady = true
        this.getPendingExcelPreview()

        return
      }
      reader.readAsArrayBuffer(file)
    } else if (fileType.toLowerCase() === 'csv') {
      if (this.licenceState.value.submit_rows_limit !== Infinity) {
        this.eventService.showInfoModal(
          'Notice',
          'Excel files only. To unlock CSV uploads, please contact support@datacontroller.io'
        )
        this.excelFileReady = true
        this.excelFileParsing = false
        this.uploader.queue.pop()

        return
      }

      if (this.encoding === 'WLATIN1') {
        let reader = new FileReader()
        const self = this
        // Closure to capture the file information.
        reader.onload = (theFile: any) => {
          let encoded = iconv.decode(
            Buffer.from(theFile.target.result),
            'CP-1252'
          )
          let blob = new Blob([encoded], { type: fileType })
          let encodedFile: File = this.blobToFile(blob, this.filename)
          this.uploader.queue.pop()
          this.uploader.addToQueue([encodedFile])
          this.excelFileReady = true
        }
        this.excelFileReady = true
        this.excelFileParsing = false
        reader.readAsArrayBuffer(file)
        this.getFile()
      } else {
        this.excelFileReady = true
        this.excelFileParsing = false
        this.getFile()
      }
    } else {
      let abortMsg =
        'Invalid file type "<b>' +
        this.filename +
        '</b>". Please upload csv or excel file.'
      this.eventService.showAbortModal(null, abortMsg)

      this.excelFileReady = true
      this.excelFileParsing = false
      this.uploader.queue.pop()
    }
  }

  /**
   * Submits attached excel file that is in preview mode
   */
  public submitExcel() {
    if (this.licenceState.value.submit_rows_limit !== Infinity) {
      this.submitLimitNotice = true
      return
    }

    this.getFile()
  }

  /**
   * This method will run validations and upload all of the pending files that are in the uploader queue
   */
  public getFile() {
    if (this.checkInvalid()) {
      this.eventService.showAbortModal(null, 'Invalid values are present.')
      return
    }

    this.validatePrimaryKeys()

    if (this.duplicatePkIndexes.length !== 0) {
      this.pkDups = true
      this.submit = false
      return
    } else {
      this.pkDups = false
    }

    this.uploadLoading = true
    let filesToUpload: UploadFile[] = []

    for (const file of this.uploader.queue) {
      filesToUpload.push({
        file: file,
        fileName: file.name
      })
    }

    this.sasService
      .uploadFile(this.uploadUrl, filesToUpload, { table: this.libds })
      .then(
        (res: any) => {
          if (typeof res.sasjsAbort === 'undefined') {
            if (typeof res.sasparams === 'undefined') {
              return
            } else {
              this.uploadLoading = false
              let params = res.sasparams[0]
              this.successEnable = true
              this.tableId = params.DSID
              this.router.navigateByUrl('/stage/' + this.tableId)
            }
          } else {
            // handle succesfull response
            const abortRes = res
            const abortMsg = abortRes.sasjsAbort[0].MSG
            const macMsg = abortRes.sasjsAbort[0].MAC

            this.uploadLoading = false
            this.filename = ''
            if (this.fileUploadInputCompList.first) {
              //clear the attached file to input
              this.fileUploadInputCompList.first.nativeElement.value = ''
            }
            this.uploader.queue = []
            this.eventService.showAbortModal('', abortMsg, {
              SYSWARNINGTEXT: abortRes.SYSWARNINGTEXT,
              SYSERRORTEXT: abortRes.SYSERRORTEXT,
              MAC: macMsg
            })
          }
        },
        (err: any) => {
          this.uploadLoading = false
          if (this.fileUploadInputCompList.first) {
            //clear the attached file to input
            this.fileUploadInputCompList.first.nativeElement.value = ''
          }
          this.uploader.queue = []
          this.eventService.catchResponseError('file upload', err)
        }
      )
  }

  /**
   * After excel file is attached and parsed, this function will display it's content in the HOT table in read only mode
   */
  public getPendingExcelPreview() {
    this.queryTextSaved = this.queryText
    this.queryText = ''

    this.excelUploadState = 'Parsing'

    this.toggleHotPlugin('contextMenu', false)

    let previewDatasource: any[] = []

    this.data.map((item) => {
      let itemObject: any = {}

      this.headerShow.map((header: any, index: number) => {
        itemObject[header] = item[index]
      })

      // If Delete? column is not set in the file, we set it to NO
      if (!itemObject['_____DELETE__THIS__RECORD_____'])
        itemObject['_____DELETE__THIS__RECORD_____'] = 'No'

      previewDatasource.push(itemObject)
    })

    this.dataSourceUnchanged = this.helperService.deepClone(this.dataSource)

    this.dataSource = previewDatasource
    this.hotTable.data = previewDatasource

    const hot = this.hotInstance

    this.excelUploadState = 'Validating-HOT'

    hot.updateSettings(
      {
        data: this.dataSource,
        maxRows: Infinity
      },
      false
    )
    hot.render()

    this.appendUploadState(`Validating rows`)

    hot.validateCells(() => {
      this.showUploadModal = false
      this.uploadPreview = true

      this.excelFileParsing = false
      this.excelUploadState = null
    })
  }

  /**
   * Drops the attached excel file
   * @param discardData wheter to discard data parsed from the file or to keep it in the table after dropping a attached excel file
   */
  public discardPendingExcel(discardData?: boolean) {
    this.hotInstance.updateSettings({
      maxRows: this.licenceState.value.editor_rows_allowed
    })

    if (discardData) this.cancelEdit()

    if (this.fileUploadInputCompList.first) {
      this.fileUploadInputCompList.first.nativeElement.value = ''
    }

    this.uploadPreview = false
    this.excelFileReady = false
    this.uploader.queue = []

    if (!isNaN(parseInt(this.router.url.split('/').pop() || ''))) {
      if (this.queryTextSaved.length > 0) {
        this.queryText = this.queryTextSaved
        this.queryTextSaved = ''
      }
    }
  }

  /**
   * Drops attached excel file, keeps it's data in the DC table
   * User can now edit the table and submit. Witout the file present.
   */
  public previewTableEditConfirm() {
    this.discardPendingExcel()
    this.convertToCorrectTypes(this.dataSource)
    this.editTable(true)
  }

  private appendUploadState(state: string, replaceLast: boolean = false) {
    this.cdf.detectChanges()

    if (this.uploadStaterCompList.first) {
      if (replaceLast) {
        this.uploadStaterCompList.first.replaceLastState(state)
      } else {
        this.uploadStaterCompList.first.appendState(state)
      }
    }
  }

  findValidHeaders(
    row: string[],
    headers: string[],
    rowNumber: number,
    tabName: string
  ): Array<any> {
    let headersFound = false
    let missingErrorArray = []
    let j = 0

    while (j < row.length) {
      if (headersFound) {
        // return;
      } else {
        if (headers.indexOf(row[j]) !== -1) {
          let breakIndex
          let rowStart = 0
          let rowEnd = 0
          let arrStart = 0
          let foundHeadersArray: string[] = []
          let spaceBreak = false

          for (let i = j; i < row.length; i++) {
            if (
              row[i] === '' ||
              (foundHeadersArray.indexOf(row[i]) !== -1 &&
                this.isColHeader(row[i]))
            ) {
              if (row[i] === '') {
                spaceBreak = true
              }

              breakIndex = i
              break
            } else {
              foundHeadersArray.push(row[i])
            }
          }

          let tempArray: string[] = []

          if (breakIndex !== undefined) {
            tempArray = row.slice(j, breakIndex)
            arrStart = j
            rowEnd = breakIndex

            if (spaceBreak) {
              rowStart = j
              j = breakIndex
            } else {
              rowStart = j
              j = breakIndex - 1
            }
          } else {
            tempArray = row.slice(j)
            rowStart = j
            arrStart = j
            rowEnd = row.length
            j = row.length
          }

          let foundHeaders = true

          //We check if there are missing headers
          headers.forEach((x) => {
            if (tempArray.indexOf(x) === -1) {
              foundHeaders = false
            }
          })

          if (foundHeaders) {
            headersFound = true

            let mapHeaders: any[] = headers

            let csvArrayHeadersMap = mapHeaders.reduce(function (map, obj) {
              map[obj] = -1
              return map
            }, {})

            let temp = row.slice(rowStart, rowEnd)

            headers.forEach((x) => {
              csvArrayHeadersMap[x] = temp.indexOf(x) + rowStart
            })

            return [true, csvArrayHeadersMap]
          } else {
            let missingHeaders = getMissingHeaders(tempArray, headers)

            let missingMessage = '<b>TAB(' + tabName + ')</b>'
            missingErrorArray.push([
              missingMessage +
                ' - ' +
                missingHeaders[1].join(',') +
                ' ( missing ' +
                missingHeaders[0].join(',') +
                ' )',
              missingHeaders[1].length
            ])
          }
        }
      }
      j++
    }
    return [false, missingErrorArray]
  }

  isColPk(col: string) {
    return this.headerPks.indexOf(col) > -1
  }

  isReadonlyCol(col: string | number) {
    const colRules = this.dcValidator?.getRule(col)

    return colRules?.readOnly
  }

  isColHeader(col: string) {
    return this.headerArray.indexOf(col.toUpperCase()) > -1
  }

  removeQuery() {
    this.sasStoreService.removeClause()
  }

  updateDateTimeCols(headers: any, data: any) {
    if (this.dateHeaders.length > 0) {
      let dateCols: number[] = []
      this.dateHeaders.forEach((element) => {
        if (headers.indexOf(element) !== -1) {
          dateCols.push(headers.indexOf(element))
        }
      })
      data.forEach((row: any[]) => {
        dateCols.forEach((element) => {
          const obj = row[element]
          if (isStringNumber(obj.v)) {
            const date = excelDateToJSDate(Number(obj.v))

            obj.v =
              date.getFullYear() +
              '-' +
              ('0' + (date.getMonth() + 1)).slice(-2) +
              '-' +
              ('0' + date.getDate()).slice(-2)
          } else {
            if (obj && obj.v && obj.v.toString().indexOf(':') === -1) {
              const date = new Date(obj.v)
              if (date.toUTCString() !== 'Invalid Date') {
                obj.v = dateFormat(date)
              }
            }
          }
          row[element] = obj
        })
      })
    }
    if (this.timeHeaders.length > 0) {
      let timeCols: number[] = []
      this.timeHeaders.forEach((element) => {
        if (headers.indexOf(element) !== -1) {
          timeCols.push(headers.indexOf(element))
        }
      })
      data.forEach((row: any[]) => {
        timeCols.forEach((element) => {
          const obj = row[element]
          if (
            isStringNumber(obj.v) ||
            isStringDecimal(obj.v) ||
            obj.v.includes('E-')
          ) {
            const date = excelDateToJSDate(Number(obj.v))

            obj.v = dateToUtcTime(date)
          }
          row[element] = obj
        })
      })
    }
    if (this.dateTimeHeaders.length > 0) {
      let dateTimeCols: number[] = []
      this.dateTimeHeaders.forEach((element) => {
        if (headers.indexOf(element) !== -1) {
          dateTimeCols.push(headers.indexOf(element))
        }
      })
      data.forEach((row: any[]) => {
        dateTimeCols.forEach((element) => {
          const obj = row[element]
          if (isStringNumber(obj.v) || isStringDecimal(obj.v)) {
            let date = excelDateToJSDate(Number(obj.v))
            obj.v = dateFormat(date) + ' ' + dateToUtcTime(date)
          } else {
            if (obj.v.indexOf(' ') === -1 && obj.v.indexOf(':') !== -1) {
              let str = obj.v.substring(0, obj.v.indexOf(':'))
              str = str + ' ' + obj.v.substring(obj.v.indexOf(':') + 1)
              obj.v = str
            }
            let date = new Date(obj.v)
            if (date.toUTCString() !== 'Invalid Date') {
              obj.v = dateFormat(date) + ' ' + dateToTime(date)
            }
          }
          row[element] = obj
        })
      })
    }
    return data
  }

  updateXLRuleCols(headers: any, data: any) {
    if (this.xlRules.length > 0) {
      const xlRuleCols: any = []
      this.xlRules.forEach((element: any) => {
        if (headers.indexOf(element.XL_COLUMN) !== -1) {
          element['index'] = headers.indexOf(element.XL_COLUMN)
          xlRuleCols.push(element)
        }
      })
      data.forEach((row: any[]) => {
        xlRuleCols.forEach((element: any) => {
          const obj = row[element.index]
          if (element.XL_RULE === 'FORMULA') {
            if ('f' in obj) {
              if (obj['t'] === 'n') {
                obj['v'] = '=' + obj['f']
              } else {
                obj['w'] = '=' + obj['f']
              }
            }
          }
          row[element] = obj
        })
      })
    }
    return data
  }

  private blobToFile(theBlob: Blob, fileName: string): File {
    const b: any = theBlob
    b.lastModifiedDate = new Date()
    b.name = fileName
    return b as File
  }

  public toBstr(res: any) {
    let bytes = new Uint8Array(res)
    let binary = ''
    let length = bytes.byteLength
    for (let i = 0; i < length; i++) {
      binary += String.fromCharCode(bytes[i])
    }
    return binary
  }

  async sendClause() {
    this.submitLoading = true
    let nullVariableArr = []
    let emptyVariablesArr = []

    // to check number of empty clauses
    if (typeof this.clauses === 'undefined') {
      this.nullVariables = true
      this.submitLoading = false
      return
    } else {
      let query = this.clauses.queryObj

      if (query[0].elements.length < 1) {
        // Clear cached filtering data
        if (globals.rootParam === 'home' || globals.rootParam === 'editor') {
          globals.editor.filter.clauses = []
          globals.editor.filter.query = []
          globals.editor.filter.groupLogic = ''
        }

        // Reset filtering
        this.router.navigate(['/editor/' + this.libds], {
          queryParamsHandling: 'preserve'
        })

        return
      }

      for (let index = 0; index < query.length; index++) {
        const el = query[index].elements
        nullVariableArr = el.filter(function (item: any) {
          return item.variable === null
        })
        if (nullVariableArr.length) {
          emptyVariablesArr.push(el)
        }
      }
    }

    if (emptyVariablesArr.length) {
      this.nullVariables = true
      this.submitLoading = false
      return
    } else {
      try {
        if (this.clauses !== undefined && this.libds) {
          const filterQuery: FilterQuery = {
            groupLogic: this.clauses.groupLogic,
            filterGroups: []
          }
          this.clauses.queryObj.forEach((group: any) => {
            const filterGroup: FilterGroup = {
              filterClauses: []
            }
            group.elements.forEach((clause: any) => {
              filterGroup.filterClauses.push(
                this.helperService.deepClone(clause)
              )
            })
            filterGroup.clauseLogic = group.clauseLogic
            filterQuery.filterGroups.push(
              this.helperService.deepClone(filterGroup)
            )
          })

          const filterQueryClauseTable =
            this.sasStoreService.createFilterQueryTable(filterQuery)

          await this.sasStoreService
            .saveQuery(this.libds, filterQueryClauseTable)
            .then((res: any) => {
              const id = res.result[0].FILTER_RK
              const table = res.result[0].FILTER_TABLE

              this.queryFilter = { id: id, table: table }

              this.router
                .navigate(['/'], {
                  skipLocationChange: true,
                  queryParamsHandling: 'preserve'
                })
                .then(() =>
                  this.router.navigate(
                    [
                      '/editor/' +
                        this.queryFilter.table +
                        '/' +
                        this.queryFilter.id
                    ],
                    {
                      queryParamsHandling: 'preserve'
                    }
                  )
                )

              this.filter = false
            })
            .catch((err: any) => {
              this.submitLoading = false
            })
        }
      } catch (error: any) {
        this.queryErr = true
        this.submitLoading = false
        this.queryErrMessage = error
      }
    }
  }

  public openQb() {
    if (this.libds) {
      // this.libTab = this.libds;
      this.filter = true
      this.cdf.detectChanges()

      this.submitLoading = false
      this.sasStoreService.setQueryVariables(this.libds, this.cols)
    }
  }

  editTable(previewEdit?: boolean, newRow?: boolean) {
    this.toggleHotPlugin('contextMenu', true)

    const hot = this.hotInstance

    let columnSorting = hot.getPlugin('multiColumnSorting')
    let columnSortConfig = columnSorting.getSortConfig()
    let sortConfigs = Array.isArray(columnSortConfig)
      ? columnSortConfig
      : [columnSortConfig]

    setTimeout(() => {
      if (!previewEdit) {
        this.dataSourceUnchanged = this.helperService.deepClone(this.dataSource)
        if (newRow) {
          this.dataSourceUnchanged.pop()
        }
      }
      this.hotTable.readOnly = false
      this.hotTable.data = this.dataSource

      hot.updateSettings(
        {
          readOnly: this.hotTable.readOnly
        },
        false
      )

      hot.render()

      for (let sortConfig of sortConfigs) {
        columnSorting.sort(sortConfig)
      }

      this.reSetCellValidationValues()
    }, 0)
  }

  convertToCorrectTypes(dataSource: any) {
    for (let row of dataSource) {
      for (let colKey in row) {
        let colSpecs = this.cols.find((x: any) => x.NAME === colKey)

        if (colSpecs) {
          if (
            row[colKey] !== '' &&
            colSpecs.TYPE === 'num' &&
            !colSpecs.DDTYPE.includes('TIME') &&
            !colSpecs.DDTYPE.includes('DATE')
          )
            row[colKey] = parseInt(row[colKey])
        }
      }
    }
  }

  cancelEdit() {
    this.toggleHotPlugin('contextMenu', false)

    this.cellValidationSource = []

    const hot = this.hotInstance
    let columnSorting = hot.getPlugin('multiColumnSorting')
    let columnSortConfig = columnSorting.getSortConfig()
    let sortConfigs = Array.isArray(columnSortConfig)
      ? columnSortConfig
      : [columnSortConfig]

    if (this.dataSourceUnchanged) {
      this.dataSource = this.helperService.deepClone(this.dataSourceUnchanged)
    }

    this.hotTable.data = this.dataSource
    this.hotTable.readOnly = true

    hot.updateSettings(
      {
        readOnly: this.hotTable.readOnly,
        data: this.dataSource
      },
      false
    )

    hot.validateRows(this.modifedRowsIndexes)
    // this.editRecordListeners();
    for (let sortConfig of sortConfigs) {
      columnSorting.sort(sortConfig)
    }

    this.checkRowLimit()
  }

  timesClicked: number = 0
  public hotClicked() {
    if (this.timesClicked === 1 && this.hotTable.readOnly) {
      this.editTable()
    }

    if (this.timesClicked === 0) {
      this.timesClicked++

      setTimeout(() => {
        this.timesClicked = 0
      }, 200)
    }
  }

  public cleanExceed() {
    this.exceedCells = []
  }

  public approversToggle() {
    this.showApprovers = !this.showApprovers
  }

  public addRow() {
    this.addingNewRow = true

    setTimeout(() => {
      const hot = this.hotInstance

      let dsInsertIndex = this.dataSource.length
      hot.alter('insert_row_below', dsInsertIndex, 1)
      hot.updateSettings({ data: this.dataSource }, false)
      hot.selectCell(this.dataSource.length - 1, 0)
      hot.render()

      if (this.dataSource[dsInsertIndex]) {
        this.dataSource[dsInsertIndex]['noLinkOption'] = true
      }

      this.addingNewRow = false

      this.reSetCellValidationValues()
    })
  }

  public cancelSubmit() {
    this.dataSource = this.helperService.deepClone(this.dataSourceBeforeSubmit)
    this.dataSourceBeforeSubmit = []
    this.hotTable.data = this.dataSource

    const hot = this.hotInstance
    hot.updateSettings(
      {
        data: this.dataSource,
        colHeaders: this.headerColumns,
        columns: this.cellValidation,
        modifyColWidth: function (width: number, col: number) {
          if (col === 0) {
            return 60
          }
          if (width > 500) return 500
          else return width
        }
      },
      false
    )

    hot.selectCell(0, 0)
    hot.render()
    hot.validateRows(this.modifedRowsIndexes)

    this.reSetCellValidationValues()
  }

  public getRowsSubmittingCount() {
    if (this.sasService.getSasjsConfig().debug) {
      this.loggerService.log(this.dataSource)
      this.loggerService.log(this.dataSourceUnchanged)
    }

    let rowsUpdated = 0
    let rowsDeleted = 0
    let rowsAdded = 0
    this.modifedRowsIndexes = []
    this.dataModified = []

    for (let i = 0; i < this.dataSource.length; i++) {
      let dataRow = this.helperService.deepClone(this.dataSource[i])

      if (dataRow._____DELETE__THIS__RECORD_____ === 'Yes') {
        this.dataModified.push(dataRow)
        rowsDeleted++
      } else {
        let dataRowUnchanged = this.dataSourceUnchanged.find((row: any) => {
          for (let pkCol of this.headerPks) {
            if (row[pkCol] !== dataRow[pkCol]) {
              return false
            }
          }

          return true
        })

        if (dataRowUnchanged) {
          if (JSON.stringify(dataRow) !== JSON.stringify(dataRowUnchanged)) {
            this.dataModified.push(dataRow)
            this.modifedRowsIndexes.push(i)
            rowsUpdated++
          }
        } else {
          this.dataModified.push(dataRow)
          this.modifedRowsIndexes.push(i)
          rowsAdded++
        }
      }
    }

    this.rowsChanged = {
      rowsUpdated,
      rowsDeleted,
      rowsAdded
    }
  }

  public validatePrimaryKeys() {
    const hot = this.hotInstance

    let myTable = hot.getData()
    this.pkFields = []
    for (let index = 0; index < myTable.length; index++) {
      let pkRow = ''
      for (let ind = 1; ind < this.readOnlyFields + 1; ind++) {
        pkRow = pkRow + '|' + myTable[index][ind]
      }
      this.pkFields.push(pkRow)
    }

    let results = []
    let rows = this.dataSource.length

    for (let j = 0; j < this.pkFields.length; j++) {
      for (let i = 0; i < this.pkFields.length; i++) {
        if (this.pkFields[j] === this.pkFields[i] && i !== j) {
          results.push(i)
        }
      }
    }

    if (this.pkFields.length > rows) {
      for (let n = rows; n < this.pkFields.length; n++) {
        for (let p = rows; p < this.pkFields.length; p++) {
          if (n < p && this.pkFields[n] === this.pkFields[p]) {
            results.push(p)
          }
        }
      }
    }

    let cellMeta
    for (let k = 0; k < results.length; k++) {
      for (let index = 1; index < this.readOnlyFields + 1; index++) {
        cellMeta = hot.getCellMeta(results[k], index)
        cellMeta.valid = false
        cellMeta.dupKey = true
        hot.render()
      }
    }

    this.duplicatePkIndexes = [...new Set(results.sort())]
  }

  /**
   * After any change or update to the hot datasource we lose cell validation values.
   * This function is called in those places, to update all cells with values if existing.
   * Note that was discussed:
   * Rows with same data does not have arrows until you click on them (arrows gets lost after addRow)
   * That is because this function resets the values for the found hashes, and if multiple rows have same data,
   * hash table contains only first row that is hashed, so others don't get arrow re-set
   *
   * @param specificRowForceValue re-set will apply force values only to this row. That is used in cases
   * when we don't want to re-set force values of every row in the table
   */
  public reSetCellValidationValues(
    setForcedValues: boolean = false,
    specificRowForceValue?: number
  ) {
    const hot = this.hotInstance

    for (let entry of this.cellValidationSource) {
      const colSource = entry.values.map(
        (el: any) => el[this.cellValidationFields.RAW_VALUE]
      )

      hot.batch(() => {
        const cellMeta = hot.getCellMeta(entry.row, entry.col)
        const cellRule = this.dcValidator?.getRule(
          (cellMeta.data as string) || ''
        )
        let cellSource: string[] | number[] | undefined

        if (cellRule) {
          cellSource = this.dcValidator?.getDqDropdownSource(cellRule)
        }

        if (!cellSource) cellSource = []

        const combinedSource = [...new Set([...cellSource, ...colSource])]

        this.currentEditRecordValidator?.updateRule(entry.col, {
          source: combinedSource
        })

        hot.setCellMeta(entry.row, entry.col, 'source', combinedSource)

        if (entry.values.length > 0) {
          hot.setCellMeta(entry.row, entry.col, 'renderer', 'autocomplete')
          hot.setCellMeta(entry.row, entry.col, 'editor', 'autocomplete.custom')
          hot.setCellMeta(entry.row, entry.col, 'strict', entry.strict)
          hot.setCellMeta(entry.row, entry.col, 'filter', false)

          this.currentEditRecordValidator?.updateRule(entry.col, {
            renderer: 'autocomplete',
            editor: 'autocomplete.custom',
            strict: entry.strict,
            filter: false
          })
        }

        this.reSetExtendedCellValidationValues(
          entry,
          undefined,
          setForcedValues,
          specificRowForceValue
        )

        hot.render()
      })
    }
  }

  public reSetExtendedCellValidationValues(
    cellValidationEntry?: CellValidationSource,
    row?: number,
    setForcedValues: boolean = false,
    specificRowForceValue?: number
  ) {
    const hot = this.hotInstance

    if (cellValidationEntry) {
      if (!row) row = cellValidationEntry.row

      const extendedValuesObject =
        this.getExtendedValuesByCellValue(cellValidationEntry)

      this.setExtendedValuesToCells(
        cellValidationEntry,
        row,
        extendedValuesObject,
        setForcedValues,
        specificRowForceValue
      )

      return
    }

    for (let entry of this.cellValidationSource) {
      const extendedValuesObject = this.getExtendedValuesByCellValue(entry)

      this.setExtendedValuesToCells(
        entry,
        entry.row,
        extendedValuesObject,
        setForcedValues,
        specificRowForceValue
      )
    }
  }

  private setExtendedValuesToCells(
    cellValidationEntry: CellValidationSource,
    row: number,
    extendedValues: DynamicExtendedCellValidation[],
    setForcedValues: boolean = false,
    specificRowForceValue?: number
  ) {
    const hot = this.hotInstance

    let uniqueCells: any[] = []

    for (let element of extendedValues) {
      if (uniqueCells.indexOf(element.EXTRA_COL_NAME) < 0)
        uniqueCells.push(element.EXTRA_COL_NAME)
    }

    for (let cell of uniqueCells) {
      const valuesForCol = extendedValues.filter(
        (x) => x.EXTRA_COL_NAME === cell
      )
      let colSource: any = valuesForCol.map(
        (el: DynamicExtendedCellValidation) =>
          el.DISPLAY_TYPE === 'C' ? el.RAW_VALUE_CHAR : el.RAW_VALUE_NUM
      )
      const cellCol = hot.propToCol(cell)
      const dynamicValidationEl =
        this.disabledBasicDynamicCellValidationMap.find(
          (x) => x.row === row && x.col === cellCol
        )

      if (!dynamicValidationEl) {
        this.disabledBasicDynamicCellValidationMap.push({
          row,
          col: cellCol,
          active: false
        })
      }

      hot.setCellMeta(row, cellCol, 'renderer', 'autocomplete')
      hot.setCellMeta(row, cellCol, 'editor', 'autocomplete.custom')
      hot.setCellMeta(row, cellCol, 'strict', cellValidationEntry.strict)
      hot.setCellMeta(row, cellCol, 'filter', false)

      this.currentEditRecordValidator?.updateRule(cellCol, {
        renderer: 'autocomplete',
        editor: 'autocomplete.custom',
        strict: cellValidationEntry.strict,
        filter: false
      })

      const cellMeta = hot.getCellMeta(row, cellCol)

      const cellRule = this.dcValidator?.getRule(
        (cellMeta.data as string) || ''
      )
      let cellSource: string[] | number[] | undefined

      if (cellRule) {
        cellSource = this.dcValidator?.getDqDropdownSource(cellRule)
      }

      if (!cellSource) cellSource = []

      if (cellRule?.type === 'numeric') {
        cellSource = this.helperService.convertArrayValues(
          cellSource,
          'number'
        ) as number[]
        colSource = this.helperService.convertArrayValues(
          colSource,
          'number'
        ) as number[]
      } else {
        cellSource = this.helperService.convertArrayValues(
          cellSource,
          'string'
        ) as string[]
        colSource = this.helperService.convertArrayValues(
          colSource,
          'string'
        ) as string[]
      }

      const combinedSource = [...new Set([...cellSource, ...colSource])]

      hot.setCellMeta(row, cellCol, 'source', combinedSource)

      this.currentEditRecordValidator?.updateRule(cellCol, {
        source: combinedSource
      })

      if (setForcedValues) {
        if (specificRowForceValue && specificRowForceValue !== row) {
          return
        }

        const forceValue = valuesForCol.find(
          (x: DynamicExtendedCellValidation) => x.FORCE_FLAG === 1
        )

        if (forceValue) {
          // Adding timeout here makes forced values cell to re-validate itself
          setTimeout(() => {
            hot.setDataAtCell(
              row,
              cellCol,
              forceValue.DISPLAY_TYPE === 'C'
                ? forceValue.RAW_VALUE_CHAR
                : forceValue.RAW_VALUE_NUM,
              'force_cell_validation_value'
            )

            if (this.currentEditRecordIndex === row) {
              this.dataSource[this.currentEditRecordIndex][cell] =
                forceValue.DISPLAY_TYPE === 'C'
                  ? forceValue.RAW_VALUE_CHAR
                  : forceValue.RAW_VALUE_NUM
            }
          })
        }
      }
    }
  }

  /**
   * Parses values of extended cell validation for the given dynamic cell validation entry
   * @param cellValidationEntry stored dynamic cell validation entry from which to parse extended validation values
   * @param rowOverride if not provided, row that is used is row found in `cellValidationEntry`. This is needed when for example we change the w in hot
   * we need to get `cellValue` from that row and not from hashed cell validation source.
   * @returns extended values object
   */
  private getExtendedValuesByCellValue(
    cellValidationEntry: CellValidationSource,
    rowOverride?: number
  ): DynamicExtendedCellValidation[] {
    const hot = this.hotInstance

    const cellValue = hot.getDataAtCell(
      rowOverride ? rowOverride : cellValidationEntry.row,
      cellValidationEntry.col
    )
    const valueIndex = (cellValidationEntry.values.find(
      (x) => x[this.cellValidationFields.RAW_VALUE] === cellValue
    ) || [])[this.cellValidationFields.DISPLAY_INDEX]

    const filteredValues = cellValidationEntry.extended_values?.filter(
      (x) => x[0] === valueIndex
    )
    const prepObj = this.helperService.deepClone(
      this.extendedCellValidationFields
    )

    const extendedValuesObject = []

    for (let extendedValue of filteredValues || []) {
      let tempObj: any = {}

      for (let key of Object.keys(prepObj)) {
        tempObj[key] = extendedValue[prepObj[key]]
      }

      extendedValuesObject.push(tempObj)
    }

    return extendedValuesObject
  }

  public checkSave() {
    this.getRowsSubmittingCount()

    if (
      this.rowsChanged.rowsAdded === 0 &&
      this.rowsChanged.rowsUpdated === 0 &&
      this.rowsChanged.rowsDeleted === 0
    ) {
      this.badEditTitle = 'No changes to submit'
      this.badEditCause = 'Please modify some values and try again.'
      this.badEdit = true
      return
    }

    const hot = this.hotInstance

    this.dataSourceBeforeSubmit = this.helperService.deepClone(this.dataSource)

    for (let i = 0; i < this.dataSource.length; i++) {
      delete this.dataSource[i].noLinkOption
    }

    hot.updateSettings(
      {
        data: this.dataSource,
        colHeaders: this.headerColumns,
        columns: this.cellValidation,
        modifyColWidth: function (width: number, col: number) {
          if (width > 500) return 500
          else return width
        }
      },
      false
    )

    this.reSetCellValidationValues()

    EditorComponent.cnt = 0
    EditorComponent.nonPkCnt = 0
    // this.saveLoading = true;

    /**
     * Below code should be analized, not sure what is the purpose of exceedCells
     */
    let myTableData = hot.getData()

    // If the last row is empty, remove it before validation
    if (myTableData.length > 1 && hot.isEmptyRow(myTableData.length - 1)) {
      hot.alter('remove_row', myTableData.length - 1)
    }

    this.validatePrimaryKeys()

    if (this.duplicatePkIndexes.length !== 0) {
      this.pkDups = true
      this.submit = false
      this.cancelSubmit()
      return
    } else {
      this.pkDups = false
    }

    hot.validateRows(this.modifedRowsIndexes, () => {
      if (this.checkInvalid()) {
        let abortMsg = 'Invalid Values are Present'

        this.eventService.showInfoModal('Validation error', abortMsg)

        return
      }

      this.submit = true
      this.validationDone = 1

      setTimeout(() => {
        let txt: any = document.getElementById('formFields_8')
        if (txt) txt.focus()
      }, 200)
    })

    // let cnt = 0;
    // hot.addHook("afterValidate", () => {
    //   this.updateSoftSelectColumns(true);
    //   cnt++;
    //   if (cnt === long) {
    //     this.validationDone = 1;
    //   }
    // });
  }

  public async saveTable(data: any) {
    const hot = this.hotInstance
    let hotData = hot.getData()

    data = data.filter((dataRow: any) => {
      let elModified = this.dataModified.find((row) => {
        for (let pkCol of this.headerPks) {
          if (row[pkCol] !== dataRow[pkCol]) {
            return false
          }
        }

        return true
      })

      return !!elModified
    })

    data = data.map((row: any) => {
      let deleteColValue = row['_____DELETE__THIS__RECORD_____']

      delete row['_____DELETE__THIS__RECORD_____']
      row['_____DELETE__THIS__RECORD_____'] = deleteColValue

      // If cell is numeric and value is dot `.` we change it to `null`
      Object.keys(row).map((key: string) => {
        const colRule = this.dcValidator?.getRule(key)

        if (colRule?.type === 'numeric' && row[key] === '.') row[key] = null
      })

      return row
    })

    this.loggerService.log('Data submitted', data)

    if (this.checkInvalid()) {
      let abortMsg = 'Invalid Values are Present'
      this.eventService.showInfoModal('Validation error', abortMsg)
      this.cancelSubmit()
      this.submit = false
      return
    }

    this.validationDone = 0
    this.saveLoading = true

    if (
      EditorComponent.cnt < 1 &&
      this.duplicatePkIndexes.length === 0 &&
      EditorComponent.nonPkCnt < 1
    ) {
      this.saveLoading = true
      this.disableSubmit = false
      this.submit = true
      let updateParams: any = {}
      updateParams.ACTION = 'LOAD'
      this.message = this.message.replace(/\n/g, '. ')
      updateParams.MESSAGE = this.message
      // updateParams.APPROVER = this.approver;
      updateParams.LIBDS = this.libds

      if (this.cols) {
        const submitData = data.slice(
          0,
          this.licenceState.value.submit_rows_limit
        )

        const success = await this.sasStoreService
          .updateTable(
            updateParams,
            submitData,
            'SASControlTable',
            'editors/stagedata',
            this.$dataFormats
          )
          .then((res: any) => {
            if (typeof res.sasparams !== 'undefined') {
              this.router.navigateByUrl('/stage/' + res.sasparams[0].DSID)

              return true
            }

            let error = `Submit request failed`

            if (res) {
              let errorText =
                typeof res === 'string' ? res : JSON.stringify(res)

              error += `\n${errorText}`
            }

            this.eventService.showAbortModal(
              'editors/stagedata',
              error,
              null,
              'Submit error'
            )
          })
          .catch((err: any) => {
            console.log('err', err)
            EditorComponent.cnt = 0
            EditorComponent.nonPkCnt = 0
            this.disableSubmit = true
            this.submit = false

            let errorText = typeof err === 'string' ? err : JSON.stringify(err)

            this.eventService.showAbortModal(
              'editors/stagedata',
              `Submit request failed\n${errorText}`,
              null,
              'Submit error'
            )

            return false
          })

        if (success) return //stop code execution if route redirected
      }
    }

    if (EditorComponent.cnt >= 1) {
      this.pkNull = true
      this.submit = true
    } else {
      this.submit = false
    }
    if (EditorComponent.nonPkCnt >= 1) {
      this.noPkNull = true
      this.submit = true
    } else {
      this.submit = false
    }

    this.cancelSubmit()
    EditorComponent.cnt = 0
    EditorComponent.nonPkCnt = 0
    this.disableSubmit = true
  }

  public validatorRuleSource(colName: string) {
    return this.dcValidator?.getRule(colName)
  }

  public checkInvalid() {
    const hotElement = (this.hotInstanceCompList.first.container as any)
      .nativeElement
    const invalidCells = hotElement.querySelectorAll('.htInvalid')

    return invalidCells.length > 0
  }

  public goToEditor() {
    this.router.navigateByUrl('/')
  }

  closeRecordEdit(confirmButtonClicked?: boolean) {
    this.currentEditRecordIndex = -1
    this.currentEditRecord = undefined
    this.currentEditRecordValidator = undefined

    if (this.recordAction === 'ADD' && !confirmButtonClicked) {
      this.dataSource = this.helperService.deepClone(this.prevDataSource)

      const hot = this.hotInstance

      hot.updateSettings(
        {
          data: this.dataSource
        },
        false
      )
    }
  }

  confirmRecordEdit(close: boolean = true) {
    const closingRecordIndex = this.currentEditRecordIndex

    if (close) this.currentEditRecordIndex = -1

    this.columnHeader.map((colName: string) => {
      const value = this.currentEditRecord[colName]
      const isNum = this.$dataFormats?.vars[colName]?.type === 'num'
      const specialMissing = isSpecialMissing(value)

      if (isNum && !isNaN(value) && !specialMissing) {
        this.currentEditRecord[colName] = value * 1
      }
    })

    this.dataSource[closingRecordIndex] = this.currentEditRecord
    this.hotTable.data[closingRecordIndex] = this.currentEditRecord

    const hot = this.hotInstance

    hot.updateSettings(
      {
        data: this.dataSource
      },
      false
    )

    if (close) this.currentEditRecord = undefined
  }

  onNextRecord() {
    this.confirmRecordEdit(false)
    this.currentEditRecordIndex =
      this.currentEditRecordIndex >= this.dataSource.length - 1
        ? 0
        : this.currentEditRecordIndex + 1
    this.editRecord(null, this.currentEditRecordIndex)
  }

  onPreviousRecord() {
    this.confirmRecordEdit(false)
    this.currentEditRecordIndex =
      this.currentEditRecordIndex <= 0
        ? this.dataSource.length - 1
        : this.currentEditRecordIndex - 1
    this.editRecord(null, this.currentEditRecordIndex)
  }

  addRecordButtonClick() {
    if (this.restrictions.restrictAddRecord) {
      this.eventService.showDemoLimitModal('Add Record')
      return
    }

    this.addEditNewRecord()
  }

  addEditNewRecord() {
    this.addRecord()
    setTimeout(() => {
      this.editRecord(null, this.dataSource.length - 1, true)
    }, 1000)
  }

  addRecord() {
    this.addRow()
  }

  editRecord(item: Element | null, index?: number, newRecord?: boolean) {
    if (this.restrictions.restrictEditRecord) {
      this.eventService.showDemoLimitModal('Edit Record')
      return
    }

    if (index === undefined || index < 0) return

    if (this.restrictions.restrictEditRecord) {
      return
    }

    this.recordAction = newRecord ? 'ADD' : 'EDIT'

    if (this.hotTable.readOnly) {
      this.editTable(false, newRecord)
    }

    // Create copy of DC validator to be used in RECORD MODAL
    this.currentEditRecordValidator = this.helperService.deepClone(
      this.dcValidator
    )

    if (newRecord) {
      this.prevDataSource = this.helperService.deepClone(this.dataSource)
      this.prevDataSource.pop()
    } else {
      const currentEditRecordCellsMeta = this.helperService.deepClone(
        this.hotInstance.getCellMetaAtRow(
          index
        ) as Partial<Handsontable.CellProperties>[]
      )

      // Update that copy with current cells meta (dynamic validation data)
      for (let cellMeta of currentEditRecordCellsMeta) {
        if (cellMeta) {
          const data = cellMeta.prop?.toString() //------------
          delete cellMeta.prop // We convert to be able to update dcValidator rule by using CellProperties
          delete cellMeta.data //-----------------

          this.currentEditRecordValidator?.updateRule(cellMeta.col!, {
            ...cellMeta,
            data: data
          })
        }
      }
    }

    this.currentEditRecordIndex = index
    this.currentEditRecord = this.helperService.deepClone(
      this.dataSource[index]
    )
  }

  toggleHotPlugin(pluginName: string, enable: boolean) {
    const hot = this.hotInstance

    hot.batch(() => {
      let contextMenuPlugin = hot.getPlugin<any>(pluginName)

      if (!contextMenuPlugin) {
        console.warn(
          'Toggle Hot Plugin failed - Plugin named: ' +
            pluginName +
            ' - could not be found.'
        )
        return
      }

      setTimeout(() => {
        if (enable) {
          contextMenuPlugin.enablePlugin()
          return
        }

        contextMenuPlugin.disablePlugin()
      }, 100)

      hot.render()
    })
  }

  private dynamicCellValidationDisabled(row: number, col: number) {
    const rowColFound = this.disabledBasicDynamicCellValidationMap.find(
      (x) => x.row === row && x.col === col && !x.active
    )

    return !!rowColFound
  }

  /**
   * This function takes row and column numbers for the cel to be validated and pouplated with values.
   * It will send the row values without the current column to the sas.
   * Sas will return values and if length greater then zero cel becomes dropdown type and values from sas
   * put to source.
   * @param row handsontable row
   * @param column handsontable column
   */
  public dynamicCellValidation(row: number, column: number) {
    if (this.dynamicCellValidationDisabled(row, column)) return

    const hot = this.hotInstance

    const cellMeta = hot.getCellMeta(row, column)

    if (cellMeta.readOnly) return

    const cellData = hot.getDataAtCell(row, column)
    const clickedRow = this.helperService.deepClone(this.dataSource[row])
    const clickedColumnKey = Object.keys(clickedRow)[column]

    /**
     * We will hash the row (without current column) so later we check if hash is the same
     * we set the values relative to that hash
     * if not we fire the request.
     */
    const hashedRow = this.helperService.deleteKeysAndHash(
      clickedRow,
      [clickedColumnKey, 'noLinkOption'],
      false
    )

    const validationSourceIndex = this.cellValidationSource.findIndex(
      (entry: CellValidationSource) => entry.hash === hashedRow
    )

    /**
     * Set the values for found hash.
     */

    if (validationSourceIndex > -1) {
      let colSource = this.cellValidationSource[
        validationSourceIndex
      ].values.map((el) => el[this.cellValidationFields.RAW_VALUE])

      const cellHadSource =
        (hot.getCellMeta(row, column).source || []).length < 1
      const cellHasValue = cellData !== ' '

      hot.batch(() => {
        const cellMeta = hot.getCellMeta(row, column)

        const cellRule = this.dcValidator?.getRule(
          (cellMeta.data as string) || ''
        )
        let cellSource: string[] | number[] | undefined

        if (cellRule) {
          cellSource = this.dcValidator?.getDqDropdownSource(cellRule)
        }

        if (!cellSource) cellSource = []

        if (cellRule?.type === 'numeric') {
          cellSource = this.helperService.convertArrayValues(
            cellSource,
            'number'
          ) as number[]
          colSource = this.helperService.convertArrayValues(
            colSource,
            'number'
          ) as number[]
        } else {
          cellSource = this.helperService.convertArrayValues(
            cellSource,
            'string'
          ) as string[]
          colSource = this.helperService.convertArrayValues(
            colSource,
            'string'
          ) as string[]
        }

        const cellSourceCombined = [...new Set([...cellSource, ...colSource])]

        hot.setCellMeta(row, column, 'source', cellSourceCombined)

        this.currentEditRecordValidator?.updateRule(column, {
          source: cellSourceCombined
        })

        if (
          this.cellValidationSource[validationSourceIndex].values.length > 0
        ) {
          const strict = this.cellValidationSource[validationSourceIndex].strict

          hot.setCellMeta(row, column, 'renderer', 'autocomplete')
          hot.setCellMeta(row, column, 'editor', 'autocomplete.custom')
          hot.setCellMeta(row, column, 'strict', strict)
          hot.setCellMeta(row, column, 'filter', false)

          this.currentEditRecordValidator?.updateRule(column, {
            renderer: 'autocomplete',
            editor: 'autocomplete.custom',
            strict: strict,
            filter: false
          })
        }

        this.reSetExtendedCellValidationValues(
          this.cellValidationSource[validationSourceIndex],
          row,
          cellHadSource && cellHasValue
        )

        hot.render()
      })
    }

    /**
     * Send request to sas.
     */
    if (validationSourceIndex < 0) {
      const data = {
        SASControlTable: [
          {
            libds: this.libds,
            variable_nm: clickedColumnKey
          }
        ],
        source_row: [clickedRow]
      }

      const validationHook = this.dcValidator
        ?.getDqDetails(clickedColumnKey)
        .find(
          (rule: DQRule) =>
            rule.RULE_TYPE === 'SOFTSELECT_HOOK' ||
            rule.RULE_TYPE === 'HARDSELECT_HOOK'
        )

      /**
       * Do the validation only if current column has validation hooks in place.
       */
      if (validationHook) {
        this.cellValidationSource.push({
          row: row,
          col: column,
          strict: validationHook.RULE_TYPE === 'HARDSELECT_HOOK',
          values: [],
          hash: hashedRow,
          count: this.cellValidationSource.length + 1
        })

        hot.setCellMeta(row, column, 'renderer', spinnerRenderer)

        this.currentEditRecordLoadings.push(column)
        hot.render()

        this.sasService
          .request('editors/getdynamiccolvals', data, undefined, {
            suppressSuccessAbortModal: true,
            suppressErrorAbortModal: true
          })
          .then((res: any) => {
            const colSource = res.dynamic_values.map(
              (el: any) => el[this.cellValidationFields.RAW_VALUE]
            )

            if (colSource.length > 0) {
              const validationSourceIndex = this.cellValidationSource.findIndex(
                (entry: CellValidationSource) => entry.hash === hashedRow
              )

              if (validationSourceIndex > -1) {
                this.cellValidationSource[validationSourceIndex] = {
                  ...this.cellValidationSource[validationSourceIndex],
                  row: row,
                  col: column,
                  values: res.dynamic_values,
                  extended_values: res.dynamic_extended_values
                }
              }

              //Removing the spinner from cell, so validation not fail
              hot.setCellMeta(row, column, 'renderer', noSpinnerRenderer)
              this.currentEditRecordLoadings.splice(
                this.currentEditRecordLoadings.indexOf(column),
                1
              )
              hot.deselectCell()
              hot.render()

              /**
               * `cells` function of hot settings is remembering the old state of component
               * we need to update it here after we set new `cellValidationSource` (validation lookup hash table) values
               * so that it will check those values to decide whether numeric cells should be
               * converted to the dropdown
               */

              hot.batch(() => {
                /**
                 * In the case that the original value is not included in the newly created cell dropdown
                 * and validation type is HARDSELECT, the cell shoud be red
                 */
                setTimeout(() => {
                  this.reSetCellValidationValues(true, row)
                  hot.render()

                  hot.validateRows([row])
                }, 100)
              })
            }

            //Removing the spinner from cell, so validation not fail
            hot.setCellMeta(row, column, 'renderer', noSpinnerRenderer)
            this.currentEditRecordLoadings.splice(
              this.currentEditRecordLoadings.indexOf(column),
              1
            )
            hot.deselectCell()
            hot.render()

            /**
             * If hash table limit reached, remove the oldest element.
             * Oldest element is element with lowest `count` number.
             */
            if (this.cellValidationSource.length > this.validationTableLimit) {
              const oldestElement = this.cellValidationSource.reduce(
                (prev, curr) => (prev.count < curr.count ? prev : curr)
              )
              const oldestElementIndex =
                this.cellValidationSource.indexOf(oldestElement)

              this.cellValidationSource.splice(oldestElementIndex, 1)
            }
          })
          .catch((err: any) => {
            const currentRowHashIndex = this.cellValidationSource.findIndex(
              (x) => x.hash === hashedRow
            )

            this.cellValidationSource.splice(currentRowHashIndex, 1)

            hot.batch(() => {
              // Render error icon inside a cell
              hot.setCellMeta(row, column, 'renderer', errorRenderer)

              hot.render()
            })

            //Stop edit record modal loading spinner
            this.currentEditRecordLoadings.splice(
              this.currentEditRecordLoadings.indexOf(column),
              1
            )

            //Show error on edit record modal
            this.currentEditRecordErrors.push(column)

            // After waiting time remove the error icon from cell and edit record modal field
            setTimeout(() => {
              hot.setCellMeta(row, column, 'renderer', noSpinnerRenderer)
              hot.render()

              //Remove error icon on the edit record modal field
              this.currentEditRecordErrors.splice(
                this.currentEditRecordErrors.indexOf(column),
                1
              )
            }, 3000)

            this.reSetCellValidationValues()

            this.loggerService.log('getdynamiccolvals error:', err)
          })
      }
    }
  }

  checkEmptyRowWhenFilter() {
    this.zeroFilterRows = false

    if (
      typeof this.filter_pk !== 'undefined' &&
      this.hotTable.data.length === 1
    ) {
      if ([null, ''].includes(this.hotTable.data[0][this.headerPks[0]]))
        this.zeroFilterRows = true
    }
  }

  onRecordInputFocus(event: EditRecordInputFocusedEvent) {
    this.dynamicCellValidation(this.currentEditRecordIndex, event.colName)
  }

  executeDynamicCellValidationIfApplicable(colProp: any, col: any, row: any) {
    const hashedRow = this.helperService.deleteKeysAndHash(
      this.dataSource[row],
      [colProp, 'noLinkOption']
    )

    const cellValidation = this.cellValidationSource.find(
      (entry: CellValidationSource) =>
        entry.hash === hashedRow && col === entry.col
    )

    if (
      cellValidation &&
      cellValidation.extended_values &&
      cellValidation.extended_values.length > 0
    ) {
      const extendedValidationObject = this.getExtendedValuesByCellValue(
        cellValidation,
        row
      )

      this.setExtendedValuesToCells(
        cellValidation,
        row,
        extendedValidationObject,
        true
      )
    }
  }

  datasetInfoModalRowClicked(value: Version | DSMeta) {
    if ((<Version>value).LOAD_REF !== undefined) {
      // Type is Version
      const row = value as Version
      const url = `/stage/${row.LOAD_REF}`

      this.router.navigate([url])
    }
  }

  viewboxManager() {
    this.viewboxes = true
  }

  get totalRowsChanged() {
    return (
      this.rowsChanged.rowsUpdated +
      this.rowsChanged.rowsDeleted +
      this.rowsChanged.rowsAdded
    )
  }

  /**
   * Function checks if selected hot cell is solo cell selected
   * and if it is, set the `filter` property based on filter param.
   *
   * @param filter
   */
  private setCellFilter(filter: boolean) {
    const hotSelected = this.hotInstance.getSelected()
    const selection = hotSelected ? hotSelected[0] : hotSelected

    // When we open a dropdown we want filter disabled so value in cell
    // don't filter out items, since we want to see them all.
    // But when we start typing we want to be able to start filtering values
    // again
    if (selection) {
      const startRow = selection[0]
      const endRow = selection[2]
      const startCell = selection[1]
      const endCell = selection[3]

      if (startRow === endRow && startCell === endCell) {
        const cellMeta = this.hotInstance.getCellMeta(startRow, startCell)

        // If filter is not already set at the value in the param, set it
        if (cellMeta && cellMeta.filter === !filter) {
          this.hotInstance.setCellMeta(startRow, startCell, 'filter', filter)
        }
      }
    }
  }

  async ngOnInit() {
    this.licenceService.hot_license_key.subscribe(
      (hot_license_key: string | undefined) => {
        this.hotTable.licenseKey = hot_license_key
      }
    )

    this._query = this.sasStoreService.query.subscribe((query: any) => {
      if (query.libds === this.libds) {
        this.whereString = query.string
        this.clauses = query.obj
        // this.libds = query.libds
      }
    })

    // recover lib and table parameters from url; filter pk is optional - if filter is applied
    let myParams: any = {}
    if (typeof this.route.snapshot.params['libMem'] !== 'undefined') {
      this.libds = this.route.snapshot.params['libMem']
      this.filter_pk = this.route.snapshot.params['filterId']

      if (this.route.snapshot.url[0].path === 'edit-record') {
        if (typeof this.filter_pk !== 'undefined') {
          this.recordAction = 'EDIT'
        } else {
          this.recordAction = 'ADD'
        }
      }

      myParams.LIBDS = this.libds
      if (typeof this.filter_pk !== 'undefined') {
        myParams.FILTER_RK = parseInt(this.filter_pk)
      }
      myParams.OUTDEST = 'WEB'

      if (this.libds) {
        globals.editor.library = this.libds.split('.')[0]
        globals.editor.table = this.libds.split('.')[1]
      }
    }

    if (this.libds) {
      this.getdataError = false

      await this.sasStoreService
        .callService(myParams, 'SASControlTable', 'editors/getdata', this.libds)
        .then((res: EditorsGetDataServiceResponse) => {
          this.initSetup(res)
        })
        .catch((err: any) => {
          this.getdataError = true
          this.tableTrue = true
        })
    }
  }

  ngAfterViewInit() {}

  initSetup(response: EditorsGetDataServiceResponse) {
    this.hotInstance = this.hotRegisterer.getInstance('hotInstance')

    if (this.getdataError) return
    if (!response) return
    if (!response.data) return

    this.cols = response.data.cols
    this.dsmeta = response.data.dsmeta
    this.versions = response.data.versions || []

    const notes = this.dsmeta.find((item) => item.NAME === 'NOTES')
    const longDesc = this.dsmeta.find((item) => item.NAME === 'DD_LONGDESC')
    const shortDesc = this.dsmeta.find((item) => item.NAME === 'DD_SHORTDESC')

    if (notes && notes.VALUE) {
      this.dsNote = notes.VALUE
    } else if (longDesc && longDesc.VALUE) {
      this.dsNote = longDesc.VALUE
    } else if (shortDesc && shortDesc.VALUE) {
      this.dsNote = shortDesc.VALUE
    } else {
      this.dsNote = ''
    }

    const hot: Handsontable = this.hotInstance

    const approvers: Approver[] = response.data.approvers

    if (this.cols) {
      this.headerArray = parseTableColumns(this.cols)
    }
    // Note: the above this.headerArray is being reassigned without being used.
    // So, above assignment does not make sense.

    approvers.forEach((item: Approver) => {
      this.approvers.push(item.PERSONNAME)
    })

    this.tableTrue = true
    this.libds = response.libds
    this.hotTable.data = response.data.sasdata

    this.headerColumns = response.data.sasparams[0].COLHEADERS.split(',')
    this.headerPks = response.data.sasparams[0].PK.split(' ')
    this.columnLevelSecurityFlag = !!response.data.sasparams[0].CLS_FLAG

    if (this.columnLevelSecurityFlag)
      this.setRestrictions({
        restrictAddRow: true,
        removeEditRecordButton: true,
        removeAddRecordButton: true
      })

    this.checkEmptyRowWhenFilter()

    if (this.headerColumns.indexOf('_____DELETE__THIS__RECORD_____') !== -1) {
      this.headerColumns[
        this.headerColumns.indexOf('_____DELETE__THIS__RECORD_____')
      ] = 'Delete?'
    }

    this.headerArray = this.headerColumns.slice(1)

    if (response.data.sasparams[0].DTVARS !== '') {
      this.dateHeaders = response.data.sasparams[0].DTVARS.split(' ')
    }
    if (response.data.sasparams[0].TMVARS !== '') {
      this.timeHeaders = response.data.sasparams[0].TMVARS.split(' ')
    }
    if (response.data.sasparams[0].DTTMVARS !== '') {
      this.dateTimeHeaders = response.data.sasparams[0].DTTMVARS.split(' ')
    }
    if (response.data.xl_rules.length > 0) {
      this.xlRules = this.helperService.deepClone(response.data.xl_rules)
    }

    this.dcValidator = new DcValidator(
      response.data.sasparams[0],
      response.data.$sasdata,
      this.cols,
      response.data.dqrules,
      response.data.dqdata
    )

    this.cellValidation = this.dcValidator.getRules()

    // to take datasource
    this.dataSource = response.data.sasdata
    this.$dataFormats = response.data.$sasdata

    // Note: this.headerColumns and this.columnHeader contains same data
    // need to resolve redundancy

    // default schema
    for (let i = 0; i < this.headerColumns.length; i++) {
      const colType = this.cellValidation[i].type

      this.hotDataSchema[this.cellValidation[i].data] = getHotDataSchema(
        colType,
        this.cellValidation[i]
      )
    }

    // this.addActionButtons();
    // this.addActionColumns();
    // now all validation params we have in this.cellValidation

    this.checkRowLimit()

    hot.updateSettings(
      {
        data: this.dataSource,
        colHeaders: this.headerColumns,
        columns: this.cellValidation,
        height: this.hotTable.height,
        formulas: this.hotTable.formulas,
        stretchH: 'all',
        readOnly: this.hotTable.readOnly,
        hiddenColumns: {
          indicators: true,
          columns: this.dcValidator.getHiddenColumns()
        },
        modifyColWidth: function (width: number, col: number) {
          if (col === 0) {
            return 60
          }
          if (width > 500) return 500
          else return width
        },
        copyPaste: this.hotTable.copyPaste,
        manualColumnFreeze: false, //https://handsontable.com/docs/7.0.3/demo-freezing.html
        // false due to https://forum.handsontable.com/t/gh-5112-column-freeze-sorting/3236
        multiColumnSorting: true, // https://handsontable.com/docs/7.0.0/demo-multicolumn-sorting.html
        manualColumnResize: true,
        filters: false,
        manualRowResize: true,
        viewportRowRenderingOffset: 50,
        // show a bar on the left to enable users to select an entire row
        rowHeaders: (index: number) => {
          return ' '
        },
        rowHeaderWidth: 15,
        rowHeights: 24,
        maxRows: this.licenceState.value.editor_rows_allowed || Infinity,
        invalidCellClassName: 'htInvalid',
        dropdownMenu: {
          items: {
            make_read_only: {
              name: 'make_read_only'
            },
            alignment: {
              name: 'alignment'
            },
            sp1: {
              name: '---------'
            },
            info: {
              name: 'test info',
              renderer: (
                hot: Handsontable.Core,
                wrapper: HTMLElement,
                row: number,
                col: number,
                prop: string | number,
                itemValue: string
              ) => {
                const elem = document.createElement('span')
                let colInfo: DataFormat | undefined
                let textInfo = 'No info found'

                if (this.hotInstance) {
                  const hotSelected: [number, number, number, number][] =
                    this.hotInstance.getSelected() || []
                  const selectedCol: number = hotSelected
                    ? hotSelected[0][1]
                    : -1
                  const colName = this.hotInstance?.colToProp(selectedCol)
                  colInfo = this.$dataFormats?.vars[colName]

                  if (colInfo)
                    textInfo = `LABEL: ${colInfo?.label}<br>TYPE: ${colInfo?.type}<br>LENGTH: ${colInfo?.length}<br>FORMAT: ${colInfo?.format}`
                }

                elem.innerHTML = textInfo

                return elem
              }
            }
          }
        },
        // filters: true,
        dataSchema: this.hotDataSchema,
        contextMenu: this.hotTable.settings.contextMenu,
        //, '---------','freeze_column','unfreeze_column'],
        currentHeaderClassName: 'customH',
        afterGetColHeader: (col: number, th: any) => {
          const column = this.columnHeader[col]

          // header columns styling - primary keys
          const isPKCol = column && this.isColPk(column)
          const isReadonlyCol = column && this.isReadonlyCol(column)

          if (isPKCol) th.classList.add('primaryKeyHeaderStyle')
          if (isReadonlyCol && !isPKCol) th.classList.add('readonlyCell')

          // Remove header arrow from Delete column
          if (col === 0) {
            th.classList.add('firstColumnHeaderStyle')
          }

          // Dark mode
          th.classList.add('darkTH')
        },
        afterGetCellMeta: (
          row: number,
          col: number,
          cellProperties: Handsontable.CellProperties
        ) => {
          const isReadonlyCol = col && this.isReadonlyCol(col)
          if (isReadonlyCol) cellProperties.className = 'readonlyCell'
        }
      },
      false
    )

    this.hotTable.hidden = false
    this.toggleHotPlugin('contextMenu', false)
    /**
     * This is needed if freeze column is enabled
     */
    // hot.getPlugin('manualColumnFreeze').freezeColumn(0);

    this.queryText = response.data.sasparams[0].FILTER_TEXT
    this.columnHeader = response.data.sasparams[0].COLHEADERS.split(',')
    // First column is always used to mark records for deletion
    this.columnHeader[0] = 'Delete?'
    this.readOnlyFields = response.data.sasparams[0].PKCNT

    let hotInstaceEl = document.getElementById('hotInstance')

    if (hotInstaceEl) {
      hotInstaceEl.addEventListener('mousedown', (event) => {
        if (!this.uploadPreview) {
          this.hotClicked()
        }

        setTimeout(() => {
          let menuDebugItem: any =
            document.querySelector('.debug-switch-item') || undefined
          if (menuDebugItem) menuDebugItem.click()
        }, 100)
      })
    }

    hot.addHook(
      'afterSelection',
      (
        row: number,
        column: number,
        row2: number,
        column2: number,
        preventScrolling: any,
        selectionLayerLevel: any
      ) => {
        /**
         * This is needed if freeze column is enabled
         */
        // if (column === 0) {
        //   delete contextMenuToSet.items.unfreeze_column;
        // }

        if (
          row === row2 &&
          column === column2 &&
          this.hotTable.readOnly === false
        ) {
          this.dynamicCellValidation(row, column)
        }

        /**
         * This is needed if freeze column is enabled
         */
        // if (column === 0) {
        //   if (!this.firstColumnSelected) {
        //     hot.updateSettings({
        //       contextMenu: contextMenuToSet
        //     });
        //     this.firstColumnSelected = true;
        //   }
        // } else {
        //   if (this.firstColumnSelected) {
        //     hot.updateSettings({
        //       contextMenu: contextMenuToSet
        //     });
        //     this.firstColumnSelected = false;
        //   }
        // }
      }
    )

    hot.addHook('afterBeginEditing', () => {
      // When we open a dropdown we want filter disabled so value in cell
      // don't filter out items, since we want to see them all.
      this.setCellFilter(false)
    })

    hot.addHook('beforeKeyDown', () => {
      // When we start typing, we are enabling the filter since we want to find
      // values faster.
      this.setCellFilter(true)
    })

    hot.addHook('afterChange', (source: any, change: any) => {
      if (change === 'edit') {
        const hot = this.hotInstance

        const row = source[0][0]
        const colProp = source[0][1]
        const col = hot.propToCol(colProp)

        // On edit we enabled filter for this cell, now when editing is finished
        // We want filter to be disabled again, to be ready for next dropdown opening.
        const cellMeta = hot.getCellMeta(row, col)
        if (cellMeta && cellMeta.filter === false)
          hot.setCellMeta(row, col, 'filter', true)

        this.executeDynamicCellValidationIfApplicable(colProp, col, row)
      }
    })

    hot.addHook('afterRender', (isForced: boolean) => {
      this.eventService.dispatchEvent('resize')
    })

    hot.addHook('afterCreateRow', (source: any, change: any) => {
      if (source > this.dataSource.length) {
        // don't scroll if row is not added to the end (bottom)
        let wtHolder = document.querySelector('.wtHolder')

        setTimeout(() => {
          if (wtHolder) wtHolder.scrollTop = wtHolder.scrollHeight
        })
      }
    })

    hot.addHook('beforePaste', (data: any, cords: any) => {
      const startCol = cords[0].startCol

      // We iterate trough pasting data to convert to numbers if needed
      data[0] = data[0].map((value: any, index: number) => {
        const colName = this.columnHeader[startCol + index]
        const isColNum = this.$dataFormats?.vars[colName]?.type === 'num'
        const specialMissing = isSpecialMissing(value)

        if (isColNum && !isNaN(value) && !specialMissing) value = value * 1

        return value
      })
    })

    hot.addHook('afterRemoveRow', () => {
      this.checkRowLimit()
    })

    hot.addHook('afterCreateRow', () => {
      this.checkRowLimit()
    })

    this.uploadUrl = 'services/editors/loadfile'

    if (this.recordAction !== null) {
      if (this.recordAction === 'ADD') {
        this.addRecord()
        this.editRecord(null, this.dataSource.length - 1, true)
      } else {
        if (this.dataSource.length === 1) {
          this.editRecord(null, 0)
        }
      }
    }

    if (response.data.query.length > 0) {
      if (
        (globals.rootParam === 'home' || globals.rootParam === 'editor') &&
        globals.editor.filter.clauses.length === 0
      ) {
        globals.editor.filter.query = this.helperService.deepClone(
          response.data.query
        )
        globals.editor.filter.libds = this.route.snapshot.params['libMem']
        this.sasStoreService.initializeGlobalFilterClause('editor', this.cols)
      }
    }

    hot.render()
  }
}
