feat: implemented the logic for xlmap component
Build / Build-and-ng-test (pull_request) Failing after 13s Details

This commit is contained in:
Sabir Hassan 2024-01-16 12:21:45 +05:00
parent d67d4e2f86
commit 50696bb926
3 changed files with 897 additions and 16 deletions

View File

@ -1,4 +1,8 @@
<app-sidebar>
<div *ngIf="xlmapsLoading" class="my-10-mx-auto text-center">
<clr-spinner clrMedium></clr-spinner>
</div>
<clr-tree>
<clr-tree-node class="search-node">
<div class="tree-search-wrapper">
@ -8,7 +12,7 @@
placeholder="Filter by Id"
name="input"
[(ngModel)]="searchString"
(keyup)="mapListOnFilter()"
(keyup)="xlmapListOnFilter()"
autocomplete="off"
/>
<clr-icon
@ -17,19 +21,177 @@
></clr-icon>
<clr-icon
*ngIf="searchXLMapTreeInput.value.length > 0"
(click)="searchString = ''; mapListOnFilter()"
(click)="searchString = ''; xlmapListOnFilter()"
shape="times"
></clr-icon>
</div>
</clr-tree-node>
<ng-container *ngFor="let xlmap of xlmaps">
<clr-tree-node (click)="xlmapOnClick(xlmap)">
<p class="m-0 cursor-pointer list-padding">
<clr-tree-node>
<button
(click)="xlmapOnClick(xlmap)"
class="clr-treenode-link"
[class.table-active]="isActiveXLMap(xlmap)"
>
<clr-icon shape="file"></clr-icon>
{{ xlmap }}
</p>
</button>
</clr-tree-node>
</ng-container>
</clr-tree>
</app-sidebar>
<div class="content-area">
<div *ngIf="!selectedXLMapId" class="no-table-selected">
<clr-icon
shape="warning-standard"
size="60"
class="is-info icon-dc-fill"
></clr-icon>
<h3 *ngIf="xlmaps.length > 0" class="text-center color-gray">
Please select a map
</h3>
<h3 *ngIf="xlmaps.length < 1" class="text-center color-gray">
No excel map is found
</h3>
</div>
<div class="loadingSpinner" *ngIf="isLoading">
<span class="spinner"> Loading... </span>
<div>
<h4>{{ isLoadingDesc }}</h4>
</div>
</div>
<div
appDragNdrop
(fileDraggedOver)="onShowUploadModal()"
class="card h-100 d-flex clr-flex-column"
*ngIf="!isLoading && xlmapData"
>
<div class="clr-row m-0 clr-justify-content-center">
<div class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4">
<span class="btn btn-sm" [routerLink]="['/home/files']">
<clr-icon shape="caret" dir="left" size="20"></clr-icon>Back to map
selection
</span>
</div>
<div
*ngIf="status === StatusEnum.ReadyToUpload"
class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
>
<button
type="button"
class="btn btn-sm btn-success btn-block mr-0"
(click)="onShowUploadModal()"
>
<clr-icon shape="upload"></clr-icon>
<span>Upload</span>
</button>
</div>
<div
*ngIf="status === StatusEnum.ReadyToSubmit"
class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
>
<button
type="button"
class="btn btn-sm btn-success btn-block mr-0"
(click)="submitExcel()"
>
<clr-icon shape="upload"></clr-icon>
<span>Submit</span>
</button>
</div>
<div
*ngIf="status === StatusEnum.ReadyToSubmit"
class="d-flex clr-justify-content-center clr-col-12 clr-col-lg-4"
>
<button
type="button"
class="btn btn-sm btn-outline-danger btn-block mr-0"
(click)="discardExtractedData()"
>
<clr-icon shape="times"></clr-icon>
<span>Discard</span>
</button>
</div>
</div>
<div class="clr-row m-0">
<h3 class="viewerTitle d-flex clr-col-12 clr-justify-content-center">
{{ displayTitle }}
</h3>
<h5 class="viewerTitle d-flex clr-col-12 clr-justify-content-center">
Source dataset for mapping rules: {{ dcLib }}.MPE_XLMAP_RULES
</h5>
<h5 class="viewerTitle d-flex clr-col-12 clr-justify-content-center">
Target dataset for uploaded data: {{ xlmapData.TARGET_DS }}
</h5>
</div>
<div class="clr-flex-1">
<hot-table
hotId="hotInstance"
id="hotTable"
[multiColumnSorting]="true"
[viewportRowRenderingOffset]="50"
[data]="hotTable.data"
[colHeaders]="hotTable.colHeaders"
[columns]="hotTable.columns"
[filters]="true"
[height]="hotTable.height"
stretchH="all"
[modifyColWidth]="maxWidthChecker"
[cells]="hotTable.cells"
[maxRows]="hotTable.maxRows"
[manualColumnResize]="true"
[rowHeaders]="hotTable.rowHeaders"
[rowHeaderWidth]="hotTable.rowHeaderWidth"
[rowHeights]="hotTable.rowHeights"
[licenseKey]="hotTable.licenseKey"
>
</hot-table>
</div>
</div>
<clr-modal
appFileDrop
(fileOver)="fileOverBase($event)"
(fileDrop)="getFileDesc($event, true)"
[clrModalSize]="'xl'"
[clrModalStaticBackdrop]="false"
[clrModalClosable]="true"
[(clrModalOpen)]="showUploadModal"
class="relative"
>
<h3 class="modal-title">Upload File</h3>
<div class="modal-body">
<div class="drop-area">
<span>Drop file anywhere to upload!</span>
</div>
<div class="clr-col-md-12">
<div class="clr-row card-block mt-15 d-flex justify-content-between">
<div class="clr-col-md-3 filterBtn">
<span class="filterBtn w-100">
<label
for="file-upload"
class="btn btn-sm btn-outline profile-buttons w-100"
>
Browse
</label>
</span>
<input
hidden
#fileUploadInput
id="file-upload"
type="file"
appFileSelect
(change)="getFileDesc($event)"
/>
</div>
</div>
</div>
</div>
</clr-modal>
</div>

View File

@ -0,0 +1,78 @@
.card {
margin-top: 0;
flex: 1;
display: flex;
flex-direction: column;
}
clr-tree-node button {
white-space: nowrap;
}
.no-table-selected {
position: relative;
}
.header-row {
.title-col {
display: flex;
align-items: center;
}
.options-col {
display: flex;
justify-content: flex-end;
}
}
.sw {
margin: 1rem 0rem 0.5rem 1rem;
}
.viewerTitle {
text-align: center;
margin-bottom: 15px;
}
.cardFlex {
display: flex;
justify-content: center;
}
.content-area {
padding: 0.5rem !important;
display: flex;
flex-direction: column;
}
hot-table {
::ng-deep {
.primaryKeyHeaderStyle {
background: #306b006e;
}
}
}
.drop-area {
position: fixed;
top: 0;
left: 0;
bottom: 0;
right: 0;
display: flex;
justify-content: center;
margin: 1px;
border: 2px dashed #fff;
z-index: -1;
span {
font-size: 20px;
margin-top: 20px;
color: #fff;
}
}

View File

@ -1,34 +1,641 @@
import { Component, AfterContentInit } from '@angular/core'
import {
AfterContentInit,
AfterViewInit,
Component,
HostBinding,
OnInit
} from '@angular/core'
import { ActivatedRoute, Router } from '@angular/router'
import { UploadFile } from '@sasjs/adapter'
import * as XLSX from '@sheet/crypto'
import { globals } from '../_globals'
import { EventService } from '../services'
import { HotTableInterface } from '../models/HotTable.interface'
import {
EventService,
LicenceService,
LoggerService,
SasService,
SasStoreService
} from '../services'
interface XLMapRule {
XLMAP_ID: string
XLMAP_SHEET: string
XLMAP_RANGE_ID: string
XLMAP_START: string
XLMAP_FINISH: string
}
interface XLMapData {
TARGET_DS: string
xlmaprules: XLMapRule[]
}
interface XLUploadEntry {
LOAD_REF: string
XLMAP_ID: string
XLMAP_RANGE_ID: string
ROW_NO: number
COL_NO: number
VALUE_TXT: string
}
enum Status {
NoMapSelected,
FetchingRules,
ReadyToUpload,
ExtractingData,
ReadyToSubmit,
SubmittingExtractedData,
Submitting
}
@Component({
selector: 'app-xlmap',
templateUrl: './xlmap.component.html',
styleUrls: ['./xlmap.component.scss']
})
export class XLMapComponent implements AfterContentInit {
export class XLMapComponent implements AfterContentInit, AfterViewInit, OnInit {
@HostBinding('class.content-container') contentContainerClass = true
StatusEnum = Status
public displayTitle = ''
public dcLib = globals.dcLib
public xlmaps: string[] = []
public xlmapData: XLMapData | undefined = undefined
public selectedXLMapId = ''
public searchString = ''
public loading = true
public xlmapsLoading = true
public isLoading = false
public isLoadingDesc = ''
public status = Status.NoMapSelected
public xlmapRulesHeaders = [
'XLMAP_SHEET',
'XLMAP_RANGE_ID',
'XLMAP_START',
'XLMAP_FINISH'
]
public xlmapRulesColumns = [
{
data: 'XLMAP_SHEET'
},
{
data: 'XLMAP_RANGE_ID'
},
constructor(private eventService: EventService) {}
{
data: 'XLMAP_START'
},
{
data: 'XLMAP_FINISH'
}
]
public xlmapOnClick(xlmap: any) {
// todo/current: implement logic
public xlUploadHeader = ['XLMAP_RANGE_ID', 'ROW_NO', 'COL_NO', 'VALUE_TXT']
public xlUploadColumns = [
{
data: 'XLMAP_RANGE_ID'
},
{
data: 'ROW_NO'
},
{
data: 'COL_NO'
},
{
data: 'VALUE_TXT'
}
]
public hotTable: HotTableInterface = {
data: [],
colHeaders: this.xlmapRulesHeaders,
columns: this.xlmapRulesColumns,
height: '100%',
rowHeaderWidth: 15,
rowHeaders: () => {
return ' '
},
rowHeights: 20,
maxRows:
this.licenceService.licenceState.value.viewer_rows_allowed || Infinity,
settings: {}
}
public mapListOnFilter() {
// todo/current: implement logic
public uploadPreview = false
public showUploadModal = false
public hasBaseDropZoneOver = false
public filename = ''
public submitLimitNotice = false
constructor(
private eventService: EventService,
private licenceService: LicenceService,
private loggerService: LoggerService,
private route: ActivatedRoute,
private router: Router,
private sasStoreService: SasStoreService,
private sasService: SasService
) {}
private blobToFile(theBlob: Blob, fileName: string): File {
const b: any = theBlob
b.lastModifiedDate = new Date()
b.name = fileName
return b as File
}
public xlmapOnClick(xlmap: string) {
if (xlmap !== this.selectedXLMapId) {
this.selectedXLMapId = xlmap
this.viewXLMapRules()
this.router.navigateByUrl('/home/files/' + xlmap)
}
}
public xlmapListOnFilter() {
if (this.searchString.length > 0) {
const array: string[] = globals.editor.xlmaps
this.xlmaps = array.filter((item) =>
item.toLowerCase().includes(this.searchString.toLowerCase())
)
} else {
this.xlmaps = globals.editor.xlmaps
}
}
public getFromGlobals() {
this.xlmaps = globals.editor.xlmaps
this.loading = false
this.xlmapsLoading = false
}
ngAfterContentInit(): void {
public isActiveXLMap(id: string) {
return this.selectedXLMapId === id
}
public maxWidthChecker(width: any, col: any) {
if (width > 200) return 200
else return width
}
public onShowUploadModal() {
if (!this.uploadPreview) this.showUploadModal = true
}
/**
* Called by FileDropDirective
* @param e true if file is dragged over the drop zone
*/
public fileOverBase(e: boolean): void {
this.hasBaseDropZoneOver = e
}
public getFileDesc(event: any, dropped = false) {
this.showUploadModal = false
this.isLoading = true
this.isLoadingDesc = 'Extracting Data'
this.status = Status.ExtractingData
const file = dropped ? event[0] : event.target.files[0]
const filename = file.name
this.filename = filename
// this.appendUploadState(`Loading ${filename} into the browser`)
const fileType = filename.slice(
filename.lastIndexOf('.') + 1,
filename.lastIndexOf('.') + 4
)
if (fileType.toLowerCase() === 'xls') {
const reader = new FileReader()
reader.onload = async (theFile: any) => {
/* read workbook */
const bstr = this.toBstr(theFile.target.result)
let wb: XLSX.WorkBook | undefined = undefined
const xlsxOptions: XLSX.ParsingOptions = {
type: 'binary',
cellDates: false,
cellFormula: true,
cellStyles: true,
cellNF: false,
cellText: false
}
try {
wb = XLSX.read(bstr, {
...xlsxOptions
})
} catch (err: any) {
this.eventService.showAbortModal(
null,
err,
undefined,
'Error reading file'
)
}
if (!wb) {
this.isLoading = false
this.isLoadingDesc = ''
// todo: show abort message when data not found
return
}
this.extractData(wb)
return
}
reader.readAsArrayBuffer(file)
} else {
const abortMsg =
'Invalid file type "<b>' +
this.filename +
'</b>". Please upload excel file.'
this.eventService.showAbortModal(null, abortMsg)
}
}
public toBstr(res: any) {
const bytes = new Uint8Array(res)
let binary = ''
const length = bytes.byteLength
for (let i = 0; i < length; i++) {
binary += String.fromCharCode(bytes[i])
}
return binary
}
public discardExtractedData() {
this.isLoading = false
this.isLoadingDesc = ''
this.status = Status.ReadyToUpload
this.displayTitle = `Rules for ${this.selectedXLMapId}`
this.hotTable.colHeaders = this.xlmapRulesHeaders
this.hotTable.columns = this.xlmapRulesColumns
this.hotTable.data = this.xlmapData?.xlmaprules || []
}
/**
* Submits attached excel file that is in preview mode
*/
public submitExcel() {
if (this.licenceService.licenceState.value.submit_rows_limit !== Infinity) {
this.submitLimitNotice = true
return
}
this.submit()
}
public submit() {
if (!this.hotTable.data.length) return
this.status = Status.Submitting
this.isLoading = true
this.isLoadingDesc = 'Submitting extracted data'
const filesToUpload: UploadFile[] = []
const csvContent =
Object.keys(this.hotTable.data[0]).join(',') +
'\n' +
this.hotTable.data
.map((row: any) => Object.values(row).join(','))
.join('\n')
const blob = new Blob([csvContent], { type: 'application/csv' })
const file: File = this.blobToFile(blob, this.filename + '.csv')
filesToUpload.push({
file: file,
fileName: file.name
})
const uploadUrl = 'services/editors/loadfile'
this.sasService
.uploadFile(uploadUrl, filesToUpload, {
table: this.xlmapData?.TARGET_DS
})
.then((res: any) => {
if (res.sasjsAbort) {
const abortRes = res
const abortMsg = abortRes.sasjsAbort[0].MSG
const macMsg = abortRes.sasjsAbort[0].MAC
this.filename = ''
this.eventService.showAbortModal('', abortMsg, {
SYSWARNINGTEXT: abortRes.SYSWARNINGTEXT,
SYSERRORTEXT: abortRes.SYSERRORTEXT,
MAC: macMsg
})
} else if (res.sasparams) {
const params = res.sasparams[0]
const tableId = params.DSID
this.router.navigateByUrl('/stage/' + tableId)
}
})
.catch((err: any) => {
this.filename = ''
this.eventService.catchResponseError('file upload', err)
})
}
public extractData(wb: XLSX.WorkBook) {
const extractedData: XLUploadEntry[] = []
this.xlmapData?.xlmaprules.forEach((rule) => {
let sheetName = rule.XLMAP_SHEET
if (sheetName.startsWith('/')) {
const temp = sheetName.split('/')[1]
const sheetIndex = parseInt(temp) - 1
sheetName = wb.SheetNames[sheetIndex]
}
const sheet = wb.Sheets[sheetName]
const arrayOfObjects = <any[]>XLSX.utils.sheet_to_json(sheet, {
raw: true,
header: 'A',
blankrows: true
})
const start = this.getCellAddress(rule.XLMAP_START, arrayOfObjects)
const finish = this.getFinishingCell(
start,
rule.XLMAP_FINISH,
arrayOfObjects
)
const range = `${start}:${finish}`
const rangedData = <any[]>XLSX.utils.sheet_to_json(sheet, {
raw: true,
range: range,
header: 'A',
blankrows: true
})
for (let i = 0; i < rangedData.length; i++) {
const row = rangedData[i]
// Get the keys of the object (excluding '__rowNum__')
const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
for (let j = 0; j < keys.length; j++) {
const key = keys[j]
const val = row[key]
extractedData.push({
LOAD_REF: '0',
XLMAP_ID: rule.XLMAP_ID,
XLMAP_RANGE_ID: rule.XLMAP_RANGE_ID,
ROW_NO: i + 1,
COL_NO: j + 1,
VALUE_TXT: val
})
}
}
})
this.status = Status.ReadyToSubmit
this.isLoading = false
this.isLoadingDesc = ''
this.displayTitle = `Extracted Data for ${this.selectedXLMapId}`
this.hotTable.colHeaders = this.xlUploadHeader
this.hotTable.columns = this.xlUploadColumns
this.hotTable.data = extractedData
}
public getCellAddress(rule: string, arrayOfObjects: any[]) {
if (rule.startsWith('ABSOLUTE ')) {
rule = rule.replace('ABSOLUTE ', '')
}
if (rule.startsWith('RELATIVE ')) {
const rowAndCol = this.extractRowAndCol(rule)
if (rowAndCol) {
const { row, column } = rowAndCol
// Generate an A1-Style address string from a SheetJS cell address
// Spreadsheet applications typically display ordinal row numbers,
// where 1 is the first row, 2 is the second row, etc. The numbering starts at 1.
// SheetJS follows JavaScript counting conventions,
// where 0 is the first row, 1 is the second row, etc. The numbering starts at 0.
// Therefore, we have to subtract 1 from row and column to match SheetJS indexing convention
rule = XLSX.utils.encode_cell({ r: row - 1, c: column - 1 })
} else {
// todo: handle the case when could not find row and col indexes
}
}
if (rule.startsWith('MATCH ')) {
let targetValue = ''
// using a regular expression to match "C[x]:" and extract the value after it
const match = rule.match(/C\[\d+\]:(.+)/)
// Check if there is a match
if (match) {
// Extract the value after "C[x]:"
targetValue = match[1]
} else {
// todo: throw/display error when target value is not found
}
// Split the string by spaces to get target row/column
const splittedArray = rule.split(' ')
// Extract the second word
const secondWord = splittedArray[1]
let targetColumn = ''
let targetRow = -1
let cellAddress = ''
// Check if the secondWord is a number
if (!isNaN(Number(secondWord))) {
targetRow = parseInt(secondWord)
} else {
targetColumn = secondWord
}
if (targetRow !== -1) {
// sheetJS index starts from 0,
// therefore, decremented 1 to make it correct row address for js array
const row = arrayOfObjects[targetRow - 1]
for (const col in row) {
if (col !== '__rowNum__' && row[col] === targetValue) {
cellAddress = col + targetRow
break
}
}
} else {
for (let i = 0; i < arrayOfObjects.length; i++) {
const row = arrayOfObjects[i]
if (row[targetColumn] === targetValue) {
// sheetJS index starts from 0,
// therefore, incremented 1 to make it correct row address
const rowIndex = i + 1
cellAddress = targetColumn + rowIndex
break
}
}
}
const temp = XLSX.utils.decode_cell(cellAddress)
const rowAndCol = this.extractRowAndCol(rule)
if (rowAndCol) {
const { row, column } = rowAndCol
rule = XLSX.utils.encode_cell({ r: temp.r + row, c: temp.c + column })
} else {
// todo: handle the case when regex does not match
}
}
return rule
}
public getFinishingCell(
start: string,
finish: string,
arrayOfObjects: any[]
) {
if (finish === '') {
return start
}
if (finish.startsWith('ABSOLUTE ')) {
finish = finish.replace('ABSOLUTE ', '')
}
if (finish.startsWith('RELATIVE ')) {
const rowAndCol = this.extractRowAndCol(finish)
if (rowAndCol) {
const { row, column } = rowAndCol
const { r, c } = XLSX.utils.decode_cell(start)
finish = XLSX.utils.encode_cell({ r: r + row, c: c + column })
} else {
// todo: handle the case when could not find row and col indexes
}
}
if (finish.startsWith('MATCH ')) {
finish = this.getCellAddress(finish, arrayOfObjects)
}
if (finish === 'LASTDOWN') {
const { r, c } = XLSX.utils.decode_cell(start)
const colName = XLSX.utils.encode_col(c)
let lastNonBlank = r
for (let i = r + 1; i < arrayOfObjects.length; i++) {
const row = arrayOfObjects[i]
if (!row[colName]) {
break
}
lastNonBlank = i
}
finish = colName + lastNonBlank
}
if (finish === 'BLANKROW') {
const { r } = XLSX.utils.decode_cell(start)
let lastNonBlankRow = r
for (let i = r + 1; i < arrayOfObjects.length; i++) {
const row = arrayOfObjects[i]
if (this.isBlankRow(row)) {
break
}
lastNonBlankRow = i
}
const row = arrayOfObjects[lastNonBlankRow]
// Get the keys of the object (excluding '__rowNum__')
const keys = Object.keys(row).filter((key) => key !== '__rowNum__')
// Find the key with the highest alphanumeric value (assumes keys are letters)
const lastColumn = keys.reduce(
(maxKey, currentKey) => (currentKey > maxKey ? currentKey : maxKey),
''
)
finish = lastColumn + lastNonBlankRow
}
return finish
}
public extractRowAndCol(str: string) {
// Regular expression to match and capture the values inside square brackets
const regex = /R\[(\d+)\]C\[(\d+)\]/
// Match the regular expression against the input string
const match = str.match(regex)
if (!match) {
return null
}
// Extract values from the match groups
const row = parseInt(match[1], 10)
const column = parseInt(match[2], 10)
return {
row,
column
}
}
public isBlankRow(row: any) {
for (const key in row) {
if (key !== '__rowNum__') {
return false
}
}
return true
}
async viewXLMapRules() {
this.isLoading = true
this.isLoadingDesc = 'Loading excel rules'
this.status = Status.FetchingRules
await this.sasStoreService
.getXLMapRules(this.selectedXLMapId)
.then((res) => {
this.xlmapData = {
TARGET_DS: res.xlmapinfo[0].TARGET_DS,
xlmaprules: res.xlmaprules
}
this.displayTitle = `Rules for ${this.selectedXLMapId}`
this.status = Status.ReadyToUpload
this.hotTable.colHeaders = this.xlmapRulesHeaders
this.hotTable.columns = this.xlmapRulesColumns
this.hotTable.data = this.xlmapData.xlmaprules
this.hotTable.cells = () => ({ readOnly: true })
})
.catch((err) => {
this.loggerService.error(err)
})
this.isLoading = false
this.isLoadingDesc = ''
}
async loadWithoutParameters() {
if (globals.editor.startupSet) {
this.getFromGlobals()
} else {
@ -37,4 +644,38 @@ export class XLMapComponent implements AfterContentInit {
})
}
}
async loadWithParameters() {
if (globals.editor.startupSet) {
this.getFromGlobals()
} else {
this.eventService.onStartupDataLoaded.subscribe(() => {
this.getFromGlobals()
})
}
// todo/current: if id is not a valid xlmap id, show appropriate view
this.selectedXLMapId = this.route.snapshot.params['id']
this.viewXLMapRules()
}
ngOnInit() {
this.licenceService.hot_license_key.subscribe(
(hot_license_key: string | undefined) => {
this.hotTable.licenseKey = hot_license_key
}
)
}
ngAfterViewInit() {
return
}
ngAfterContentInit(): void {
if (typeof this.route.snapshot.params['id'] !== 'undefined') {
this.loadWithParameters()
} else {
this.loadWithoutParameters()
}
}
}