dc/dc
dc/dc
3
1
Fork 0
Code Issues 15 Pull Requests 1 Actions Packages Projects Releases 36 Wiki Activity

feat: adding ability to define the target table for excel maps
Some checks failed
Build / Build-and-ng-test (pull_request) Failing after 13s
Details

Browse Source
This commit is contained in:
zmaj 2024-01-11 18:11:22 +00:00
parent e747e6e4e7
commit c86fba9dc7
17 changed files with 248 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sas/sasjs/db/datactrl/mpe_excel_upload.ddl → sas/sasjs/db/datactrl/mpe_xlmap_data.ddl
View File

@ -1,18 +1,18 @@
/** /**
@file @file
@brief DDL for MPE_EXCEL_UPLOAD @brief DDL for MPE_XLMAP_DATA
@version 9.3 @version 9.3
@author 4GL Apps Ltd @author 4GL Apps Ltd
@copyright 4GL Apps Ltd @copyright 4GL Apps Ltd
**/ **/
create table &curlib..mpe_excel_upload( create table &curlib..MPE_XLMAP_DATA(
LOAD_REF char(32) not null, LOAD_REF char(32) not null,
XLMAP_ID char(32) not null, XLMAP_ID char(32) not null,
XLMAP_RANGE_ID char(32) not null, XLMAP_RANGE_ID char(32) not null,
ROW_NO num not null, ROW_NO num not null,
COL_NO num not null, COL_NO num not null,
VALUE_TXT char(4000), VALUE_TXT char(4000),
constraint pk_mpe_excel_upload constraint pk_MPE_XLMAP_DATA
primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO)); primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO));

17
sas/sasjs/db/datactrl/mpe_xlmap_info.ddl Normal file
View File

@ -0,0 +1,17 @@
/**
@file
@brief DDL for mpe_xlmap_info
@version 9.3
@author 4GL Apps Ltd
@copyright 4GL Apps Ltd
**/
create table &curlib..mpe_xlmap_info(
tx_from num not null,
XLMAP_ID char(32) not null,
XLMAP_DESCRIPTION char(1000) not null,
XLMAP_TARGETLIBDS char(41) not null,
tx_to num not null,
constraint pk_mpe_xlmap_info
primary key(tx_from,XLMAP_ID));

6
sas/sasjs/db/datactrl/mpe_excel_map.ddl → sas/sasjs/db/datactrl/mpe_xlmap_rules.ddl
View File

@ -1,13 +1,13 @@
/** /**
@file @file
@brief DDL for MPE_EXCEL_MAP @brief DDL for mpe_xlmap_rules
@version 9.3 @version 9.3
@author 4GL Apps Ltd @author 4GL Apps Ltd
@copyright 4GL Apps Ltd @copyright 4GL Apps Ltd
**/ **/
create table &curlib..mpe_excel_map( create table &curlib..mpe_xlmap_rules(
tx_from num not null, tx_from num not null,
XLMAP_ID char(32) not null, XLMAP_ID char(32) not null,
XLMAP_RANGE_ID char(32) not null, XLMAP_RANGE_ID char(32) not null,
@ -15,5 +15,5 @@ create table &curlib..mpe_excel_map(
XLMAP_START char(1000) not null, XLMAP_START char(1000) not null,
XLMAP_FINISH char(1000), XLMAP_FINISH char(1000),
tx_to num not null, tx_to num not null,
constraint pk_mpe_excel_map constraint pk_mpe_xlmap_rules
primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID)); primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID));

17
sas/sasjs/db/migrations/20230115_v6.5_release.sas
View File

@ -13,7 +13,7 @@ libname &dclib "/your/dc/path";
* New MPE_SUBMIT table * New MPE_SUBMIT table
*/ */
proc sql; proc sql;
create table &dclib..mpe_excel_map( create table &dclib..mpe_xlmap_rules(
tx_from num not null, tx_from num not null,
XLMAP_ID char(32) not null, XLMAP_ID char(32) not null,
XLMAP_RANGE_ID char(32) not null, XLMAP_RANGE_ID char(32) not null,
@ -21,15 +21,24 @@ create table &dclib..mpe_excel_map(
XLMAP_START char(1000) not null, XLMAP_START char(1000) not null,
XLMAP_FINISH char(1000), XLMAP_FINISH char(1000),
tx_to num not null, tx_to num not null,
constraint pk_mpe_excel_map constraint pk_mpe_xlmap_rules
primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID)); primary key(tx_from,XLMAP_ID,XLMAP_RANGE_ID));
create table &dclib..mpe_excel_upload( create table &dclib..MPE_XLMAP_DATA(
LOAD_REF char(32) not null, LOAD_REF char(32) not null,
XLMAP_ID char(32) not null, XLMAP_ID char(32) not null,
XLMAP_RANGE_ID char(32) not null, XLMAP_RANGE_ID char(32) not null,
ROW_NO num not null, ROW_NO num not null,
COL_NO num not null, COL_NO num not null,
VALUE_TXT char(4000), VALUE_TXT char(4000),
constraint pk_mpe_excel_upload constraint pk_MPE_XLMAP_DATA
primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO)); primary key(LOAD_REF, XLMAP_ID, XLMAP_RANGE_ID, ROW_NO, COL_NO));
create table &dclib..mpe_xlmap_info(
tx_from num not null,
XLMAP_ID char(32) not null,
XLMAP_DESCRIPTION char(1000) not null,
XLMAP_TARGETLIBDS char(41) not null,
tx_to num not null,
constraint pk_mpe_xlmap_info
primary key(tx_from,XLMAP_ID));

2
sas/sasjs/macros/mpe_loader.sas
View File

@ -451,7 +451,7 @@ run;
%do i=1 %to %sysfunc(countw(&pk)); %do i=1 %to %sysfunc(countw(&pk));
%let iWord=%scan(&pk,&i); %let iWord=%scan(&pk,&i);
call symputx('duplist',symget('duplist')!! call symputx('duplist',symget('duplist')!!
" &iWord="!!trim(&iWord)); " &iWord="!!cats(&iWord));
%end; %end;
run; run;
%let msg=This upload contains duplicates on the Primary Key columns %trim( %let msg=This upload contains duplicates on the Primary Key columns %trim(

97
sas/sasjs/macros/mpe_makedata.sas
View File

@ -270,149 +270,159 @@ insert into &lib..mpe_datadictionary set
,tx_to='31DEC5999:23:59:59'dt; ,tx_to='31DEC5999:23:59:59'dt;
/** /**
* MPE_EXCEL_MAP * mpe_xlmap_info
*/ */
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_info set
tx_from=0
,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1'
,xlmap_description='Basel 3 Key Metrics report'
,XLMAP_TARGETLIBDS="&lib..MPE_XLMAP_DATA";
/**
* mpe_xlmap_rules
*/
insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:a' ,xlmap_range_id='KM1:a'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH 4 R[2]C[0]:a'; ,xlmap_start='MATCH 4 R[2]C[0]:a';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:b' ,xlmap_range_id='KM1:b'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH 4 R[2]C[0]:b'; ,xlmap_start='MATCH 4 R[2]C[0]:b';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:c' ,xlmap_range_id='KM1:c'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH 4 R[2]C[0]:c'; ,xlmap_start='MATCH 4 R[2]C[0]:c';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:d' ,xlmap_range_id='KM1:d'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH 4 R[2]C[0]:d'; ,xlmap_start='MATCH 4 R[2]C[0]:d';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:e' ,xlmap_range_id='KM1:e'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH 4 R[2]C[0]:e'; ,xlmap_start='MATCH 4 R[2]C[0]:e';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:f' ,xlmap_range_id='KM1:f'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH 4 R[2]C[0]:f'; ,xlmap_start='MATCH 4 R[2]C[0]:f';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1/a' ,xlmap_range_id='KM1:1/a'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[0]C[1]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[0]C[1]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1/b' ,xlmap_range_id='KM1:1/b'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[0]C[2]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[0]C[2]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1/c' ,xlmap_range_id='KM1:1/c'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[0]C[3]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[0]C[3]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1/d' ,xlmap_range_id='KM1:1/d'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[0]C[4]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[0]C[4]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1/e' ,xlmap_range_id='KM1:1/e'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[0]C[5]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[0]C[5]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1/f' ,xlmap_range_id='KM1:1/f'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[0]C[6]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[0]C[6]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1a/e' ,xlmap_range_id='KM1:1a/e'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[1]C[5]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[1]C[5]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:1a/f' ,xlmap_range_id='KM1:1a/f'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='MATCH C R[1]C[6]:Common Equity Tier 1 (CET1)'; ,xlmap_start='MATCH C R[1]C[6]:Common Equity Tier 1 (CET1)';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:2/a' ,xlmap_range_id='KM1:2/a'
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='ABSOLUTE D10'; ,xlmap_start='ABSOLUTE D10';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:2/b' ,xlmap_range_id='KM1:2/b'
,xlmap_sheet='/3' ,xlmap_sheet='/3'
,xlmap_start='ABSOLUTE E10'; ,xlmap_start='ABSOLUTE E10';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:2/c' ,xlmap_range_id='KM1:2/c'
,xlmap_sheet='/3' ,xlmap_sheet='/3'
,xlmap_start='RELATIVE R[10]C[6]'; ,xlmap_start='RELATIVE R[10]C[6]';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:2/d' ,xlmap_range_id='KM1:2/d'
,xlmap_sheet='/3' ,xlmap_sheet='/3'
,xlmap_start='RELATIVE R[10]C[8]'; ,xlmap_start='RELATIVE R[10]C[8]';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:2/e' ,xlmap_range_id='KM1:2/e'
,xlmap_sheet='/3' ,xlmap_sheet='/3'
,xlmap_start='RELATIVE R[10]C[9]'; ,xlmap_start='RELATIVE R[10]C[9]';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
,xlmap_range_id='KM1:2/f' ,xlmap_range_id='KM1:2/f'
,xlmap_sheet='/3' ,xlmap_sheet='/3'
,xlmap_start='RELATIVE R[10]C[10]'; ,xlmap_start='RELATIVE R[10]C[10]';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
@ -420,7 +430,7 @@ insert into &lib..mpe_excel_map set
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='ABSOLUTE H11' ,xlmap_start='ABSOLUTE H11'
,xlmap_finish='RELATIVE R[0]C[1]'; ,xlmap_finish='RELATIVE R[0]C[1]';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-KM1' ,xlmap_id='BASEL-KM1'
@ -428,7 +438,7 @@ insert into &lib..mpe_excel_map set
,xlmap_sheet='KM1' ,xlmap_sheet='KM1'
,xlmap_start='RELATIVE R[12]C[4]' ,xlmap_start='RELATIVE R[12]C[4]'
,xlmap_finish='ABSOLUTE I13'; ,xlmap_finish='ABSOLUTE I13';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-CR2' ,xlmap_id='BASEL-CR2'
@ -436,7 +446,7 @@ insert into &lib..mpe_excel_map set
,xlmap_sheet='CR2' ,xlmap_sheet='CR2'
,xlmap_start='ABSOLUTE D8' ,xlmap_start='ABSOLUTE D8'
,xlmap_finish='BLANKROW'; ,xlmap_finish='BLANKROW';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='BASEL-CR2' ,xlmap_id='BASEL-CR2'
@ -444,7 +454,7 @@ insert into &lib..mpe_excel_map set
,xlmap_sheet='CR2' ,xlmap_sheet='CR2'
,xlmap_start='ABSOLUTE D18' ,xlmap_start='ABSOLUTE D18'
,xlmap_finish='LASTDOWN'; ,xlmap_finish='LASTDOWN';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='Sample' ,xlmap_id='Sample'
@ -452,7 +462,7 @@ insert into &lib..mpe_excel_map set
,xlmap_sheet='/1' ,xlmap_sheet='/1'
,xlmap_start='ABSOLUTE B3' ,xlmap_start='ABSOLUTE B3'
,xlmap_finish='ABSOLUTE B8'; ,xlmap_finish='ABSOLUTE B8';
insert into &lib..mpe_excel_map set insert into &lib..mpe_xlmap_rules set
tx_from=0 tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,xlmap_id='Sample' ,xlmap_id='Sample'
@ -461,6 +471,8 @@ insert into &lib..mpe_excel_map set
,xlmap_start='ABSOLUTE B13' ,xlmap_start='ABSOLUTE B13'
,xlmap_finish='ABSOLUTE E16'; ,xlmap_finish='ABSOLUTE E16';
/** /**
* MPE_GROUPS * MPE_GROUPS
*/ */
@ -1177,20 +1189,33 @@ insert into &lib..mpe_selectbox set
set tx_from=0 set tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,libref="&lib" ,libref="&lib"
,dsn='MPE_EXCEL_MAP' ,dsn='MPE_XLMAP_INFO'
,num_of_approvals_required=1
,loadtype='TXTEMPORAL'
,var_txfrom='TX_FROM'
,var_txto='TX_TO'
,buskey='XLMAP_ID'
,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
,post_edit_hook='services/hooks/mpe_xlmap_info_postedit'
;
insert into &lib..mpe_tables
set tx_from=0
,tx_to='31DEC5999:23:59:59'dt
,libref="&lib"
,dsn='MPE_XLMAP_RULES'
,num_of_approvals_required=1 ,num_of_approvals_required=1
,loadtype='TXTEMPORAL' ,loadtype='TXTEMPORAL'
,var_txfrom='TX_FROM' ,var_txfrom='TX_FROM'
,var_txto='TX_TO' ,var_txto='TX_TO'
,buskey='XLMAP_ID XLMAP_RANGE_ID' ,buskey='XLMAP_ID XLMAP_RANGE_ID'
,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads' ,notes='Docs: https://docs.datacontroller.io/complex-excel-uploads'
,post_edit_hook='services/hooks/mpe_excel_map_postedit' ,post_edit_hook='services/hooks/mpe_xlmap_rules_postedit'
; ;
insert into &lib..mpe_tables insert into &lib..mpe_tables
set tx_from=0 set tx_from=0
,tx_to='31DEC5999:23:59:59'dt ,tx_to='31DEC5999:23:59:59'dt
,libref="&lib" ,libref="&lib"
,dsn='MPE_EXCEL_UPLOAD' ,dsn='MPE_XLMAP_DATA'
,num_of_approvals_required=1 ,num_of_approvals_required=1
,loadtype='UPDATE' ,loadtype='UPDATE'
,buskey='LOAD_REF XLMAP_ID XLMAP_RANGE_ID ROW_NO COL_NO' ,buskey='LOAD_REF XLMAP_ID XLMAP_RANGE_ID ROW_NO COL_NO'
@ -1472,7 +1497,17 @@ insert into &lib..MPE_VALIDATIONS set
insert into &lib..MPE_VALIDATIONS set insert into &lib..MPE_VALIDATIONS set
tx_from=0 tx_from=0
,base_lib="&lib" ,base_lib="&lib"
,base_ds="MPE_EXCEL_MAP" ,base_ds="MPE_XLMAP_INFO"
,base_col="XLMAP_ID"
,rule_type='CASE'
,rule_value='UPCASE'
,rule_active=1
,tx_to='31DEC5999:23:59:59'dt;
insert into &lib..MPE_VALIDATIONS set
tx_from=0
,base_lib="&lib"
,base_ds="MPE_XLMAP_RULES"
,base_col="XLMAP_ID" ,base_col="XLMAP_ID"
,rule_type='CASE' ,rule_type='CASE'
,rule_value='UPCASE' ,rule_value='UPCASE'

56
sas/sasjs/macros/mpe_makedatamodel.sas
View File

@ -270,24 +270,7 @@ proc datasets lib=&lib noprint;
quit; quit;
proc sql; proc sql;
create table &lib..mpe_excel_map( create table &lib..MPE_XLMAP_DATA(
tx_from num &notnull,
tx_to num &notnull,
XLMAP_ID char(32) &notnull,
XLMAP_RANGE_ID char(32) &notnull,
XLMAP_SHEET char(32) &notnull,
XLMAP_START char(1000) &notnull,
XLMAP_FINISH char(1000)
);quit;
proc datasets lib=&lib noprint;
modify mpe_excel_map;
index create
pk_mpe_excel_map=(tx_to xlmap_id xlmap_range_id)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_excel_upload(
LOAD_REF char(32) &notnull, LOAD_REF char(32) &notnull,
XLMAP_ID char(32) &notnull, XLMAP_ID char(32) &notnull,
XLMAP_RANGE_ID char(32) &notnull, XLMAP_RANGE_ID char(32) &notnull,
@ -296,9 +279,42 @@ create table &lib..mpe_excel_upload(
VALUE_TXT char(4000) VALUE_TXT char(4000)
);quit; );quit;
proc datasets lib=&lib noprint; proc datasets lib=&lib noprint;
modify mpe_excel_upload; modify MPE_XLMAP_DATA;
index create index create
pk_mpe_excel_upload=(load_ref xlmap_id xlmap_range_id row_no col_no) pk_MPE_XLMAP_DATA=(load_ref xlmap_id xlmap_range_id row_no col_no)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_xlmap_info(
tx_from num &notnull,
tx_to num &notnull,
XLMAP_ID char(32) &notnull,
XLMAP_DESCRIPTION char(1000) &notnull,
XLMAP_TARGETLIBDS char(41) &notnull
);quit;
proc datasets lib=&lib noprint;
modify mpe_xlmap_info;
index create
pk_mpe_xlmap_info=(tx_to xlmap_id)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_xlmap_rules(
tx_from num &notnull,
tx_to num &notnull,
XLMAP_ID char(32) &notnull,
XLMAP_RANGE_ID char(32) &notnull,
XLMAP_SHEET char(32) &notnull,
XLMAP_START char(1000) &notnull,
XLMAP_FINISH char(1000),
XLMAP_TARGETLIBDS char(41)
);quit;
proc datasets lib=&lib noprint;
modify mpe_xlmap_rules;
index create
pk_mpe_xlmap_rules=(tx_to xlmap_id xlmap_range_id)
/nomiss unique; /nomiss unique;
quit; quit;

6
sas/sasjs/macros/mpe_xlmapvalidate.sas
View File

@ -15,10 +15,10 @@
%local ismap; %local ismap;
proc sql noprint; proc sql noprint;
select count(*) into: ismap select count(*) into: ismap
from &dclib..MPE_EXCEL_MAP from &dclib..mpe_xlmap_info
where XLMAP_ID="&tgtds"; where XLMAP_TARGETLIBDS="&tgtds" and &dc_dttmtfmt. le TX_TO ;
%if &tgtds=&dclib..MPE_EXCEL_UPLOAD or &ismap>0 %then %do; %if "&tgtds"="&dclib..MPE_XLMAP_DATA" or &ismap>0 %then %do;
data &inds; data &inds;
set &inds; set &inds;
LOAD_REF="&mperef"; LOAD_REF="&mperef";

6
sas/sasjs/macros/mpe_xlmapvalidate.test.sas
View File

@ -8,7 +8,7 @@
@li mp_assertscope.sas @li mp_assertscope.sas
<h4> SAS Includes </h4> <h4> SAS Includes </h4>
@li mpe_excel_upload.ddl ul @li mpe_xlmap_data.ddl ul
@author 4GL Apps Ltd @author 4GL Apps Ltd
@copyright 4GL Apps Ltd. This code may only be used within Data Controller @copyright 4GL Apps Ltd. This code may only be used within Data Controller
@ -23,7 +23,7 @@ proc sql;
%inc ul; %inc ul;
data work.test1; data work.test1;
if 0 then set work.mpe_excel_upload; if 0 then set work.MPE_XLMAP_DATA;
LOAD_REF='0'; LOAD_REF='0';
XLMAP_ID='Sample'; XLMAP_ID='Sample';
XLMAP_RANGE_ID='Range 1'; XLMAP_RANGE_ID='Range 1';
@ -47,7 +47,7 @@ run;
desc=Checking load ref was not applied desc=Checking load ref was not applied
) )
%mpe_xlmapvalidate(DCTEST2,work.test1,&dclib,&dclib..MPE_EXCEL_UPLOAD) %mpe_xlmapvalidate(DCTEST2,work.test1,&dclib,&dclib..MPE_XLMAP_DATA)
data _null_; data _null_;
set work.test1; set work.test1;

3
sas/sasjs/services/auditors/postdata.test.2.sas
View File

@ -34,7 +34,8 @@ run;
%mp_testservice(&_program, %mp_testservice(&_program,
viyacontext=&defaultcontext, viyacontext=&defaultcontext,
inputdatasets=work.sascontroltable work.jsdata, inputdatasets=work.sascontroltable work.jsdata,
outlib=web1 outlib=web1,
mdebug=&sasjs_mdebug
) )
%let status=0; %let status=0;

6
sas/sasjs/services/editors/getdata.sas
View File

@ -537,8 +537,8 @@ run;
%put params; %put params;
%let ismap=0; %let ismap=0;
proc sql noprint; proc sql noprint;
select count(*) into: ismap from &mpelib..mpe_excel_map select count(*) into: ismap from &mpelib..mpe_xlmap_info
where xlmap_id="&orig_libds"; where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
data sasparams; data sasparams;
length colHeaders $20000 filter_text $32767; length colHeaders $20000 filter_text $32767;
@ -557,7 +557,7 @@ data sasparams;
if %mf_nobs(work.cls_rules)=0 then cls_flag=0; if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
else cls_flag=1; else cls_flag=1;
put (_all_)(=); put (_all_)(=);
if "&orig_libds"="&mpelib..MPE_EXCEL_UPLOAD" or &ismap ne 0 then ismap=1; if "&orig_libds"="&mpelib..MPE_XLMAP_DATA" or &ismap ne 0 then ismap=1;
else ismap=0; else ismap=0;
run; run;

34
sas/sasjs/services/editors/getxlmaps.sas
View File

@ -14,7 +14,7 @@
<h5> xlmaprules </h5> <h5> xlmaprules </h5>
Filtered output of the dc.mpe_excel_map table Filtered output of the dc.MPE_XLMAP_RULES table
|XLMAP_ID|XLMAP_RANGE_ID|XLMAP_SHEET|XLMAP_START|XLMAP_FINISH| |XLMAP_ID|XLMAP_RANGE_ID|XLMAP_SHEET|XLMAP_START|XLMAP_FINISH|
|---|---|---|---|---| |---|---|---|---|---|
@ -26,7 +26,7 @@
|TARGET_DS| |TARGET_DS|
|---| |---|
|DCXXX.MPE_EXCEL_UPLOAD| |DCXXX.MPE_XLMAP_DATA|
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mp_abort.sas @li mp_abort.sas
@ -42,6 +42,12 @@
%mpeinit() %mpeinit()
data _null_;
set work.getxlmaps_in;
putlog (_all_)(=);
call symputx('xlmap_id',xlmap_id);
run;
proc sql noprint; proc sql noprint;
create table work.xlmaprules as create table work.xlmaprules as
select xlmap_id select xlmap_id
@ -49,28 +55,24 @@ create table work.xlmaprules as
,XLMAP_SHEET ,XLMAP_SHEET
,XLMAP_START ,XLMAP_START
,XLMAP_FINISH ,XLMAP_FINISH
from &mpelib..mpe_excel_map from &mpelib..MPE_XLMAP_RULES
where &dc_dttmtfmt. lt tx_to where &dc_dttmtfmt. lt tx_to and xlmap_id="&xlmap_id"
and xlmap_id in (select xlmap_id from work.getxlmaps_in)
order by xlmap_sheet, xlmap_range_id; order by xlmap_sheet, xlmap_range_id;
%global target_ds;
select XLMAP_TARGETLIBDS into: target_ds
from &mpelib..MPE_XLMAP_INFO
where &dc_dttmtfmt. lt tx_to and xlmap_id="&xlmap_id";
%mp_abort(iftrue= (&syscc ne 0) %mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas ,mac=&_program..sas
,msg=%str(syscc=&syscc) ,msg=%str(syscc=&syscc)
) )
%let chk=0;
proc sql noprint;
select count(*) into: chk
from &mpelib..mpe_tables
where catx('.',libref,dsn) in (select xlmap_id from work.getxlmaps_in);
data work.xlmapinfo; data work.xlmapinfo;
set work.getxlmaps_in; target_ds=coalescec("&target_ds","&mpelib..MPE_XLMAP_DATA");
length target_ds $32; output;
if &chk=0 then target_ds="&mpelib..MPE_EXCEL_UPLOAD"; stop;
else target_ds=xlmap_id;
keep target_ds;
run; run;
%webout(OPEN) %webout(OPEN)

2
sas/sasjs/services/editors/getxlmaps.test.sas
View File

@ -52,7 +52,7 @@ data work.xlmapinfo;
call symputx('tgtds',target_ds); call symputx('tgtds',target_ds);
run; run;
%mp_assert( %mp_assert(
iftrue=(&tgtds=&dclib..MPE_EXCEL_UPLOAD), iftrue=(&tgtds=&dclib..MPE_XLMAP_DATA),
desc=Checking correct target table is returned, desc=Checking correct target table is returned,
outds=work.test_results outds=work.test_results
) )

1
sas/sasjs/services/editors/stagedata.test.2.sas
View File

@ -24,6 +24,7 @@ proc format lib=DCTEST.DCFMTS cntlout=work.fmtextract;
run; run;
data work.jsdata; data work.jsdata;
set work.fmtextract; set work.fmtextract;
fmtrow=_n_;
if _n_<5 then _____DELETE__THIS__RECORD_____='Yes'; if _n_<5 then _____DELETE__THIS__RECORD_____='Yes';
else _____DELETE__THIS__RECORD_____='No'; else _____DELETE__THIS__RECORD_____='No';
if _n_>20 then stop; if _n_>20 then stop;

70
sas/sasjs/services/hooks/mpe_xlmap_info_postedit.sas Normal file
View File

@ -0,0 +1,70 @@
/**
@file
@brief Post Edit Hook script for the MPE_XLMAP_INFO table
@details Post edit hooks provide additional backend validation for user
provided data. The incoming dataset is named `work.staging_ds` and is
provided in mpe_loader.sas.
Available macro variables:
@li DC_LIBREF - The DC control library
@li LIBREF - The library of the dataset being edited (is assigned)
@li DS - The dataset being edited
<h4> SAS Macros </h4>
@li mf_existds.sas
@li mf_getvarlist.sas
@li mf_wordsinstr1butnotstr2.sas
@li dc_assignlib.sas
@li mp_validatecol.sas
**/
data work.staging_ds;
set work.staging_ds;
/* apply the first excel map to all cells */
length tgtds $41;
retain tgtds;
drop tgtds is_libds;
if _n_=1 then do;
if missing(XLMAP_TARGETLIBDS) or index(XLMAP_TARGETLIBDS,)
then tgtds="&dc_libref..MPE_XLMAP_DATA";
else tgtds=upcase(XLMAP_TARGETLIBDS);
%mp_validatecol(XLMAP_TARGETLIBDS,LIBDS,is_libds)
call symputx('tgtds',tgtds);
call symputx('is_libds',is_libds);
end;
XLMAP_TARGETLIBDS=tgtds;
run;
%mp_abort(iftrue=(&is_libds ne 1)
,mac=Invalid target dataset (&tgtds)
,msg=%superq(errmsg)
)
/**
* make sure that the supplied target dataset exists and
* has the necessary columns
*/
%dc_assignlib(READ,%scan(&tgtds,1,.))
%mp_abort(iftrue=(%mf_existds(libds=work.someview) ne 1)
,mac=Target dataset (&tgtds) could not be opened
,msg=%superq(errmsg)
)
%let tgtvars=%upcase(%mf_getvarlist(&tgtds));
%let srcvars=%upcase(%mf_getvarlist(&dc_libref..MPE_XLMAP_DATA));
%let badvars1=%mf_wordsInStr1ButNotStr2(Str1=&srcvars,Str2=&tgtvars);
%let badvars2=%mf_wordsInStr1ButNotStr2(Str1=&tgtvars,Str2=&srcvars);
%mp_abort(iftrue=(%length(&badvars1.X)>1)
,mac=Target dataset (&tgtds) has missing vars: &badvars1
,msg=%superq(errmsg)
)
%mp_abort(iftrue=(%length(&badvars2.X)>1)
,mac=Target dataset (&tgtds) has unrecognised vars: &badvars2
,msg=%superq(errmsg)
)

8
sas/sasjs/services/hooks/mpe_excel_map_postedit.sas → sas/sasjs/services/hooks/mpe_xlmap_rules_postedit.sas
View File

@ -1,6 +1,6 @@
/** /**
@file @file
@brief Post Edit Hook script for the MPE_EXCEL_MAP table @brief Post Edit Hook script for the MPE_XLMAP_RULES table
@details Post edit hooks provide additional backend validation for user @details Post edit hooks provide additional backend validation for user
provided data. The incoming dataset is named `work.staging_ds` and is provided data. The incoming dataset is named `work.staging_ds` and is
provided in mpe_loader.sas. provided in mpe_loader.sas.
@ -10,10 +10,14 @@
@li LIBREF - The library of the dataset being edited (is assigned) @li LIBREF - The library of the dataset being edited (is assigned)
@li DS - The dataset being edited @li DS - The dataset being edited
**/ **/
/* ensure uppercasing */
data work.staging_ds; data work.staging_ds;
set work.staging_ds; set work.staging_ds;
/* ensure uppercasing */
XLMAP_ID=upcase(XLMAP_ID); XLMAP_ID=upcase(XLMAP_ID);
run; run;

2
sas/sasjs/services/public/startupservice.sas
View File

@ -132,7 +132,7 @@ create table saslibs as
proc sql; proc sql;
create table work.xlmaps as create table work.xlmaps as
select distinct XLMAP_ID select distinct XLMAP_ID
from &mpelib..mpe_excel_map from &mpelib..MPE_XLMAP_RULES
where &dc_dttmtfmt. lt tx_to; where &dc_dttmtfmt. lt tx_to;
%webout(OPEN) %webout(OPEN)