dc/dc
dc
/
dc
2
1
Fork 0
Code Issues 7 Pull Requests 1 Actions Packages Projects Releases 27 Wiki Activity

fix: renaming TABLE macvar to LOAD_REF in postdata.sas
Build / Build-and-ng-test (pull_request) Failing after 13s Details 

also adding a sample post approve hook for xlmap dataloads
This commit is contained in:
zmaj 2024-01-18 16:31:11 +00:00
parent 51b043b6d2
commit 01915a2db9
2 changed files with 48 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
sas/sasjs/services/auditors/postdata.sas
View File

@ -56,12 +56,12 @@
data _null_;
set work.sascontroltable;
call symputx('ACTION',ACTION);
call symputx('TABLE',TABLE);
call symputx('LOAD_REF',TABLE);
/* DIFFTIME is when the DIFF was generated on the frontend */
call symputx('DIFFTIME',DIFFTIME);
run;
%global action is_err err_msg;
%global action is_err err_msg msg;
%let is_err=0;
%let user=%mf_getuser();
@ -80,7 +80,7 @@ RUN;
%let isfmtcat=0;
data APPROVE1;
set &mpelib..mpe_submit;
where TABLE_ID="&TABLE";
where TABLE_ID="&LOAD_REF";
/* fetch mpe_submit data */
libds=cats(base_lib,'.',base_ds);
REVIEWED_ON=put(reviewed_on_dttm,datetime19.);
@ -115,9 +115,9 @@ run;
)
%mp_abort(
iftrue=(%mf_verifymacvars(difftime orig_libds libds table)=0)
iftrue=(%mf_verifymacvars(difftime orig_libds libds load_ref)=0)
,mac=&_program
,msg=%str(Missing: difftime orig_libds libds table)
,msg=%str(Missing: difftime orig_libds libds load_ref)
)
/* security checks */
@ -186,7 +186,7 @@ run;
%let prev_upload_check=1;
proc sql;
select count(*) into: prev_upload_check from &mpelib..mpe_review
where TABLE_ID="&TABLE" and REVIEWED_BY_NM="&user"
where TABLE_ID="&LOAD_REF" and REVIEWED_BY_NM="&user"
and REVIEW_STATUS_ID ne "SUBMITTED";
%let authcheck=%mf_getattrn(work.authAPP,NLOBS);
%if &authcheck=0 or &prev_upload_check=1 %then %do;
@ -233,7 +233,7 @@ run;
%else %let oldloc=%qsysfunc(getoption(LOG));
%if %length(&oldloc)>0 %then %do;
proc printto
log="&mpelocapprovals/&TABLE/approval.log";
log="&mpelocapprovals/&LOAD_REF/approval.log";
run;
data _null_;
if _n_=1 then do;
@ -247,7 +247,7 @@ run;
%end;
%else %do;
proc printto
log="&mpelocapprovals/&TABLE/approval.log";
log="&mpelocapprovals/&LOAD_REF/approval.log";
run;
%end;
@ -285,11 +285,11 @@ select PRE_APPROVE_HOOK, POST_APPROVE_HOOK, LOADTYPE, var_txfrom, var_txto
,msg=%str(Missing: mpelocapprovals orig_libds)
)
/* get dataset from approvals location */
/* get dataset from approvals location (has same name as load_ref) */
%let tmplib=%mf_getuniquelibref();
libname &tmplib "&mpelocapprovals/&TABLE";
libname &tmplib "&mpelocapprovals/&LOAD_REF";
data STAGING_DS;
set &tmplib..&TABLE;
set &tmplib..&LOAD_REF;
run;
%mp_abort(iftrue= (&syscc ne 0)
@ -313,7 +313,7 @@ run;
%let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
data work.append_review;
if 0 then set &mpelib..mpe_review;
TABLE_ID="&TABLE";
TABLE_ID="&LOAD_REF";
BASE_TABLE="&orig_libds";
REVIEW_STATUS_ID="APPROVED";
REVIEWED_BY_NM="&user";
@ -323,7 +323,7 @@ run;
stop;
run;
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
ctl_ds=&mpelib..mpe_lockanytable
)
proc append base=&mpelib..mpe_review data=work.append_review;
@ -335,7 +335,7 @@ run;
/* update mpe_submit table */
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval),
lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval),
ctl_ds=&mpelib..mpe_lockanytable
)
proc sql;
@ -343,7 +343,7 @@ run;
set num_of_approvals_remaining=&num_of_approvals_remaining-1,
reviewed_by_nm="&user",
reviewed_on_dttm=&sastime
where table_id="&table";
where table_id="&LOAD_REF";
%mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_submit,
ctl_ds=&mpelib..mpe_lockanytable
@ -369,7 +369,7 @@ run;
)
%mpe_targetloader(libds=&orig_libds
,now= &sastime
,etlsource=&TABLE
,etlsource=&LOAD_REF
,STAGING_DS=STAGING_DS
,dclib=&mpelib
%if &action=APPROVE_TABLE %then %do;
@ -405,7 +405,7 @@ run;
proc sql noprint;
select max(processed_dttm)-1 format=datetime19. into: tstamp
from &mpelib..mpe_dataloads
where libref="&libref" and dsn="&ds" and ETLSOURCE="&TABLE";
where libref="&libref" and dsn="&ds" and ETLSOURCE="&LOAD_REF";
quit;
%if &tstamp=. %then %let tstamp=%sysfunc(datetime(),datetime19.);
@ -498,7 +498,7 @@ run;
else if _____orig then _____status='ORIGINAL';
run;
proc export data=TEMPDIFFS dbms=csv replace
outfile="&mpelocapprovals/&TABLE/&tempDIFFS_CSV" ;
outfile="&mpelocapprovals/&LOAD_REF/&tempDIFFS_CSV" ;
run;
proc sql noprint;
select filesize format=sizekmg10.1, filesize as filesize_raw
@ -545,7 +545,7 @@ run;
proc sort data=&mpelib..mpe_submit(where=(
submit_status_cd='SUBMITTED'
and cats(base_lib,'.',base_ds)="&orig_libds"
and table_id ne "&TABLE"
and table_id ne "&LOAD_REF"
)) out=submits;
by descending submitted_on_dttm;
run;
@ -599,7 +599,7 @@ run;
data work.outds_mod; run;
data work.outds_del; run;
%end;
libname approve "&mpelocapprovals/&TABLE";
libname approve "&mpelocapprovals/&LOAD_REF";
data; set &libds;stop;run;
%let emptybasetable=&syslast;
data approve.ActualDiffs;
@ -621,7 +621,7 @@ run;
run;
proc export data=approve.ActualDiffs
outfile="&mpelocapprovals/&TABLE/ActualDiffs.csv"
outfile="&mpelocapprovals/&LOAD_REF/ActualDiffs.csv"
dbms=csv
replace;
run;
@ -631,7 +631,7 @@ run;
%let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
data work.append_review;
if 0 then set &mpelib..mpe_review;
TABLE_ID="&TABLE";
TABLE_ID="&LOAD_REF";
BASE_TABLE="&orig_libds";
REVIEW_STATUS_ID="APPROVED";
REVIEWED_BY_NM="&user";
@ -641,7 +641,7 @@ run;
stop;
run;
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
ctl_ds=&mpelib..mpe_lockanytable
)
proc append base=&mpelib..mpe_review data=work.append_review;
@ -653,7 +653,7 @@ run;
/* update mpe_submit table */
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval in auditors/postdata),
lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval in auditors/postdata),
ctl_ds=&mpelib..mpe_lockanytable
)
proc sql;
@ -662,7 +662,7 @@ run;
num_of_approvals_remaining=&num_of_approvals_remaining-1,
reviewed_by_nm="&user",
reviewed_on_dttm=&sastime
where table_id="&table";
where table_id="&LOAD_REF";
%mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_submit,
ctl_ds=&mpelib..mpe_lockanytable
@ -688,7 +688,7 @@ run;
%mpe_alerts(alert_event=APPROVED
, alert_lib=&libref
, alert_ds=&ds
, dsid=&TABLE
, dsid=&LOAD_REF
)
%removecolsfromwork(___TMP___MD5)

22
sas/sasjs/services/hooks/sample_xlmap_data_postapprove.sas Normal file
View File

@ -0,0 +1,22 @@
/**
@file
@brief Sample XLMAP Data hook program (sample_xlmap_data_postapprove)
@details This hook script should NOT be modified in place, as the changes
would be lost in your next Data Controller deployment.
Instead, create a copy of this hook script and place it OUTSIDE the
Data Controller metadata folder.
Available macro variables:
@li LOAD_REF - The Load Reference (unique upload id)
@li ORIG_LIBDS - The target library.dataset that was just loaded
**/
data _null_;
set work.staging_ds;
putlog 'load ref is in the staged data: ' load_ref;
stop;
run;
%put the unique identifier (LOAD_REF) is also a macro variable: &LOAD_REF;