dc/dc
4
1
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 61 Wiki Activity

fix: renaming TABLE macvar to LOAD_REF in postdata.sas
Some checks failed
Build / Build-and-ng-test (pull_request) Failing after 13s
Details

Browse Source 
also adding a sample post approve hook for xlmap dataloads
This commit is contained in:
zmaj
2024-01-18 16:31:11 +00:00
parent 51b043b6d2
commit 01915a2db9
2 changed files with 48 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
sas/sasjs/services/auditors/postdata.sas
View File

@ -56,12 +56,12 @@
data _null_; data _null_;
set work.sascontroltable; set work.sascontroltable;
call symputx('ACTION',ACTION); call symputx('ACTION',ACTION);
call symputx('TABLE',TABLE); call symputx('LOAD_REF',TABLE);
/* DIFFTIME is when the DIFF was generated on the frontend */ /* DIFFTIME is when the DIFF was generated on the frontend */
call symputx('DIFFTIME',DIFFTIME); call symputx('DIFFTIME',DIFFTIME);
run; run;
%global action is_err err_msg; %global action is_err err_msg msg;
%let is_err=0; %let is_err=0;
%let user=%mf_getuser(); %let user=%mf_getuser();
@ -80,7 +80,7 @@ RUN;
%let isfmtcat=0; %let isfmtcat=0;
data APPROVE1; data APPROVE1;
set &mpelib..mpe_submit; set &mpelib..mpe_submit;
where TABLE_ID="&TABLE"; where TABLE_ID="&LOAD_REF";
/* fetch mpe_submit data */ /* fetch mpe_submit data */
libds=cats(base_lib,'.',base_ds); libds=cats(base_lib,'.',base_ds);
REVIEWED_ON=put(reviewed_on_dttm,datetime19.); REVIEWED_ON=put(reviewed_on_dttm,datetime19.);
@ -115,9 +115,9 @@ run;
) )
%mp_abort( %mp_abort(
iftrue=(%mf_verifymacvars(difftime orig_libds libds table)=0) iftrue=(%mf_verifymacvars(difftime orig_libds libds load_ref)=0)
,mac=&_program ,mac=&_program
,msg=%str(Missing: difftime orig_libds libds table) ,msg=%str(Missing: difftime orig_libds libds load_ref)
) )
/* security checks */ /* security checks */
@ -186,7 +186,7 @@ run;
%let prev_upload_check=1; %let prev_upload_check=1;
proc sql; proc sql;
select count(*) into: prev_upload_check from &mpelib..mpe_review select count(*) into: prev_upload_check from &mpelib..mpe_review
where TABLE_ID="&TABLE" and REVIEWED_BY_NM="&user" where TABLE_ID="&LOAD_REF" and REVIEWED_BY_NM="&user"
and REVIEW_STATUS_ID ne "SUBMITTED"; and REVIEW_STATUS_ID ne "SUBMITTED";
%let authcheck=%mf_getattrn(work.authAPP,NLOBS); %let authcheck=%mf_getattrn(work.authAPP,NLOBS);
%if &authcheck=0 or &prev_upload_check=1 %then %do; %if &authcheck=0 or &prev_upload_check=1 %then %do;
@ -233,7 +233,7 @@ run;
%else %let oldloc=%qsysfunc(getoption(LOG)); %else %let oldloc=%qsysfunc(getoption(LOG));
%if %length(&oldloc)>0 %then %do; %if %length(&oldloc)>0 %then %do;
proc printto proc printto
log="&mpelocapprovals/&TABLE/approval.log"; log="&mpelocapprovals/&LOAD_REF/approval.log";
run; run;
data _null_; data _null_;
if _n_=1 then do; if _n_=1 then do;
@ -247,7 +247,7 @@ run;
%end; %end;
%else %do; %else %do;
proc printto proc printto
log="&mpelocapprovals/&TABLE/approval.log"; log="&mpelocapprovals/&LOAD_REF/approval.log";
run; run;
%end; %end;
@ -285,11 +285,11 @@ select PRE_APPROVE_HOOK, POST_APPROVE_HOOK, LOADTYPE, var_txfrom, var_txto
,msg=%str(Missing: mpelocapprovals orig_libds) ,msg=%str(Missing: mpelocapprovals orig_libds)
) )
/* get dataset from approvals location */ /* get dataset from approvals location (has same name as load_ref) */
%let tmplib=%mf_getuniquelibref(); %let tmplib=%mf_getuniquelibref();
libname &tmplib "&mpelocapprovals/&TABLE"; libname &tmplib "&mpelocapprovals/&LOAD_REF";
data STAGING_DS; data STAGING_DS;
set &tmplib..&TABLE; set &tmplib..&LOAD_REF;
run; run;
%mp_abort(iftrue= (&syscc ne 0) %mp_abort(iftrue= (&syscc ne 0)
@ -313,7 +313,7 @@ run;
%let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1); %let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
data work.append_review; data work.append_review;
if 0 then set &mpelib..mpe_review; if 0 then set &mpelib..mpe_review;
TABLE_ID="&TABLE"; TABLE_ID="&LOAD_REF";
BASE_TABLE="&orig_libds"; BASE_TABLE="&orig_libds";
REVIEW_STATUS_ID="APPROVED"; REVIEW_STATUS_ID="APPROVED";
REVIEWED_BY_NM="&user"; REVIEWED_BY_NM="&user";
@ -323,7 +323,7 @@ run;
stop; stop;
run; run;
%mp_lockanytable(LOCK, %mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_review,ref=%str(&table Approval), lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
ctl_ds=&mpelib..mpe_lockanytable ctl_ds=&mpelib..mpe_lockanytable
) )
proc append base=&mpelib..mpe_review data=work.append_review; proc append base=&mpelib..mpe_review data=work.append_review;
@ -335,7 +335,7 @@ run;
/* update mpe_submit table */ /* update mpe_submit table */
%mp_lockanytable(LOCK, %mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval), lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval),
ctl_ds=&mpelib..mpe_lockanytable ctl_ds=&mpelib..mpe_lockanytable
) )
proc sql; proc sql;
@ -343,7 +343,7 @@ run;
set num_of_approvals_remaining=&num_of_approvals_remaining-1, set num_of_approvals_remaining=&num_of_approvals_remaining-1,
reviewed_by_nm="&user", reviewed_by_nm="&user",
reviewed_on_dttm=&sastime reviewed_on_dttm=&sastime
where table_id="&table"; where table_id="&LOAD_REF";
%mp_lockanytable(UNLOCK, %mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_submit, lib=&mpelib,ds=mpe_submit,
ctl_ds=&mpelib..mpe_lockanytable ctl_ds=&mpelib..mpe_lockanytable
@ -369,7 +369,7 @@ run;
) )
%mpe_targetloader(libds=&orig_libds %mpe_targetloader(libds=&orig_libds
,now= &sastime ,now= &sastime
,etlsource=&TABLE ,etlsource=&LOAD_REF
,STAGING_DS=STAGING_DS ,STAGING_DS=STAGING_DS
,dclib=&mpelib ,dclib=&mpelib
%if &action=APPROVE_TABLE %then %do; %if &action=APPROVE_TABLE %then %do;
@ -405,7 +405,7 @@ run;
proc sql noprint; proc sql noprint;
select max(processed_dttm)-1 format=datetime19. into: tstamp select max(processed_dttm)-1 format=datetime19. into: tstamp
from &mpelib..mpe_dataloads from &mpelib..mpe_dataloads
where libref="&libref" and dsn="&ds" and ETLSOURCE="&TABLE"; where libref="&libref" and dsn="&ds" and ETLSOURCE="&LOAD_REF";
quit; quit;
%if &tstamp=. %then %let tstamp=%sysfunc(datetime(),datetime19.); %if &tstamp=. %then %let tstamp=%sysfunc(datetime(),datetime19.);
@ -498,7 +498,7 @@ run;
else if _____orig then _____status='ORIGINAL'; else if _____orig then _____status='ORIGINAL';
run; run;
proc export data=TEMPDIFFS dbms=csv replace proc export data=TEMPDIFFS dbms=csv replace
outfile="&mpelocapprovals/&TABLE/&tempDIFFS_CSV" ; outfile="&mpelocapprovals/&LOAD_REF/&tempDIFFS_CSV" ;
run; run;
proc sql noprint; proc sql noprint;
select filesize format=sizekmg10.1, filesize as filesize_raw select filesize format=sizekmg10.1, filesize as filesize_raw
@ -545,7 +545,7 @@ run;
proc sort data=&mpelib..mpe_submit(where=( proc sort data=&mpelib..mpe_submit(where=(
submit_status_cd='SUBMITTED' submit_status_cd='SUBMITTED'
and cats(base_lib,'.',base_ds)="&orig_libds" and cats(base_lib,'.',base_ds)="&orig_libds"
and table_id ne "&TABLE" and table_id ne "&LOAD_REF"
)) out=submits; )) out=submits;
by descending submitted_on_dttm; by descending submitted_on_dttm;
run; run;
@ -599,7 +599,7 @@ run;
data work.outds_mod; run; data work.outds_mod; run;
data work.outds_del; run; data work.outds_del; run;
%end; %end;
libname approve "&mpelocapprovals/&TABLE"; libname approve "&mpelocapprovals/&LOAD_REF";
data; set &libds;stop;run; data; set &libds;stop;run;
%let emptybasetable=&syslast; %let emptybasetable=&syslast;
data approve.ActualDiffs; data approve.ActualDiffs;
@ -621,7 +621,7 @@ run;
run; run;
proc export data=approve.ActualDiffs proc export data=approve.ActualDiffs
outfile="&mpelocapprovals/&TABLE/ActualDiffs.csv" outfile="&mpelocapprovals/&LOAD_REF/ActualDiffs.csv"
dbms=csv dbms=csv
replace; replace;
run; run;
@ -631,7 +631,7 @@ run;
%let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1); %let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
data work.append_review; data work.append_review;
if 0 then set &mpelib..mpe_review; if 0 then set &mpelib..mpe_review;
TABLE_ID="&TABLE"; TABLE_ID="&LOAD_REF";
BASE_TABLE="&orig_libds"; BASE_TABLE="&orig_libds";
REVIEW_STATUS_ID="APPROVED"; REVIEW_STATUS_ID="APPROVED";
REVIEWED_BY_NM="&user"; REVIEWED_BY_NM="&user";
@ -641,7 +641,7 @@ run;
stop; stop;
run; run;
%mp_lockanytable(LOCK, %mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_review,ref=%str(&table Approval), lib=&mpelib,ds=mpe_review,ref=%str(&LOAD_REF Approval),
ctl_ds=&mpelib..mpe_lockanytable ctl_ds=&mpelib..mpe_lockanytable
) )
proc append base=&mpelib..mpe_review data=work.append_review; proc append base=&mpelib..mpe_review data=work.append_review;
@ -653,7 +653,7 @@ run;
/* update mpe_submit table */ /* update mpe_submit table */
%mp_lockanytable(LOCK, %mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval in auditors/postdata), lib=&mpelib,ds=mpe_submit,ref=%str(&LOAD_REF Approval in auditors/postdata),
ctl_ds=&mpelib..mpe_lockanytable ctl_ds=&mpelib..mpe_lockanytable
) )
proc sql; proc sql;
@ -662,7 +662,7 @@ run;
num_of_approvals_remaining=&num_of_approvals_remaining-1, num_of_approvals_remaining=&num_of_approvals_remaining-1,
reviewed_by_nm="&user", reviewed_by_nm="&user",
reviewed_on_dttm=&sastime reviewed_on_dttm=&sastime
where table_id="&table"; where table_id="&LOAD_REF";
%mp_lockanytable(UNLOCK, %mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_submit, lib=&mpelib,ds=mpe_submit,
ctl_ds=&mpelib..mpe_lockanytable ctl_ds=&mpelib..mpe_lockanytable
@ -688,7 +688,7 @@ run;
%mpe_alerts(alert_event=APPROVED %mpe_alerts(alert_event=APPROVED
, alert_lib=&libref , alert_lib=&libref
, alert_ds=&ds , alert_ds=&ds
, dsid=&TABLE , dsid=&LOAD_REF
) )
%removecolsfromwork(___TMP___MD5) %removecolsfromwork(___TMP___MD5)

22
sas/sasjs/services/hooks/sample_xlmap_data_postapprove.sas Normal file
View File

@ -0,0 +1,22 @@
/**
@file
@brief Sample XLMAP Data hook program (sample_xlmap_data_postapprove)
@details This hook script should NOT be modified in place, as the changes
would be lost in your next Data Controller deployment.
Instead, create a copy of this hook script and place it OUTSIDE the
Data Controller metadata folder.
Available macro variables:
@li LOAD_REF - The Load Reference (unique upload id)
@li ORIG_LIBDS - The target library.dataset that was just loaded
**/
data _null_;
set work.staging_ds;
putlog 'load ref is in the staged data: ' load_ref;
stop;
run;
%put the unique identifier (LOAD_REF) is also a macro variable: &LOAD_REF;