dc/sas/sasjs/services/auditors/postdata.sas
Allan 08e39c4fca
All checks were successful
Build / Build-and-ng-test (pull_request) Successful in 24s
fix: re-enabling full REPLACE uploads
Implemented by provision of the necessary temp tables
2023-08-24 00:09:25 +01:00

712 lines
19 KiB
SAS

/**
@file postdata.sas
@brief Either returns the file diffs or actually loads the data to target
@details Before loading the target, a check is made against the time the
target was last updated (backend) and the time the DIFF was generated
(frontend). If the target was updated whilst the DIFF was on the screen,
then the provided diff may have been incorrect and so a new DIFF should be
generated and approved before load.
Only 100 rows (of each DIFF type) are displayed on the DIFF screen.
<h4> Service Inputs </h4>
<h5> SASCONTROLTABLE </h5>
|ACTION:$char10.|TABLE:$char32.|DIFFTIME:$char29.|
|---|---|---|
|SHOW_DIFFS|DC20220208T142124517_124703_1184|"Tue, 08 Feb 2022 14:23:05 GMT"|
<h4> SAS Macros </h4>
@li bitemporal_dataloader.sas
@li dc_assignlib.sas
@li mf_existds.sas
@li mf_existvar.sas
@li mf_getattrn.sas
@li mf_getengine.sas
@li mf_getquotedstr.sas
@li mf_getuniquelibref.sas
@li mf_getuser.sas
@li mf_getvarlist.sas
@li mf_nobs.sas
@li mf_verifymacvars.sas
@li mp_abort.sas
@li mp_cntlout.sas
@li mp_lockanytable.sas
@li mpe_accesscheck.sas
@li mpe_alerts.sas
@li mpe_runhook.sas
@li mpe_targetloader.sas
@li removecolsfromwork.sas
@version 9.2
@author 4GL Apps Ltd
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
and may not be re-distributed or re-sold without the express permission of
4GL Apps Ltd.
**/
/* this could be a config setting if required */
%let maxdiff=100;
%mpeinit()
/* load parameters */
data _null_;
set work.sascontroltable;
call symputx('ACTION',ACTION);
call symputx('TABLE',TABLE);
/* DIFFTIME is when the DIFF was generated on the frontend */
call symputx('DIFFTIME',DIFFTIME);
run;
%global action is_err err_msg;
%let is_err=0;
%let user=%mf_getuser();
%let sastime=%sysfunc(datetime());
data sastime;
dt_sastime=&sastime;
run;
PROC FORMAT;
picture yymmddhhmmss other='%0Y-%0m-%0d %0H:%0M:%0S' (datatype=datetime);
picture flatdate other='%0Y%0m%0d_%0H%0M%0S' (datatype=datetime);
RUN;
/* SHOW_DIFFS works by getting the temp tables from the bitemporal loader */
/* so we share much of the logic from the actual load process */
%let isfmtcat=0;
data APPROVE1;
set &mpelib..mpe_submit;
where TABLE_ID="&TABLE";
/* fetch mpe_submit data */
libds=cats(base_lib,'.',base_ds);
REVIEWED_ON=put(reviewed_on_dttm,datetime19.);
call symputx('REVIEW_STATUS_ID',submit_status_cd,'l');
call symputx('NUM_OF_APPROVALS_REQUIRED',NUM_OF_APPROVALS_REQUIRED);
call symputx('num_of_approvals_remaining',num_of_approvals_remaining);
/* other stuff that's useful to do in data step */
call symputx('orig_libds',libds);
call symputx('libds',libds);
if substr(cats(reverse(libds)),1,3)=:'CF-' then do;
libds=scan(libds,1,'-');
putlog "Format Catalog Captured";
call symputx('isfmtcat',1);
libds='work.fmtextract';
call symputx('libds',libds);
end;
putlog (_all_)(=);
/* convert provided string DIFFTIME back to a numeric SAS datetime */
if "&action" ne "SHOW_DIFFS" then do;
call symputx('DIFFTIME',input(symget('DIFFTIME'),anydtdtm18.));
end;
length difftime $32;
DIFFTIME=put(&sastime,datetime19.2);
run;
%mp_cntlout(
iftrue=(&isfmtcat=1)
,libcat=&orig_libds
,fmtlist=0
,cntlout=work.fmtextract
)
%mp_abort(
iftrue=(%mf_verifymacvars(difftime orig_libds libds table)=0)
,mac=&_program
,msg=%str(Missing: difftime orig_libds libds table)
)
/* security checks */
%mpe_accesscheck(&orig_libds,outds=authEDIT,user=&user,access_level=EDIT)
%mpe_accesscheck(&orig_libds,outds=authAPP,user=&user,access_level=APPROVE)
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program
,msg=%str(syscc=&syscc Before entering postdata macro)
)
%mp_abort(
iftrue=(
%mf_getattrn(work.authEDIT,NLOBS)=0 & %mf_getattrn(work.authAPP,NLOBS)=0
)
,mac=&_program
,msg=%str(&user not authorised to view approval screen for &orig_libds)
)
%macro quickmacro(inds,outds);
data &outds ;
%if %length(&VAR_BUSFROM)>0 %then %do;
format &VAR_BUSFROM &VAR_BUSTO yymmddhhmmss.;
%end;
if 0 then set &emptybasetable;
set &inds;
%if %mf_existvar(&libds,&var_txfrom) %then %do;
drop &var_txfrom &var_txto;
%end;
%if %mf_existvar(&inds,_____DELETE__THIS__RECORD_____) %then %do;
drop _____DELETE__THIS__RECORD_____;
%end;
%if %mf_existvar(&inds,&VAR_PROCESSED) %then %do;
drop &VAR_PROCESSED;
%end;
run;
%mend quickmacro;
%macro postdata();
%if %quote(&REVIEW_STATUS_ID)=%quote(REJECTED)
or %quote(&REVIEW_STATUS_ID)=%quote(APPROVED) %then
%do;
data params; set approve1; run;
%webout(OPEN)
%webout(OBJ,PARAMS)
%webout(CLOSE)
%return;
%end;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc)
)
%if &action=APPROVE_TABLE %then %do;
/* check user is authorised to approve table */
/* user could be an editor but not an approver */
%mp_abort(iftrue= (%mf_getattrn(work.authAPP,NLOBS)=0)
,mac=&_program
,msg=%str(&user may not APPROVE changes)
)
/* see if this user has already submitted an approval */
%let prev_upload_check=1;
proc sql;
select count(*) into: prev_upload_check from &mpelib..mpe_review
where TABLE_ID="&TABLE" and REVIEWED_BY_NM="&user"
and REVIEW_STATUS_ID ne "SUBMITTED";
%let authcheck=%mf_getattrn(work.authAPP,NLOBS);
%if &authcheck=0 or &prev_upload_check=1 %then %do;
%put WARNING: authcheck=&authcheck prev_upload_check=&prev_upload_check;
data apPARAMS;
AUTHORISED=&authcheck;
PREV_UPLOAD_CHECK=&prev_upload_check;
run;
%webout(OPEN)
%webout(OBJ,apPARAMS);
%webout(CLOSE)
%return;
%end;
/* now check if table has been updated since DIFF screen shown */
%local fmt_tm usernm last_load etlsource;
%let last_load=0;
proc sql noprint;
select max(processed_dttm) format=16.2 into: last_load
from &mpelib..mpe_dataloads
where libref="%scan(&orig_libds,1,.)" and dsn="%scan(&orig_libds,2,.)";
select processed_dttm format=datetime19., user_nm, etlsource
into: fmt_tm, :usernm, :etlsource
from &mpelib..mpe_dataloads
where libref="%scan(&orig_libds,1,.)" and dsn="%scan(&orig_libds,2,.)"
and processed_dttm=&last_load;
%put TIMECHECK: &last_load>&difftime;
%if %sysevalf(&last_load>&difftime,boolean)=1 %then %do;
%let is_err=1;
%let err_msg=&orig_libds was updated in batch %trim(&etlsource
) by %trim(&usernm) on &fmt_tm - please refresh the page!!;
%return;
%end;
%if &syscc ne 0 %then %do;
%let is_err=1;
%let err_msg=syscc=&syscc before logchange;
%return;
%end;
/* upload about to commence so ensure logs */
options notes mprint source2;
%local oldloc;
%if %symexist(SYSPRINTTOLOG) %then %let oldloc=&SYSPRINTTOLOG;
%else %let oldloc=%qsysfunc(getoption(LOG));
%if %length(&oldloc)>0 %then %do;
proc printto
log="&mpelocapprovals/&TABLE/approval.log";
run;
data _null_;
if _n_=1 then do;
length oldloc $1000;
oldloc=symget('oldloc');
putlog "****** redirected:" oldloc " *****";
end;
infile &oldloc;
input; putlog _infile_;
run;
%end;
%else %do;
proc printto
log="&mpelocapprovals/&TABLE/approval.log";
run;
%end;
%if &syscc ne 0 %then %do;
%let is_err=1;
%let err_msg=syscc=&syscc after logchange;
%return;
%end;
%end;
/**
* upload the actual table
*/
%local libref ds;
%let libref=%scan(&orig_libds,1,.);
%let ds=%scan(&orig_libds,2,.);
proc sql noprint;
select PRE_APPROVE_HOOK, POST_APPROVE_HOOK, LOADTYPE, var_txfrom, var_txto
,BUSKEY, VAR_BUSFROM, VAR_BUSTO
,AUDIT_LIBDS, NOTES, coalesce(NUM_OF_APPROVALS_REQUIRED,1)
,VAR_PROCESSED
into: PRE_APPROVE_HOOK, :POST_APPROVE_HOOK, :LOADTYPE,:var_txfrom,:var_txto
,:BUSKEY,:VAR_BUSFROM,:VAR_BUSTO
,:AUDIT_LIBDS, :TABLE_DESC, :NUM_OF_APPROVALS_REQUIRED_TOT
,:VAR_PROCESSED
from &mpelib..mpe_tables
where &dc_dttmtfmt. lt tx_to
and libref="&libref"
and dsn="&ds";
%mp_abort(
iftrue=(%mf_verifymacvars(mpelocapprovals orig_libds)=0)
,mac=&_program
,msg=%str(Missing: mpelocapprovals orig_libds)
)
/* get dataset from approvals location */
%let tmplib=%mf_getuniquelibref();
libname &tmplib "&mpelocapprovals/&TABLE";
data STAGING_DS;
set &tmplib..&TABLE;
run;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc before preapprove)
)
%dc_assignlib(WRITE,&libref)
/* run pre-approve hook - occurs both BEFORE _and_ AFTER the diff */
%mpe_runhook(PRE_APPROVE_HOOK)
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc after preapprove)
)
%if &num_of_approvals_remaining>1 and &action=APPROVE_TABLE %then %do;
/* append to mpe_review table */
%let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
data work.append_review;
if 0 then set &mpelib..mpe_review;
TABLE_ID="&TABLE";
BASE_TABLE="&orig_libds";
REVIEW_STATUS_ID="APPROVED";
REVIEWED_BY_NM="&user";
REVIEWED_ON_DTTM=&sastime;
REVIEW_REASON_TXT="APPROVAL &apprno of &num_of_approvals_required";
output;
stop;
run;
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
ctl_ds=&mpelib..mpe_lockanytable
)
proc append base=&mpelib..mpe_review data=work.append_review;
run;
%mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_review,
ctl_ds=&mpelib..mpe_lockanytable
)
/* update mpe_submit table */
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval),
ctl_ds=&mpelib..mpe_lockanytable
)
proc sql;
update &mpelib..mpe_submit
set num_of_approvals_remaining=&num_of_approvals_remaining-1,
reviewed_by_nm="&user",
reviewed_on_dttm=&sastime
where table_id="&table";
%mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_submit,
ctl_ds=&mpelib..mpe_lockanytable
)
data apReqd;
AUTHORISED=1;
ALREADY_UPDATED=0;
ALREADY_UPDATED_DTTM=.;
set approve1; /* js will test for NUM_OF_APPROVALS_REQUIRED */
run;
%removecolsfromwork(___TMP___MD5)
%webout(OPEN)
%webout(OBJ,apReqd);
%webout(CLOSE)
%return;
%end;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc entering TARGETLOADER)
)
%mpe_targetloader(libds=&orig_libds
,now= &sastime
,etlsource=&TABLE
,STAGING_DS=STAGING_DS
,dclib=&mpelib
%if &action=APPROVE_TABLE %then %do;
,LOADTARGET=YES
%end;
%else %do;
,LOADTARGET=NO
%end;
,dc_dttmtfmt=&dc_dttmtfmt.
)
%if %mf_getattrn(STAGING_DS,NLOBS)=0 %then %do;
/* empty dataset! */
data out;
set STAGING_DS;
run;
%return;
%end;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc entering SHOWDIFFS)
)
%if &action=SHOW_DIFFS %then %do;
/**
* Now prepare the SHOW DIFFS (approve) screen
*/
/*To create the CURRENT diffs, we compare with the ACTUAL data. But first
need to find out what version TIME to query it for.. */
proc sql noprint;
select max(processed_dttm)-1 format=datetime19. into: tstamp
from &mpelib..mpe_dataloads
where libref="&libref" and dsn="&ds" and ETLSOURCE="&TABLE";
quit;
%if &tstamp=. %then %let tstamp=%sysfunc(datetime(),datetime19.);
/**
* now create the DIFFS dataset
* If using a database, then utilise pass through!
* Create a temporary table inside the database for joins..
*/
options mprint;
%let engine_type=%mf_getEngine(%scan(&libds,1,.));
%put &libds engine type = &engine_type;
%local inner_table ;
%if &engine_type=OLEDB %then %do;
/* generate a unique ID for the temporary table */
data _null_;
call symputx('UNIQUE_REF'
,cats(round(datetime(),1)
,'_'
,round(ranuni(0)*100000,1)
)
,'l'
);
run;
%let inner_table=&libref.."##DIFF_&UNIQUE_REF"n;
proc sql;
create table &inner_table as
select * from work.outds_mod;
%end;
%else %let inner_table=work.outds_mod;
proc sql;
create view work.originals2 as
select b.*
from &inner_table a
inner join &libds
%if &loadtype=BITEMPORAL or &loadtype=TXTEMPORAL %then %do;
(where=("&tstamp"dt < &VAR_TXTO))
%end;
b
on 1
%do idx_pk=1 %to %sysfunc(countw(&buskey));
%let idx_val=%scan(&buskey,&idx_pk);
and a.&idx_val=b.&idx_val
%end;
order by %mf_getquotedstr(in_str=&buskey,dlm=%str(,),quote=)
;
create view bitemp5c_updates2 as
select * from work.outds_mod
order by %mf_getquotedstr(in_str=&buskey,dlm=%str(,),quote=)
;
data; set &libds;stop;run;
%let emptybasetable=&syslast;
options varlenchk=nowarn; /* for small numerics (<8) */
%quickmacro(work.outds_del,deleted)
%quickmacro(work.outds_add,new)
%quickmacro(bitemp5c_updates2,updates)
%quickmacro(originals2,originals)
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc in quickmacro)
)
/* extract colnames for md5 creation / change tracking */
proc contents noprint data=work.updates
out=cols (keep=name type length varnum format);
run;
proc sort data=cols out=cols(drop=varnum); by varnum;run;
data cols; set cols; name=upcase(name);run;
%let tempDIFFS_CSV=tempDiffs_%trim(
%sysfunc(datetime(),flatdate.)).csv;
/**
* Store temp tables so we have a record of diffs
* do not change this libname or table name as it is used in some
* post approve hooks
*/
data TEMPDIFFS (compress=no) /* for realistic file size */;
length _____status $10;
set work.deleted (in=_____del)
work.new(in=_____new)
work.updates (in=_____upd)
work.originals2 (in=_____orig);
if _____del then _____status='DELETED ';
else if _____new then _____status='NEW';
else if _____upd then _____status='UPDATED';
else if _____orig then _____status='ORIGINAL';
run;
proc export data=TEMPDIFFS dbms=csv replace
outfile="&mpelocapprovals/&TABLE/&tempDIFFS_CSV" ;
run;
proc sql noprint;
select filesize format=sizekmg10.1, filesize as filesize_raw
into: filesize,:filesize_raw
from dictionary.tables
where libname='WORK' and memtype='DATA' and memname='TEMPDIFFS';
data params;
set approve1;
DIFFS_CSV="&tempDIFFS_CSV";
FILESIZE="&filesize";
FILESIZE_RAW=&filesize_raw;
if %mf_nobs(work.originals)>&maxdiff
or %mf_nobs(work.new)>&maxdiff
or %mf_nobs(work.deleted)>&maxdiff
or %mf_nobs(work.updates)>&maxdiff
then TRUNCATED="YES";
else TRUNCATED="NO";
NUM_ADDED=%mf_getattrn(work.new,NLOBS);
NUM_DELETED=%mf_getattrn(work.deleted,NLOBS);
NUM_UPDATED=%mf_getattrn(work.updates,NLOBS);
SUBMITTED_ON=put(submitted_on_dttm,datetime19.);
%if %mf_getattrn(work.authAPP,NLOBS)>0 %then %do;
ISAPPROVER='YES';
%end;
%else %do;
ISAPPROVER='NO';
%end;
run;
/*
* The PRE_APPROVE_HOOK may have applied custom formats to the staged table.
* To ensure consistency in the DIFF screen, we should apply the same formats
* to the base table. Limit rows at the same time.
*/
data work.originals;
if 0 then set deleted new updates;
set work.originals;
if _n_>&maxdiff then stop;
run;
/* get additional submits against the same base table */
proc sort data=&mpelib..mpe_submit(where=(
submit_status_cd='SUBMITTED'
and cats(base_lib,'.',base_ds)="&orig_libds"
and table_id ne "&TABLE"
)) out=submits;
by descending submitted_on_dttm;
run;
/* filter last 10 */
data submits;
set submits;
if _n_>10 then stop;
run;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc SHOWDIFFS prior to streamout)
)
%removecolsfromwork(___TMP___MD5)
%webout(OPEN)
%webout(OBJ,params)
%webout(OBJ,cols)
%webout(OBJ,submits)
%webout(OBJ,deleted,fmt=N,missing=STRING,maxobs=&maxdiff)
%webout(OBJ,new,fmt=N,missing=STRING,maxobs=&maxdiff)
%webout(OBJ,updates,fmt=N,missing=STRING,maxobs=&maxdiff)
%webout(OBJ,ORIGINALS,fmt=N,missing=STRING)
/* need same for formatted view */
%webout(OBJ,deleted,dslabel=fmt_deleted,fmt=Y,missing=STRING,maxobs=&maxdiff)
%webout(OBJ,new,dslabel=fmt_new,fmt=Y,missing=STRING,maxobs=&maxdiff)
%webout(OBJ,updates,dslabel=fmt_updates,fmt=Y,missing=STRING,maxobs=&maxdiff)
%webout(OBJ,originals,dslabel=fmt_ORIGINALS,fmt=Y,missing=STRING)
%webout(CLOSE)
%if &engine_type=OLEDB %then %do;
proc sql; /* needs to be dropped AFTER view execution */
drop table &inner_table;
%end;
%return;
%end;
%if &action=APPROVE_TABLE %then %do;
%approve:
/**
* store temp tables so we have a record of diffs
* do not change this libname or table name as it is used in some
* post approve hooks
* for REPLACE loads, temp tables not made, so make them
*/
%if &LOADTYPE=REPLACE %then %do;
data work.outds_add; run;
data work.outds_mod; run;
data work.outds_del; run;
%end;
libname approve "&mpelocapprovals/&TABLE";
data; set &libds;stop;run;
%let emptybasetable=&syslast;
data approve.ActualDiffs;
length _____STATUS_____ $10;
if 0 then set &emptybasetable;
set work.outds_del (in=_____del)
work.outds_add (in=_____new)
work.outds_mod (in=_____upd);
if _____del then _____STATUS_____='DELETED';
else if _____new then _____STATUS_____='NEW';
else if _____upd then _____STATUS_____='UPDATED';
%if %mf_existvar(&libds,&var_txfrom) %then %do;
drop &var_txfrom &var_txto;
%end;
%if %mf_existvar(&libds,&VAR_PROCESSED) %then %do;
drop &VAR_PROCESSED;
%end;
run;
proc export data=approve.ActualDiffs
outfile="&mpelocapprovals/&TABLE/ActualDiffs.csv"
dbms=csv
replace;
run;
/* update the control table to show table as approved */
/* append to mpe_review table */
%let apprno=%eval(&num_of_approvals_required-&num_of_approvals_remaining+1);
data work.append_review;
if 0 then set &mpelib..mpe_review;
TABLE_ID="&TABLE";
BASE_TABLE="&orig_libds";
REVIEW_STATUS_ID="APPROVED";
REVIEWED_BY_NM="&user";
REVIEWED_ON_DTTM=&sastime;
REVIEW_REASON_TXT="APPROVAL &apprno of &num_of_approvals_required";
output;
stop;
run;
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_review,ref=%str(&table Approval),
ctl_ds=&mpelib..mpe_lockanytable
)
proc append base=&mpelib..mpe_review data=work.append_review;
run;
%mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_review,
ctl_ds=&mpelib..mpe_lockanytable
)
/* update mpe_submit table */
%mp_lockanytable(LOCK,
lib=&mpelib,ds=mpe_submit,ref=%str(&table Approval in auditors/postdata),
ctl_ds=&mpelib..mpe_lockanytable
)
proc sql;
update &mpelib..mpe_submit
set submit_status_cd='APPROVED',
num_of_approvals_remaining=&num_of_approvals_remaining-1,
reviewed_by_nm="&user",
reviewed_on_dttm=&sastime
where table_id="&table";
%mp_lockanytable(UNLOCK,
lib=&mpelib,ds=mpe_submit,
ctl_ds=&mpelib..mpe_lockanytable
)
/* run post-approve hook */
%mpe_runhook(POST_APPROVE_HOOK)
data apPARAMS;
AUTHORISED=1;
ALREADY_UPDATED=0;
ALREADY_UPDATED_DTTM=.;
DIFFTIME="&difftime";
if &syscc=0 then RESPONSE='SUCCESS!';
else response="SYSCC=&syscc.";
run;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program 582
,msg=%superq(msg)
)
%mpe_alerts(alert_event=APPROVED
, alert_lib=&libref
, alert_ds=&ds
, dsid=&TABLE
)
%removecolsfromwork(___TMP___MD5)
%webout(OPEN)
%webout(OBJ,apPARAMS)
%webout(CLOSE)
%return;
%end;
%mend postdata;
%postdata()
%mp_abort(mode=INCLUDE)
%mp_abort(iftrue= (&is_err=1)
,mac=&_program
,msg=%superq(err_msg)
)
%mpeterm()