Allan
08e39c4fca
All checks were successful
Build / Build-and-ng-test (pull_request) Successful in 24s
Implemented by provision of the necessary temp tables
229 lines
6.4 KiB
SAS
Executable File
229 lines
6.4 KiB
SAS
Executable File
/**
|
|
@file
|
|
@brief Generic loader for tables configured in &mpelib..mpe_tables
|
|
@details General wrapper for loading tables using ALL load types.
|
|
|
|
Called by admin/postdata. The dclib should already be assigned.
|
|
|
|
@param [in] dclib= (NOTPROVIDED) The data controller control library
|
|
@param [in] libds= the library.dataset to load
|
|
@param [in] now= (%sysfunc(datetime())) Static processed timestamp
|
|
|
|
<h4> SAS Macros </h4>
|
|
@li bitemporal_dataloader.sas
|
|
@li mf_existvar.sas
|
|
@li mp_abort.sas
|
|
@li mp_loadformat.sas
|
|
@li mp_lockanytable.sas
|
|
|
|
@version 9.2
|
|
@author 4GL Apps Ltd
|
|
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
|
|
and may not be re-distributed or re-sold without the express permission of
|
|
4GL Apps Ltd.
|
|
|
|
**/
|
|
|
|
%macro mpe_targetloader(libds= /* library.dataset to LOAD (target) */
|
|
,now= %sysfunc(datetime()) /* static processed timestamp */
|
|
,etlsource= /* process from whence the data came */
|
|
,STAGING_DS= STAGING_DS /* name of staging (work) dataset which should
|
|
be appended into the target. */
|
|
,LOADTARGET=NO /* set to yes to actually load the target */
|
|
,CLOSE_VARS= /* provide close vars to override defaults */
|
|
,dclib=NOTPROVIDED
|
|
,mdebug=0
|
|
,dc_dttmtfmt=E8601DT26.6
|
|
);
|
|
%local lib ds nobs;
|
|
|
|
/**
|
|
* if a format catalog (suffix "-FC") we assume the catalog has already been
|
|
* created by the calling program with a libds of work.fmtextract
|
|
*/
|
|
%let orig_lib=%upcase(%scan(&libds,1,.));
|
|
%let orig_ds=%upcase(%scan(&libds,2,.));
|
|
%let orig_libds=&libds;
|
|
%if %scan(&libds,2,-)=FC %then %do;
|
|
%let lib=WORK;
|
|
%let ds=FMTEXTRACT;
|
|
%let libds=&lib..&ds;
|
|
%end;
|
|
%else %do;
|
|
%let lib=&orig_lib;
|
|
%let ds=&orig_ds;
|
|
%end;
|
|
|
|
%mp_abort(iftrue= (&dclib=NOTPROVIDED)
|
|
,mac=&sysmacroname
|
|
,msg=%str(dclib=NOTPROVIDED)
|
|
)
|
|
|
|
/* get table attributes */
|
|
%let nobs=0;
|
|
data work.sumo_config;
|
|
set &mpelib..mpe_tables;
|
|
where &dc_dttmtfmt. lt tx_to
|
|
and libref="&orig_lib"
|
|
and dsn="&orig_ds";
|
|
call symputx('LOADTYPE',loadtype,'l');
|
|
call symputx('BUSKEY',buskey,'l');
|
|
call symputx('VAR_TXFROM',var_txfrom,'l');
|
|
call symputx('VAR_TXTO',var_txto,'l');
|
|
call symputx('VAR_BUSFROM',var_busfrom,'l');
|
|
call symputx('VAR_BUSTO',var_busto,'l');
|
|
call symputx('VAR_PROCESSED',VAR_PROCESSED,'l');
|
|
call symputx('RK_UNDERLYING',RK_UNDERLYING,'l');
|
|
%if %length(&CLOSE_VARS)=0 %then %do;
|
|
call symputx('CLOSE_VARS',CLOSE_VARS,'l');
|
|
%end;
|
|
call symputx('nobs',_n_,'l');
|
|
if missing(AUDIT_LIBDS) then AUDIT_LIBDS="&dclib..MPE_AUDIT";
|
|
call symputx('AUDIT_LIBDS',AUDIT_LIBDS,'l');
|
|
put (_all_)(=);
|
|
run;
|
|
|
|
/* check if table is actually configured to load */
|
|
%if &nobs ne 1 %then %do;
|
|
proc sql;
|
|
insert into &mpelib..mpe_loads
|
|
set USER_NM="%mf_getuser()"
|
|
,STATUS='FAILED (BAD DS)'
|
|
,CSV_DIR=symget('ETLSOURCE')
|
|
,PROCESSED_DTTM=&now;
|
|
%end;
|
|
%mp_abort(iftrue= (&nobs=0)
|
|
,mac=&sysmacroname
|
|
,msg=%str(Table not registered in &mpelib..mpe_tables)
|
|
)
|
|
%mp_abort(iftrue= (&nobs>1)
|
|
,mac=&sysmacroname
|
|
,msg=%str(Something is very wrong with the PK in &mpelib..mpe_tables)
|
|
)
|
|
|
|
|
|
%if &LOADTYPE=TXTEMPORAL %then %do;
|
|
|
|
%bitemporal_dataloader(bus_from=,bus_to= /* explicitly empty*/
|
|
,tech_from=&VAR_TXFROM
|
|
,tech_to = &VAR_TXTO
|
|
,base_lib=&lib
|
|
,base_dsn=&ds
|
|
,append_lib=WORK
|
|
,append_dsn=&STAGING_DS
|
|
,high_date='31DEC9999:23:59:59'dt
|
|
,PK= &buskey
|
|
,ETLSOURCE=&ETLSOURCE
|
|
,LOADTYPE=&loadtype
|
|
,RK_UNDERLYING=&RK_UNDERLYING
|
|
,LOADTARGET=&LOADTARGET
|
|
,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
|
|
,CLOSE_VARS=&CLOSE_VARS
|
|
,processed=&VAR_PROCESSED
|
|
,dclib=&dclib
|
|
,outds_audit=&AUDIT_LIBDS
|
|
)
|
|
%end;
|
|
%else %if &loadtype=REPLACE %then %do;
|
|
%if &LOADTARGET=YES %then %do;
|
|
%mp_lockanytable(LOCK,lib=&lib,ds=&ds,ref=%str(&etlsource),
|
|
ctl_ds=&dclib..mpe_lockanytable
|
|
)
|
|
data WORK.&STAGING_DS;
|
|
set WORK.&STAGING_DS;
|
|
%if %mf_existvar(&libds,&VAR_PROCESSED) %then %do;
|
|
&VAR_PROCESSED = &now;
|
|
%end;
|
|
drop _____DELETE__THIS__RECORD_____;
|
|
run;
|
|
proc sql; delete * from &libds;
|
|
proc append base=&libds data=WORK.&STAGING_DS force nowarn;run;
|
|
|
|
%mp_lockanytable(UNLOCK,lib=&lib,ds=&ds,ctl_ds=&dclib..mpe_lockanytable)
|
|
%end;
|
|
%else %do;
|
|
/* is full replace so treat everything as a mod in diff screen */
|
|
data work.outds_mod work.outds_add work.outds_del;
|
|
set work.&staging_ds;
|
|
output work.outds_mod; /* _add and _del will be empty */
|
|
run;
|
|
%end;
|
|
%end;
|
|
%else %if &loadtype=UPDATE %then %do;
|
|
%bitemporal_dataloader(bus_from=,bus_to=
|
|
,tech_from= ,tech_to = /* explicitly empty*/
|
|
,base_lib=&lib
|
|
,base_dsn=&ds
|
|
,append_lib=WORK
|
|
,append_dsn=&STAGING_DS
|
|
,high_date='31DEC9999:23:59:59'dt
|
|
,PK= &buskey
|
|
,ETLSOURCE=%superq(etlsource)
|
|
,LOADTYPE=UPDATE
|
|
,RK_UNDERLYING=&RK_UNDERLYING
|
|
,LOADTARGET=&LOADTARGET
|
|
,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
|
|
,processed=&VAR_PROCESSED
|
|
,dclib=&dclib
|
|
,outds_audit=&AUDIT_LIBDS
|
|
)
|
|
%end;
|
|
%else %if &loadtype=FORMAT_CAT %then %do;
|
|
/**
|
|
* run mp_formatload
|
|
* inputs:
|
|
* - LOADTARGET
|
|
* - CATALOG
|
|
* - STAGEDATA
|
|
* - LOADAUDIT
|
|
* outputs:
|
|
* work.outds_add
|
|
* work.outds_del
|
|
* work.outds_mod
|
|
*/
|
|
%mp_loadformat(&orig_libds
|
|
,&staging_ds
|
|
,loadtarget=&LOADTARGET
|
|
,auditlibds=&AUDIT_LIBDS
|
|
,locklibds=&dclib..mpe_lockanytable
|
|
,delete_col=_____DELETE__THIS__RECORD_____
|
|
,outds_add=outds_add
|
|
,outds_del=outds_del
|
|
,outds_mod=outds_mod
|
|
,mdebug=&mdebug
|
|
)
|
|
%end;
|
|
%else %if &loadtype=BITEMPORAL %then %do;
|
|
%bitemporal_dataloader(bus_from=&VAR_BUSFROM,bus_to=&VAR_BUSTO
|
|
,tech_from=&VAR_TXFROM
|
|
,tech_to = &VAR_TXTO
|
|
,base_lib=&lib
|
|
,base_dsn=&ds
|
|
,append_lib=WORK
|
|
,append_dsn=&STAGING_DS
|
|
,high_date='31DEC9999:23:59:59'dt
|
|
,PK= &buskey
|
|
,ETLSOURCE=%superq(etlsource)
|
|
,LOADTYPE=BITEMPORAL
|
|
,RK_UNDERLYING=&RK_UNDERLYING
|
|
,LOADTARGET=&LOADTARGET
|
|
,RK_UPDATE_MAXKEYTABLE=&LOADTARGET
|
|
,CLOSE_VARS=&CLOSE_VARS
|
|
,processed=&VAR_PROCESSED
|
|
,dclib=&dclib
|
|
,outds_audit=&AUDIT_LIBDS
|
|
)
|
|
%end;
|
|
%else %do;
|
|
%put WARNING: LOADTYPE &LOADTYPE not supported;
|
|
%let syscc=4;
|
|
%mp_abort(msg=LOADTYPE &LOADTYPE not supported,mac=mpe_targetloader.sas)
|
|
%end;
|
|
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program
|
|
,msg=%str(syscc=&syscc exiting MPE_TARGETLOADER macro)
|
|
)
|
|
|
|
%mend mpe_targetloader;
|