dc
56b6620ef4
refreshcatalog.test and viewtables.test failed to complete with ERROR: could not create /tmp/DataController/<libref>/dc_staging/refreshcatalog:mpe_refreshcatalogs: + <stamp> The staging folder a closeout writes its statements to is &dc_staging_area/&etlsource, and etlsource defaulted to 'refreshcatalog:<step>:%mf_uid()'. A default is resolved when the macro is compiled, not when it is called, so every refresh in a session reused the same folder - the first step created it, the next one could not, and the abort took the whole suite down. Whichever suite reached the staging area first decided which others failed, which is why the count moved between runs. etlsource now defaults to 0 and is set per call from %mf_getuniquename(prefix=<step>): unique, a plain name rather than a path, and resolved at the point of use. Verified on the estate: refreshcatalog 4/4 (100%), viewtables 2/2 (100%).
151 lines
2.9 KiB
SAS
151 lines
2.9 KiB
SAS
/**
|
|
@file
|
|
@brief Refreshes the data catalog
|
|
@details Assumes library is already assigned.
|
|
|
|
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshcatalogs); it
|
|
identifies this load in the MPE_DATALOADS audit row.
|
|
|
|
Usage:
|
|
|
|
%mpe_refreshcatalogs(sashelp)
|
|
|
|
<h4> SAS Macros </h4>
|
|
@li mf_getuniquename.sas
|
|
@li mp_abort.sas
|
|
@li bitemporal_dataloader.sas
|
|
|
|
@version 9.3
|
|
@author 4GL Apps Ltd
|
|
|
|
**/
|
|
|
|
%macro mpe_refreshcatalogs(lib,cat=#all
|
|
,etlsource=0
|
|
);
|
|
%let lib=%upcase(&lib);
|
|
%let cat=%upcase(&cat);
|
|
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshcatalogs);
|
|
|
|
%put running &sysmacroname &lib for &cat;
|
|
proc sql;
|
|
create table work.catdata as
|
|
select libname as libref,
|
|
memname,
|
|
objname,
|
|
objtype,
|
|
objdesc,
|
|
created,
|
|
modified,
|
|
alias,
|
|
level
|
|
from dictionary.catalogs
|
|
where upcase(libname)="&lib"
|
|
%if &cat ne #ALL %then %do;
|
|
and upcase(memname)="&cat"
|
|
%end;
|
|
;
|
|
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program
|
|
,msg=%str(syscc=&syscc afer &lib objects extraction)
|
|
)
|
|
|
|
/* load mpe_datacatalog_CATS */
|
|
proc sql;
|
|
create table datacats as select distinct libref,memname from catdata;
|
|
|
|
%bitemporal_dataloader(base_lib=&mpelib
|
|
,base_dsn=mpe_datacatalog_CATS
|
|
,append_dsn=datacats
|
|
,PK=LIBREF MEMNAME
|
|
,etlsource=&etlsource
|
|
,loadtype=TXTEMPORAL
|
|
,tech_from=TX_FROM
|
|
,tech_to=TX_TO
|
|
%if &cat = #ALL %then %do;
|
|
,close_vars=LIBREF
|
|
%end;
|
|
,dclib=&mpelib
|
|
)
|
|
|
|
/* load mpe_datacatalog_objsS */
|
|
proc sql;
|
|
create table dataobjs as
|
|
select distinct libref,
|
|
memname,
|
|
objname,
|
|
objtype,
|
|
objdesc,
|
|
alias
|
|
from catdata;
|
|
quit;
|
|
|
|
%bitemporal_dataloader(base_lib=&mpelib
|
|
,base_dsn=mpe_datacatalog_OBJS
|
|
,append_dsn=dataobjs
|
|
,PK=LIBREF MEMNAME OBJNAME OBJTYPE
|
|
,etlsource=&etlsource
|
|
,loadtype=TXTEMPORAL
|
|
,tech_from=TX_FROM
|
|
,tech_to=TX_TO
|
|
%if &cat = #ALL %then %do;
|
|
,close_vars=LIBREF MEMNAME
|
|
%end;
|
|
,dclib=&mpelib
|
|
)
|
|
|
|
%put load mpe_datastatus_OBJS;
|
|
proc sql;
|
|
create table statusobjs as
|
|
select distinct libref,
|
|
memname,
|
|
objname,
|
|
objtype,
|
|
created,
|
|
modified,
|
|
level
|
|
from catdata;
|
|
|
|
%bitemporal_dataloader(base_lib=&mpelib
|
|
,base_dsn=mpe_datastatus_OBJS
|
|
,append_dsn=statusobjs
|
|
,PK=LIBREF MEMNAME OBJNAME OBJTYPE
|
|
,etlsource=&etlsource
|
|
,loadtype=TXTEMPORAL
|
|
,tech_from=TX_FROM
|
|
,tech_to=TX_TO
|
|
%if &cat = #ALL %then %do;
|
|
,close_vars=LIBREF MEMNAME
|
|
%end;
|
|
,dclib=&mpelib
|
|
)
|
|
|
|
%put load mpe_datastatus_cats;
|
|
proc sql;
|
|
create table statuscats as
|
|
select libref,
|
|
memname,
|
|
count(*) as nobjs,
|
|
min(created) as created,
|
|
max(modified) as modified
|
|
from catdata
|
|
group by 1,2;
|
|
|
|
%bitemporal_dataloader(base_lib=&mpelib
|
|
,base_dsn=mpe_datastatus_cats
|
|
,append_dsn=statuscats
|
|
,PK=LIBREF MEMNAME
|
|
,etlsource=&etlsource
|
|
,loadtype=TXTEMPORAL
|
|
,tech_from=TX_FROM
|
|
,tech_to=TX_TO
|
|
%if &cat = #ALL %then %do;
|
|
,close_vars=LIBREF
|
|
%end;
|
|
,dclib=&mpelib
|
|
)
|
|
|
|
|
|
%mend mpe_refreshcatalogs;
|