fix(refresh): one audit folder per refresh call
refreshcatalog.test and viewtables.test failed to complete with ERROR: could not create /tmp/DataController/<libref>/dc_staging/refreshcatalog:mpe_refreshcatalogs: + <stamp> The staging folder a closeout writes its statements to is &dc_staging_area/&etlsource, and etlsource defaulted to 'refreshcatalog:<step>:%mf_uid()'. A default is resolved when the macro is compiled, not when it is called, so every refresh in a session reused the same folder - the first step created it, the next one could not, and the abort took the whole suite down. Whichever suite reached the staging area first decided which others failed, which is why the count moved between runs. etlsource now defaults to 0 and is set per call from %mf_getuniquename(prefix=<step>): unique, a plain name rather than a path, and resolved at the point of use. Verified on the estate: refreshcatalog 4/4 (100%), viewtables 2/2 (100%).
This commit is contained in:
1 parent
fd272eec89
commit
56b6620ef4
3 files changed
+21
-10
No files matched your search
@@ -3,14 +3,15 @@
|
|||||||
@brief Refreshes the data catalog
|
@brief Refreshes the data catalog
|
||||||
@details Assumes library is already assigned.
|
@details Assumes library is already assigned.
|
||||||
|
|
||||||
@param [in] etlsource identifies this load in the MPE_DATALOADS audit row.
|
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshcatalogs); it
|
||||||
|
identifies this load in the MPE_DATALOADS audit row.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
|
|
||||||
%mpe_refreshcatalogs(sashelp)
|
%mpe_refreshcatalogs(sashelp)
|
||||||
|
|
||||||
<h4> SAS Macros </h4>
|
<h4> SAS Macros </h4>
|
||||||
@li mf_uid.sas
|
@li mf_getuniquename.sas
|
||||||
@li mp_abort.sas
|
@li mp_abort.sas
|
||||||
@li bitemporal_dataloader.sas
|
@li bitemporal_dataloader.sas
|
||||||
|
|
||||||
@@ -20,11 +21,11 @@
|
|||||||
**/
|
**/
|
||||||
|
|
||||||
%macro mpe_refreshcatalogs(lib,cat=#all
|
%macro mpe_refreshcatalogs(lib,cat=#all
|
||||||
,etlsource=refreshcatalog:mpe_refreshcatalogs:%mf_uid()
|
,etlsource=0
|
||||||
);
|
);
|
||||||
%let lib=%upcase(&lib);
|
%let lib=%upcase(&lib);
|
||||||
%let cat=%upcase(&cat);
|
%let cat=%upcase(&cat);
|
||||||
%if "&etlsource"="0" %then %let etlsource=refreshcatalog:%sysmacroname:%mf_uid();
|
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshcatalogs);
|
||||||
|
|
||||||
%put running &sysmacroname &lib for &cat;
|
%put running &sysmacroname &lib for &cat;
|
||||||
proc sql;
|
proc sql;
|
||||||
|
|||||||
@@ -3,12 +3,12 @@
|
|||||||
@brief Refreshes the library data catalog
|
@brief Refreshes the library data catalog
|
||||||
@details
|
@details
|
||||||
|
|
||||||
@param [in] etlsource= (refreshcatalog:<step>:%mf_uid()) identifies this load
|
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshlibs), identifies this load
|
||||||
in the MPE_DATALOADS audit row. That table is keyed on etlsource together
|
in the MPE_DATALOADS audit row. That table is keyed on etlsource together
|
||||||
with processed_dttm, libref and dsn, so the uid keeps each load distinct.
|
with processed_dttm, libref and dsn, so the uid keeps each load distinct.
|
||||||
|
|
||||||
<h4> SAS Macros </h4>
|
<h4> SAS Macros </h4>
|
||||||
@li mf_uid.sas
|
@li mf_getuniquename.sas
|
||||||
@li dc_getlibs.sas
|
@li dc_getlibs.sas
|
||||||
@li mf_getplatform.sas
|
@li mf_getplatform.sas
|
||||||
@li bitemporal_dataloader.sas
|
@li bitemporal_dataloader.sas
|
||||||
@@ -22,8 +22,13 @@
|
|||||||
**/
|
**/
|
||||||
|
|
||||||
%macro mpe_refreshlibs(lib=0
|
%macro mpe_refreshlibs(lib=0
|
||||||
,etlsource=refreshcatalog:mpe_refreshlibs:%mf_uid()
|
,etlsource=0
|
||||||
);
|
);
|
||||||
|
/* a default of 0 rather than a %mf_uid() expression: a default is
|
||||||
|
resolved when the macro is compiled, so it would be one value for the
|
||||||
|
whole session and the audit folder below it would collide on the
|
||||||
|
second call - as it did between the refresh steps of one run */
|
||||||
|
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshlibs);
|
||||||
|
|
||||||
%dc_getlibs(outds=work.mm_getLibs)
|
%dc_getlibs(outds=work.mm_getLibs)
|
||||||
proc sort data=mm_getlibs;
|
proc sort data=mm_getlibs;
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
@brief Refreshes the data catalog
|
@brief Refreshes the data catalog
|
||||||
@details
|
@details
|
||||||
|
|
||||||
@param [in] etlsource= (refreshcatalog:<step>:%mf_uid()) identifies this load
|
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshtables), identifies this load
|
||||||
in the MPE_DATALOADS audit row. That table is keyed on etlsource together
|
in the MPE_DATALOADS audit row. That table is keyed on etlsource together
|
||||||
with processed_dttm, libref and dsn, so the uid keeps each load distinct.
|
with processed_dttm, libref and dsn, so the uid keeps each load distinct.
|
||||||
Usage:
|
Usage:
|
||||||
@@ -11,7 +11,7 @@
|
|||||||
%mpe_refreshtables(sashelp)
|
%mpe_refreshtables(sashelp)
|
||||||
|
|
||||||
<h4> SAS Macros </h4>
|
<h4> SAS Macros </h4>
|
||||||
@li mf_uid.sas
|
@li mf_getuniquename.sas
|
||||||
@li mp_abort.sas
|
@li mp_abort.sas
|
||||||
@li bitemporal_dataloader.sas
|
@li bitemporal_dataloader.sas
|
||||||
@li mf_existfeature.sas
|
@li mf_existfeature.sas
|
||||||
@@ -24,8 +24,13 @@
|
|||||||
**/
|
**/
|
||||||
|
|
||||||
%macro mpe_refreshtables(lib,ds=#all
|
%macro mpe_refreshtables(lib,ds=#all
|
||||||
,etlsource=refreshcatalog:mpe_refreshtables:%mf_uid()
|
,etlsource=0
|
||||||
);
|
);
|
||||||
|
/* a default of 0 rather than a %mf_uid() expression: a default is
|
||||||
|
resolved when the macro is compiled, so it would be one value for the
|
||||||
|
whole session and the audit folder below it would collide on the
|
||||||
|
second call - as it did between the refresh steps of one run */
|
||||||
|
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshtables);
|
||||||
%let lib=%upcase(&lib);
|
%let lib=%upcase(&lib);
|
||||||
%let ds=%upcase(&ds);
|
%let ds=%upcase(&ds);
|
||||||
%local engine; %let engine=%mf_getengine(&lib);
|
%local engine; %let engine=%mf_getengine(&lib);
|
||||||
|
|||||||
Reference in new issue
Block a user