fix(refresh): one audit folder per refresh call
Build / Build-and-ng-test (pull_request) Successful in 5m34s
Build / Build-and-test-development (pull_request) Canceled after 0s
Lighthouse Checks / lighthouse (pull_request) Canceled after 18m57s

refreshcatalog.test and viewtables.test failed to complete with

  ERROR: could not create /tmp/DataController/<libref>/dc_staging/refreshcatalog:mpe_refreshcatalogs: + <stamp>

The staging folder a closeout writes its statements to is
&dc_staging_area/&etlsource, and etlsource defaulted to
'refreshcatalog:<step>:%mf_uid()'. A default is resolved when the macro is
compiled, not when it is called, so every refresh in a session reused the same
folder - the first step created it, the next one could not, and the abort took
the whole suite down. Whichever suite reached the staging area first decided
which others failed, which is why the count moved between runs.

etlsource now defaults to 0 and is set per call from
%mf_getuniquename(prefix=<step>): unique, a plain name rather than a path, and
resolved at the point of use.

Verified on the estate: refreshcatalog 4/4 (100%), viewtables 2/2 (100%).
This commit is contained in:
dc committed 2026-10-27 23:08:47 +00:00
1 parent fd272eec89
commit 56b6620ef4
3 files changed
+21 -10

No files matched your search

+5 -4
View File
@@ -3,14 +3,15 @@
@brief Refreshes the data catalog
@details Assumes library is already assigned.
@param [in] etlsource identifies this load in the MPE_DATALOADS audit row.
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshcatalogs); it
identifies this load in the MPE_DATALOADS audit row.
Usage:
%mpe_refreshcatalogs(sashelp)
<h4> SAS Macros </h4>
@li mf_uid.sas
@li mf_getuniquename.sas
@li mp_abort.sas
@li bitemporal_dataloader.sas
@@ -20,11 +21,11 @@
**/
%macro mpe_refreshcatalogs(lib,cat=#all
,etlsource=refreshcatalog:mpe_refreshcatalogs:%mf_uid()
,etlsource=0
);
%let lib=%upcase(&lib);
%let cat=%upcase(&cat);
%if "&etlsource"="0" %then %let etlsource=refreshcatalog:%sysmacroname:%mf_uid();
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshcatalogs);
%put running &sysmacroname &lib for &cat;
proc sql;
+8 -3
View File
@@ -3,12 +3,12 @@
@brief Refreshes the library data catalog
@details
@param [in] etlsource= (refreshcatalog:<step>:%mf_uid()) identifies this load
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshlibs), identifies this load
in the MPE_DATALOADS audit row. That table is keyed on etlsource together
with processed_dttm, libref and dsn, so the uid keeps each load distinct.
<h4> SAS Macros </h4>
@li mf_uid.sas
@li mf_getuniquename.sas
@li dc_getlibs.sas
@li mf_getplatform.sas
@li bitemporal_dataloader.sas
@@ -22,8 +22,13 @@
**/
%macro mpe_refreshlibs(lib=0
,etlsource=refreshcatalog:mpe_refreshlibs:%mf_uid()
,etlsource=0
);
/* a default of 0 rather than a %mf_uid() expression: a default is
resolved when the macro is compiled, so it would be one value for the
whole session and the audit folder below it would collide on the
second call - as it did between the refresh steps of one run */
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshlibs);
%dc_getlibs(outds=work.mm_getLibs)
proc sort data=mm_getlibs;
+8 -3
View File
@@ -3,7 +3,7 @@
@brief Refreshes the data catalog
@details
@param [in] etlsource= (refreshcatalog:<step>:%mf_uid()) identifies this load
@param [in] etlsource= (0) Defaults to %mf_getuniquename(prefix=refreshtables), identifies this load
in the MPE_DATALOADS audit row. That table is keyed on etlsource together
with processed_dttm, libref and dsn, so the uid keeps each load distinct.
Usage:
@@ -11,7 +11,7 @@
%mpe_refreshtables(sashelp)
<h4> SAS Macros </h4>
@li mf_uid.sas
@li mf_getuniquename.sas
@li mp_abort.sas
@li bitemporal_dataloader.sas
@li mf_existfeature.sas
@@ -24,8 +24,13 @@
**/
%macro mpe_refreshtables(lib,ds=#all
,etlsource=refreshcatalog:mpe_refreshtables:%mf_uid()
,etlsource=0
);
/* a default of 0 rather than a %mf_uid() expression: a default is
resolved when the macro is compiled, so it would be one value for the
whole session and the audit folder below it would collide on the
second call - as it did between the refresh steps of one run */
%if "&etlsource"="0" %then %let etlsource=%mf_getuniquename(prefix=refreshtables);
%let lib=%upcase(&lib);
%let ds=%upcase(&ds);
%local engine; %let engine=%mf_getengine(&lib);