dc/dc
5
1 Fork 0
Code Issues 12 Pull Requests 2 Actions Packages Projects 1 Releases 102 Wiki Activity

revert(derived): no shadowing, and no self reference

Browse Source 
Allowing a derived variable to reuse a physical column's name was wrong: an
intra-row calculation (SUMMER = SUMMER + 1) implies a retained value, which
depends on the order rows are read.  The same argument rules out a variable
listing itself as a dependency.

Reverted:
- the name collision is an error again ('VAR_NM collides with a column in the
  target table')
- VAR_DEPS may not list the variable itself
- the physical column list is back to mf_getvarlist - the type/length check
  only existed to support shadowing
- the droplist guards in stagedata / viewdata / getrawdata, and the duplicate
  suppression in getdata, are no longer needed
- the docs and both test suites drop the shadowing case
This commit is contained in:
dc committed 2026-10-20 23:00:01 +00:00
1 parent 6a1128a463
commit 5af8e6fc35
7 files changed
+39 -120
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

sas/sasjs/macros/mpe_derived.sas
+14 -47
View File
@@ -34,11 +34,8 @@
REGION = <expression>;
Type and length are declared by the administrator, so the result is
deterministic, and a derived variable may shadow a physical column - the
expression then reads the physical value, eg SUMMER = SUMMER + 1 - in which
case VAR_TYPE and VAR_LEN must match the column, as attrib cannot change
either on an existing variable. The label is the variable name, an equals
sign and the expression.
deterministic. The label is the variable name, an equals sign and the
expression.
VAR_DEPS lists every input the expression uses - physical columns and
derived variables alike. The statements are generated in dependency order,
@@ -51,14 +48,16 @@
variable defined for it.
Semicolons, macro triggers and comments are rejected, so a row can only ever
be an expression.
be an expression. A derived variable may not reuse the name of a physical
column, and may not reference itself: an intra-row calculation would imply a
retained value, which depends on the order rows happen to arrive in.
<h4> SAS Macros </h4>
@li mf_existds.sas
@li mf_getuniquename.sas
@li mf_getvarlist.sas
@li mf_nobs.sas
@li mp_abort.sas
@li mp_getcols.sas
@param [in] libds= The target table (libref.dataset, lib must be assigned)
@param [in] dclib= The Data Controller control library
@@ -84,7 +83,7 @@
%local physlist derived_err resv_list i w left nready nplaced
cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
ds_cfg ds_byname ds_bad ds_baddep ds_dup ds_dep ds_ref ds_stale
ds_undecl ds_self ds_shadow ds_cols ds_edge ds_todo ds_ready ds_placed ds_srt;
ds_undecl ds_self ds_edge ds_todo ds_ready ds_placed ds_srt;
/* every working table gets its own name, so a caller with tables of the same
name in WORK cannot collide with this macro's intermediates */
@@ -98,8 +97,6 @@
%let ds_stale=%mf_getuniquename(prefix=dvstl);
%let ds_undecl=%mf_getuniquename(prefix=dvund);
%let ds_self=%mf_getuniquename(prefix=dvsfl);
%let ds_shadow=%mf_getuniquename(prefix=dvshw);
%let ds_cols=%mf_getuniquename(prefix=dvcol);
%let ds_edge=%mf_getuniquename(prefix=dvedg);
%let ds_todo=%mf_getuniquename(prefix=dvtd);
%let ds_ready=%mf_getuniquename(prefix=dvrdy);
@@ -157,18 +154,7 @@ proc sort data=work.&ds_cfg. out=work.&ds_byname.;
by name;
run;
/* one contents pass gives the names, types and lengths. The type and length
matter when a derived variable shadows a physical column. */
%mp_getcols(&libds, outds=work.&ds_cols)
%let physlist=;
proc sql noprint;
select quote(name) into: physlist separated by ','
from work.&ds_cols;
quit;
/* a table with no columns still has to compile into a valid in-list */
%if %length(&physlist)=0 %then %let physlist=%str(%'%');
%let physlist=%upcase(&physlist);
%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
/* names that may never be used - DC's own metadata columns */
%let resv_list=%str("_____DELETE__THIS__RECORD_____","TX_FROM","TX_TO","PROCESSED_DTTM","VAR_TXFROM","VAR_TXTO","VAR_PROCESSED");
@@ -187,6 +173,10 @@ data work.&ds_bad.;
then reason='VAR_NM must be a valid SAS name';
else if verify(substr(name,1,1),'0123456789')=0
then reason='VAR_NM must not start with a digit';
%if %length(&physlist)>0 %then %do;
else if name in (%unquote(&physlist))
then reason='VAR_NM collides with a column in the target table';
%end;
else if name in (%unquote(&resv_list))
then reason='VAR_NM is reserved';
else if type not in ('C','N')
@@ -288,19 +278,7 @@ create table work.&ds_undecl. as
/* a variable may not depend on itself */
create table work.&ds_self. as
select name, dep_nm from work.&ds_dep.
where name=dep_nm
and name not in (%unquote(&physlist));
/* a derived variable may shadow a physical column, but the two must agree on
type and length - attrib cannot change either on an existing variable */
create table work.&ds_shadow. as
select c.name, c.type, c.len, p.type as ptype, p.length as plen
from work.&ds_byname. c
inner join work.&ds_cols. p
on c.name=p.name
where c.type ne p.type
or (c.type='C' and c.len gt p.length)
or (c.type='N' and c.len ne p.length);
where name=dep_nm;
quit;
%if %mf_nobs(work.&ds_baddep.)>0 %then %do;
@@ -313,16 +291,6 @@ quit;
%mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
%end;
%if %mf_nobs(work.&ds_shadow.)>0 %then %do;
data _null_;
set work.&ds_shadow.(obs=1);
call symputx('derived_err'
,cats(name,': VAR_NM shadows a column of the table, so VAR_TYPE and'
,' VAR_LEN must match it - the column is ',ptype,'(',plen,')'));
run;
%mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
%end;
%if %mf_nobs(work.&ds_self.)>0 %then %do;
data _null_;
set work.&ds_self.(obs=1);
@@ -376,8 +344,7 @@ proc sql;
create table work.&ds_edge. as
select d.name as child, d.dep_nm as parent
from work.&ds_dep. d
where d.dep_nm in (select name from work.&ds_byname.)
and d.dep_nm ne d.name;
where d.dep_nm in (select name from work.&ds_byname.);
quit;
data work.&ds_todo.(keep=name);
sas/sasjs/macros/mpe_derived.test.sas
-43
View File
@@ -233,49 +233,6 @@ run;
outds=work.test_results
)
/**
* Test 4 - a derived variable may shadow a physical column
*/
data work.class_orig;
set sashelp.class;
run;
proc sql;
insert into &dclib..mpe_derived set
tx_from=0
,tx_to='31DEC5999:23:59:59'dt
,libref='WORK'
,dsn='CLASS'
,var_nm='AGE'
,var_deps='AGE'
,var_type='N'
,var_len=8
,var_fmt=''
,var_logic='AGE + 1'
,active=1;
quit;
%mpe_derived(libds=work.class,
dclib=&dclib,
outref=dercode4,
outview=work.vw_derived4,
outvars=work.derived_vars4
)
data work.chk4shadow;
merge work.vw_derived4(keep=name age rename=(age=age_shadow))
work.class_orig(keep=name age);
by name;
if age_shadow ne age + 1 then output;
run;
%mp_assertdsobs(work.chk4shadow,
desc=Test4 - a derived variable can shadow a physical column,
test=EQUALS 0,
outds=work.test_results
)
/* tidy up so repeated runs start clean */
proc sql;
delete from &dclib..mpe_derived where upcase(libref)='WORK';
sas/sasjs/services/editors/getdata.sas
+1 -5
View File
@@ -544,14 +544,10 @@ select upcase(loadtype)
%if &derived_nobs>0 %then %do;
%mp_getcols(work.vw_derived, outds=derived_cols_all)
proc sql;
/* a derived variable may shadow a physical column - that column is already
in cols1, and adding it again would duplicate it in the grid. The
physical row keeps its label and format; the value comes from the view. */
create table work.derived_cols as
select a.*
from work.derived_cols_all a
where a.NAME in (select NAME from work.derived_vars)
and a.NAME not in (select NAME from work.cols1);
where a.NAME in (select NAME from work.derived_vars);
quit;
data work.cols1;
set work.cols1 work.derived_cols;
sas/sasjs/services/editors/stagedata.sas
+1 -6
View File
@@ -20,7 +20,6 @@
@li mf_verifymacvars.sas
@li mf_mkdir.sas
@li mf_getuniquefileref.sas
@li mf_getvarlist.sas
@li mpe_loader.sas
@li mpe_filtermaster.sas
@li mpe_derived.sas
@@ -121,15 +120,11 @@ run;
%if %mf_nobs(work.derived_vars)>0 %then %do;
proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
run;
/* a derived variable may shadow a physical column - that column is still
editable, so it must not be dropped */
%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
proc sql noprint;
select a.name into: derived_droplist separated by ' '
from work.derived_vars a
inner join work.jsdata_vars b
on a.name=b.name
where a.name not in (%unquote(&physlist));
on a.name=b.name;
quit;
%end;
sas/sasjs/services/hooks/mpe_derived_postedit.sas
+21 -7
View File
@@ -18,9 +18,9 @@
VAR_DEPS must list every input the expression uses, physical columns
and derived variables alike. The hook checks the part that does not need
the target library: a derived variable referenced by an expression must be
declared, and the resulting graph must be acyclic - across the incoming
rows and the current rows they do not replace. A self reference is allowed,
since it may legitimately read a physical column of the same name.
declared, a variable may not depend on itself, and the resulting graph must
be acyclic - across the incoming rows and the current rows they do not
replace.
The target table is not assigned in this context, so the collision between
VAR_NM and a physical column, and the resolution of a physical dependency,
@@ -165,6 +165,11 @@ data work.derived_ref;
run;
proc sql;
create table work.derived_self as
select libref, dsn, name, dep_nm
from work.derived_dep
where name=dep_nm;
create table work.derived_undeclared as
select r.libref, r.dsn, r.name, r.tok as dep_nm
from work.derived_ref r
@@ -180,6 +185,18 @@ create table work.derived_undeclared as
);
quit;
%if %mf_nobs(work.derived_self)>0 %then %do;
data _null_;
set work.derived_self(obs=1);
call symputx('errmsg'
,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
run;
%mp_abort(iftrue=1
,mac=mpe_derived_postedit
,msg=%superq(errmsg)
)
%end;
%if %mf_nobs(work.derived_undeclared)>0 %then %do;
data _null_;
set work.derived_undeclared(obs=1);
@@ -198,13 +215,10 @@ quit;
* been placed; a round that places nothing leaves a cycle
*/
proc sql;
/* a self reference is not a graph edge - it reads the physical column of the
same name, which the macro validates where the target library is assigned */
create table work.derived_edge as
select d.libref, d.dsn, d.name as child, d.dep_nm as parent
from work.derived_dep d
where d.dep_nm in (select name from work.derived_rows)
and d.dep_nm ne d.name;
where d.dep_nm in (select name from work.derived_rows);
quit;
data work.derived_todo;
sas/sasjs/services/public/getrawdata.sas
+1 -6
View File
@@ -22,7 +22,6 @@
@li mp_streamfile.sas
@li mp_validatecol.sas
@li mf_nobs.sas
@li mf_getvarlist.sas
@li mpe_derived.sas
@li mpe_validatecol.sas
@li mpe_filtermaster.sas
@@ -159,13 +158,9 @@ run;
%let derived_droplist=;
%if &derived_nobs>0 %then %do;
%let filterds=work.vw_derived;
/* a derived variable may shadow a physical column - that column carries
real data, so it stays in the output */
%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
proc sql noprint;
select name into: derived_droplist separated by ' '
from work.derived_vars
where name not in (%unquote(&physlist));
from work.derived_vars;
quit;
%end;
%else %let filterds=&libds;
sas/sasjs/services/public/viewdata.sas
+1 -6
View File
@@ -59,7 +59,6 @@
@li mp_searchdata.sas
@li mp_validatecol.sas
@li mpe_columnlevelsecurity.sas
@li mf_getvarlist.sas
@li mpe_derived.sas
@li mpe_dsmeta.sas
@li mpe_getversions.sas
@@ -233,13 +232,9 @@ run;
%let derived_droplist=;
%if &derived_nobs>0 %then %do;
%let filterds=work.vw_derived;
/* a derived variable may shadow a physical column - that column carries
real data, so it stays in the output */
%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
proc sql noprint;
select name into: derived_droplist separated by ' '
from work.derived_vars
where name not in (%unquote(&physlist));
from work.derived_vars;
quit;
%end;
%else %let filterds=&libds;