From b0fd354ecdbd7bf29196ed983ff3cce691affeed Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Mon, 19 Oct 2026 22:58:19 +0000
Subject: [PATCH 01/48] feat(editor): derived variables in the EDIT screen

Adds user-defined computed variables, configured per table in the new
MPE_DERIVED control table. A derived variable is declared with a name,
type, length, optional format and a SAS expression.

The expression is applied in a DATA step view, and the filter is applied
by the step that reads the view, so the variables exist before the
filter is evaluated and can therefore be filtered on. A WHERE clause
cannot see a variable created in the same data step, hence the view.

- getdata: derived columns are added to the COLS payload (read only,
  labelled DERIVED: <expression>) and to the data returned to the grid
- getcolvals: values are taken from the derived view, so filter dropdowns
  work for derived columns
- validatefilter: the filter is validated against the view, and stored
  against the physical table (needs mp_filterstore targetds=)
- stagedata: derived columns are dropped before the approval package
- mpe_makedata / mpe_makedatamodel / ddl / migration: new MPE_DERIVED
  table, registered in MPE_TABLES

Values are display-time only - nothing derived is ever written back.
---
 sas/sasjs/db/datactrl/mpe_derived.ddl         |  30 +++
 sas/sasjs/db/datactrl/mpe_tables.ddl          |  12 +
 .../db/migrations/20261019_v7.19_release.sas  |  55 +++++
 sas/sasjs/macros/mpe_derived.sas              | 232 ++++++++++++++++++
 sas/sasjs/macros/mpe_makedata.sas             |  12 +
 sas/sasjs/macros/mpe_makedatamodel.sas        |  21 ++
 sas/sasjs/services/editors/getdata.sas        |  47 +++-
 sas/sasjs/services/editors/getdata.test.sas   | 128 ++++++++++
 sas/sasjs/services/editors/stagedata.sas      |  23 ++
 sas/sasjs/services/public/getcolvals.sas      |  24 +-
 .../services/public/getcolvals.test.1.sas     |  51 ++++
 sas/sasjs/services/public/validatefilter.sas  |  14 ++
 .../services/public/validatefilter.test.sas   |  62 +++++
 13 files changed, 704 insertions(+), 7 deletions(-)
 create mode 100644 sas/sasjs/db/datactrl/mpe_derived.ddl
 create mode 100644 sas/sasjs/db/migrations/20261019_v7.19_release.sas
 create mode 100644 sas/sasjs/macros/mpe_derived.sas

diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
new file mode 100644
index 0000000..dab7706
--- /dev/null
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -0,0 +1,30 @@
+/**
+  @file mpe_derived.ddl
+  @brief ddl file
+  @details Defines the derived (computed) variable configuration table.  This
+  drives MPE_DERIVED rows in MPE_TABLES, so that the table itself can be
+  maintained through the EDIT screen with the usual approve workflow.
+
+  Derived values are not persisted - the logic is applied at read time, before
+  the filter, so that the variables can be filtered on.
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd
+**/
+
+
+create table &curlib..MPE_DERIVED
+   (
+    TX_FROM float not null format=datetime19.,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VARIABLE_NM varchar(32),
+    VARIABLE_TYPE varchar(1),
+    VARIABLE_LENGTH int,
+    VARIABLE_FORMAT varchar(32),
+    VARIABLE_LOGIC varchar(4000),
+    ACTIVE int,
+    TX_TO float not null format=datetime19.,
+    constraint pk_mpe_derived
+        primary key(tx_from, libref, dsn, variable_nm));
diff --git a/sas/sasjs/db/datactrl/mpe_tables.ddl b/sas/sasjs/db/datactrl/mpe_tables.ddl
index 1da535d..756be8c 100644
--- a/sas/sasjs/db/datactrl/mpe_tables.ddl
+++ b/sas/sasjs/db/datactrl/mpe_tables.ddl
@@ -118,3 +118,15 @@ insert into &mpelib..mpe_tables
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
     ;
+  insert into &mpelib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&mpelib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VARIABLE_NM'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
new file mode 100644
index 0000000..36b1380
--- /dev/null
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -0,0 +1,55 @@
+/**
+  @file
+  @brief migration script to move to v7.19 of Data Controller
+
+  REQUIRED CHANGE - adds the MPE_DERIVED table, which holds derived (computed)
+  variable definitions for the EDIT screen, and registers it in MPE_TABLES so
+  that it can be maintained through the app with the usual approve workflow.
+
+  Replace YOURDCLIB and the library path below, then run the script.  Any
+  MPE_DERIVED rows added before this script runs will be lost when the table
+  is created, so run it before configuring any derived variables.
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+proc sql;
+create table &dclib..mpe_derived
+  (
+    tx_from num not null format=datetime19.,
+    libref varchar(8),
+    dsn varchar(32),
+    variable_nm varchar(32),
+    variable_type varchar(1),
+    variable_length num,
+    variable_format varchar(32),
+    variable_logic varchar(4000),
+    active num,
+    tx_to num not null format=datetime19.
+  );
+quit;
+
+proc datasets lib=&dclib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn variable_nm)
+    /nomiss unique;
+quit;
+
+proc sql;
+insert into &dclib..mpe_tables
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="&dclib"
+    ,dsn='MPE_DERIVED'
+    ,num_of_approvals_required=1
+    ,loadtype='TXTEMPORAL'
+    ,buskey='LIBREF DSN VARIABLE_NM'
+    ,notes='Derived (computed) variables for the EDIT screen'
+    ,var_txfrom='TX_FROM'
+    ,var_txto='TX_TO'
+  ;
+quit;
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
new file mode 100644
index 0000000..8c7dc7a
--- /dev/null
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -0,0 +1,232 @@
+/**
+  @file
+  @brief Generates the SAS statements that create derived variables
+  @details Derived (computed) variables are configured in MPE_DERIVED.  Each
+  row defines one variable, which is created in a DATA step view ahead of the
+  filter, so that it can be filtered on in the EDIT screen.
+
+  The macro writes the generated statements to a fileref, so that callers can
+  compose them into a view:
+
+      %mpe_derived(libds=&libds,dclib=&mpelib,outref=dercode)
+      data work.vw_derived / view=work.vw_derived;
+        set &libds;
+        %include dercode;
+      run;
+
+  Alternatively, pass outview= and the macro will build the view itself.
+
+  The filter must be applied by the caller, in the step that reads the view:
+
+      data out;
+        set work.vw_derived;
+        where %inc filtref;;
+      run;
+
+  A WHERE statement applies to the data set named on the SET statement, so it
+  cannot reference a variable that is created in the same data step.  Applying
+  it to the view puts the derived variables on the input, which is what makes
+  them filterable.
+
+  Each statement block is of the form:
+
+      attrib REGION length=$21 label='DERIVED: <expression>' format=$21.;
+      REGION = <expression>;
+
+  Type and length are declared by the administrator, so the result is
+  deterministic.  The label defaults to the expression itself, prefixed with
+  DERIVED: so that computed columns are obvious in the grid.
+
+  Expressions are self-contained: they may reference the physical columns of
+  the target table only, not other derived variables.  Semicolons, macro
+  triggers and comments are rejected, so a row can only ever be an expression.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  @param [in] libds= The target table (libref.dataset, lib must be assigned)
+  @param [in] dclib= The Data Controller control library
+  @param [out] outref= (0) Fileref to receive the generated statements
+  @param [out] outvars= (work.derived_vars) Dataset of derived variable names
+  @param [in] outview= (0) Optional two-level name for the derived view
+
+  @version 9.2
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%macro mpe_derived(libds=,
+  dclib=,
+  outref=0,
+  outvars=work.derived_vars,
+  outview=0
+);
+
+%local physlist derived_err;
+
+%mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Missing: libds dclib)
+)
+%mp_abort(iftrue= (&outref = 0)
+  ,mac=&sysmacroname
+  ,msg=%str(Please provide an outref for the generated statements)
+)
+
+filename &outref temp;
+
+/* ensure the outputs exist, and are empty */
+data _null_;
+  file &outref;
+run;
+data &outvars;
+  length name $32;
+  stop;
+run;
+
+%if %mf_existds(libds=&dclib..mpe_derived) ne 1 %then %do;
+  %put &sysmacroname: &dclib..MPE_DERIVED not found - no derived variables;
+  %return;
+%end;
+
+data work.derived_cfg;
+  set &dclib..mpe_derived;
+  where &dc_dttmtfmt lt tx_to
+    and active=1
+    and upcase(libref)="%scan(&libds,1,.)"
+    and upcase(dsn)="%scan(&libds,2,.)";
+  length name $32 logic $4000 fmt $32;
+  name=upcase(variable_nm);
+  logic=strip(variable_logic);
+  type=upcase(variable_type);
+  len=variable_length;
+  fmt=strip(upcase(variable_format));
+run;
+
+%if %mf_nobs(work.derived_cfg)=0 %then %return;
+
+proc sort data=work.derived_cfg out=work.derived_srt;
+  by name;
+run;
+
+/**
+  * Reject anything that could break the generated data step, or that would
+  * produce a variable we cannot describe to the client
+  */
+%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
+
+data work.derived_bad;
+  set work.derived_srt;
+  length reason $200;
+  call missing(reason);
+  if name='' then reason='VARIABLE_NM is required';
+  else if verify(name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VARIABLE_NM must be a valid SAS name';
+  else if verify(substr(name,1,1),'0123456789')=0
+    then reason='VARIABLE_NM must not start with a digit';
+  else if name in (%unquote(&physlist))
+    then reason='VARIABLE_NM collides with a column in the target table';
+  else if name in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
+    ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
+    then reason='VARIABLE_NM is reserved';
+  else if type not in ('C','N')
+    then reason='VARIABLE_TYPE must be C or N';
+  else if missing(len) or len ne int(len) or len<1
+    then reason='VARIABLE_LENGTH must be a positive integer';
+  else if type='C' and len>32767
+    then reason='VARIABLE_LENGTH exceeds 32767 for a character variable';
+  else if type='N' and len>8
+    then reason='VARIABLE_LENGTH exceeds 8 for a numeric variable';
+  else if missing(logic)
+    then reason='VARIABLE_LOGIC is required';
+  else if index(logic,';')>0
+    then reason='VARIABLE_LOGIC must be a single expression - remove the ;';
+  else if indexc(logic,'&%')>0
+    then reason='VARIABLE_LOGIC must not contain macro triggers';
+  else if index(logic,'/*')>0 or index(logic,'*/')>0
+    then reason='VARIABLE_LOGIC must not contain comments';
+  if missing(reason)=0 then output;
+run;
+
+data work.derived_dup;
+  set work.derived_srt;
+  by name;
+  length reason $200;
+  if first.name and not last.name;
+  reason='VARIABLE_NM is defined more than once';
+run;
+
+proc append base=work.derived_bad data=work.derived_dup;
+run;
+
+%if %mf_nobs(work.derived_bad)>0 %then %do;
+  data _null_;
+    set work.derived_bad;
+    putlog 'ERR' 'OR: derived variable config: ' name= reason=;
+  run;
+  data _null_;
+    set work.derived_bad(obs=1);
+    call symputx('derived_err',cats(name,': ',reason));
+  run;
+  %mp_abort(iftrue=1
+    ,mac=&sysmacroname
+    ,msg=%str(Invalid derived variable configuration - &derived_err)
+  )
+%end;
+
+/**
+  * Generate the attrib + assignment statements
+  */
+data _null_;
+  file &outref;
+  set work.derived_srt;
+  length stmt $4000 label $256 fmt2 $49;
+  /* labels cannot contain single quotes, and are limited to 256 chars */
+  label=cats('DERIVED: ',translate(logic,' ','0D0A'x));
+  label=tranwrd(label,"'","''");
+  label=substr(label,1,256);
+  if not missing(fmt) and index(fmt,'.')=0 then fmt=cats(fmt,'.');
+  if type='C' then do;
+    if missing(fmt) then fmt2=cats('$',len,'.');
+    else fmt2=fmt;
+    stmt=catx(' ','attrib',name,cats('length=$',len)
+      ,cats("label='",label,"'"),cats('format=',fmt2,';'));
+  end;
+  else do;
+    if missing(fmt) then fmt2=cats(len,'.');
+    else fmt2=fmt;
+    stmt=catx(' ','attrib',name,cats('length=',len)
+      ,cats("label='",label,"'"),cats('format=',fmt2,';'));
+  end;
+  put stmt;
+  stmt=cats(name,' = ',logic,';');
+  put stmt;
+run;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(Unable to generate derived variables for &libds)
+)
+
+data &outvars(keep=name);
+  set work.derived_srt;
+run;
+
+%if %length(&outview)>0 and &outview ne 0 %then %do;
+  data &outview / view=&outview;
+    set &libds;
+    %include &outref;
+  run;
+  %mp_abort(iftrue= (&syscc ne 0)
+    ,mac=&sysmacroname
+    ,msg=%str(Unable to build derived view &outview for &libds)
+  )
+%end;
+
+%mend mpe_derived;
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index 5416c93..7c41c41 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -1495,6 +1495,18 @@ insert into &lib..mpe_selectbox set
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
     ;
+  insert into &lib..mpe_tables
+    set tx_from=0
+      ,tx_to='31DEC5999:23:59:59'dt
+      ,libref="&lib"
+      ,dsn='MPE_DERIVED'
+      ,num_of_approvals_required=1
+      ,loadtype='TXTEMPORAL'
+      ,buskey='LIBREF DSN VARIABLE_NM'
+      ,notes='Derived (computed) variables for the EDIT screen'
+      ,var_txfrom='TX_FROM'
+      ,var_txto='TX_TO'
+    ;
   insert into &lib..mpe_tables
     set tx_from=0
       ,tx_to='31DEC5999:23:59:59'dt
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index f9862f1..b006cb4 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -692,6 +692,27 @@ proc datasets lib=&lib noprint;
     pk_mpe_validations=(tx_from base_lib base_ds base_col rule_type)
     /nomiss unique;
 quit;
+/* mpe_derived - computed variables for the EDIT screen */
+proc sql;
+create table &lib..MPE_DERIVED
+  (
+    TX_FROM num &notnull format=datetime19.3,
+    LIBREF varchar(8),
+    DSN varchar(32),
+    VARIABLE_NM varchar(32),
+    VARIABLE_TYPE varchar(1),
+    VARIABLE_LENGTH num,
+    VARIABLE_FORMAT varchar(32),
+    VARIABLE_LOGIC varchar(4000),
+    ACTIVE num,
+    TX_TO num &notnull format=datetime19.3
+);quit;
+proc datasets lib=&lib noprint;
+  modify mpe_derived;
+  index create
+    pk_mpe_derived=(tx_from libref dsn variable_nm)
+    /nomiss unique;
+quit;
 proc sql;
 create table &lib..mpe_x_test(
           primary_key_field num &notnull,
diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index f9f5c90..8d9858e 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -68,6 +68,7 @@
   @li mp_validatecol.sas
   @li mpe_accesscheck.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getlabels.sas
   @li mpe_getversions.sas
@@ -201,6 +202,21 @@ select upcase(loadtype)
   outds=work.query
 )
 
+/**
+  * Derived (computed) variables, configured in MPE_DERIVED.  They are added
+  * via a view, so that the filter (applied by the step that reads the view)
+  * can reference them - a WHERE clause cannot see a variable that is created
+  * in the same data step.
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%put &=derived_nobs;
+
 %macro mpestp_getdata();
   %if not %symexist(DC_MAXOBS_WEBEDIT) %then %do;
     %put NOTE:;%put NOTE- DC_MAXOBS_WEBEDIT not found!;
@@ -237,7 +253,12 @@ select upcase(loadtype)
 
   %if &loadtype=BITEMPORAL %then %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
       where %inc filtref;;
     run;
     proc sort data=out;
@@ -251,7 +272,12 @@ select upcase(loadtype)
   %end;
   %else %do;
     data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
-      set &libds;
+      %if &derived_nobs>0 %then %do;
+        set work.vw_derived;
+      %end;
+      %else %do;
+        set &libds;
+      %end;
       where %inc filtref;;
     run;
   %end;
@@ -513,13 +539,28 @@ select upcase(loadtype)
 /* extract col info */
 %mp_getcols(&libds, outds=cols1)
 
+/* derived variables are added to the view, so take their attributes from
+  there - this keeps type, length, format and label identical to the data */
+%if &derived_nobs>0 %then %do;
+  %mp_getcols(work.vw_derived, outds=derived_cols_all)
+  proc sql;
+  create table work.derived_cols as
+    select a.*
+    from work.derived_cols_all a
+    where a.NAME in (select NAME from work.derived_vars);
+  quit;
+  data work.cols1;
+    set work.cols1 work.derived_cols;
+  run;
+%end;
+
 /* join with cls rules.  Trim COLS payload in the SQL below - DDTYPE
   C=CHARACTER N=NUMERIC (dates kept in full), VARNUM dropped (client
   relies on column order, not position) */
 proc sql;
 create table work.cols as
   select a.NAME
-    ,coalesce(c.desc,a.NAME) as LABEL
+    ,coalesce(c.desc,a.LABEL,a.NAME) as LABEL
     ,a.FMTNAME
     ,case a.DDTYPE
       when 'CHARACTER' then 'C'
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index c12060d..a596a57 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -9,6 +9,7 @@
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
   @li mp_filterstore.sas
+  @li mpe_derived.sas
   @li mx_execute.sas
 
 **/
@@ -240,3 +241,130 @@ run;
   test=EQUALS 10,
   outds=work.test_results
 )
+
+
+/**
+  * Test 4 - derived variables
+  */
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,variable_nm='DERIVED_NOTE'
+    ,variable_type='C'
+    ,variable_length=41
+    ,variable_format=''
+    ,variable_logic="cats(libref,'.',dsn)"
+    ,active=1;
+quit;
+
+/* a filter may reference the derived variable, so validate against the view */
+%mpe_derived(libds=&dc_libref..MPE_TABLES,
+  dclib=&dc_libref,
+  outref=dercode4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.inquery4;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
+;;;;
+run;
+%mp_filterstore(
+  libds=&dc_libref..MPE_TABLES,
+  targetds=work.vw_derived4,
+  filter_summary=&dc_libref..mpe_filteranytable,
+  filter_detail=&dc_libref..mpe_filtersource,
+  lock_table=&dc_libref..mpe_lockanytable,
+  maxkeytable=&dc_libref..mpe_maxkeyvalues,
+  queryds=work.inquery4,
+  outresult=work.result4,
+  outquery=work.query4
+)
+data _null_;
+  set work.result4;
+  call symputx('filter_rk4',filter_rk);
+run;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. FILTER_RK:8.';
+  put "&dclib..MPE_TABLES,&filter_rk4";
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:sascontroltable,
+  outlib=web4,
+  mdebug=&sasjs_mdebug
+)
+
+data sasdata4;
+  set web4.sasdata;
+run;
+data cols4;
+  set web4.cols;
+run;
+
+%mp_assertcols(work.cols4,
+  desc=Test4 - derived column is in the COLS payload,
+  cols=DERIVED_NOTE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk4a;
+  set work.cols4;
+  where name='DERIVED_NOTE' and cls_rule='READ';
+run;
+%mp_assertdsobs(work.chk4a,
+  desc=Test4 - derived column is read only,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4b;
+  set work.cols4;
+  where name='DERIVED_NOTE' and label=:'DERIVED:';
+run;
+%mp_assertdsobs(work.chk4b,
+  desc=Test4 - derived column is labelled,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+%mp_assertdsobs(work.sasdata4,
+  desc=Test4 - filter on the derived column is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk4c;
+  set work.sasdata4;
+  if derived_note=cats("%scan(&dclib..MPE_TABLES,1,.)",'.','MPE_LOCK_ANYTABLE');
+run;
+%mp_assertdsobs(work.chk4c,
+  desc=Test4 - derived value is correct,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where variable_nm='DERIVED_NOTE'
+    and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;
diff --git a/sas/sasjs/services/editors/stagedata.sas b/sas/sasjs/services/editors/stagedata.sas
index 57bd64b..34f5cf2 100755
--- a/sas/sasjs/services/editors/stagedata.sas
+++ b/sas/sasjs/services/editors/stagedata.sas
@@ -22,6 +22,7 @@
   @li mf_getuniquefileref.sas
   @li mpe_loader.sas
   @li mpe_filtermaster.sas
+  @li mpe_derived.sas
   @li mp_abort.sas
   @li mp_binarycopy.sas
   @li mp_cntlout.sas
@@ -220,9 +221,31 @@ data _null_;
   put '%let ' name '=' value ';';
 run;
 
+/* derived variables are display-only - they never reach the approval package
+  or the target table, so drop any that were submitted */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outvars=work.derived_vars
+)
+%let derived_droplist=;
+%if %mf_nobs(work.derived_vars)>0 %then %do;
+  proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
+  run;
+  proc sql noprint;
+  select a.name into: derived_droplist separated by ' '
+    from work.derived_vars a
+    inner join work.jsdata_vars b
+    on a.name=b.name;
+  quit;
+%end;
+
 data approve.jsdset;
   length _____DELETE__THIS__RECORD_____ $3;
   set jsdata;
+  %if %length(&derived_droplist)>0 %then %do;
+    drop &derived_droplist;
+  %end;
 run;
 
 /**
diff --git a/sas/sasjs/services/public/getcolvals.sas b/sas/sasjs/services/public/getcolvals.sas
index 1468670..01d64e8 100644
--- a/sas/sasjs/services/public/getcolvals.sas
+++ b/sas/sasjs/services/public/getcolvals.sas
@@ -35,6 +35,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_nobs.sas
   @li mf_verifymacvars.sas
   @li dc_assignlib.sas
   @li mf_getvarformat.sas
@@ -42,6 +43,7 @@
   @li mp_cntlout.sas
   @li mp_filtercheck.sas
   @li mp_filtergenerate.sas
+  @li mpe_derived.sas
   @li mp_validatecol.sas
   @li mpe_validatecol.sas
 
@@ -150,10 +152,24 @@ run;
 )
 
 
+/**
+  * Derived (computed) variables - these exist only in a view, so the filter
+  * must be validated and applied against that view
+  */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%if &derived_nobs>0 %then %let filterds=work.vw_derived;
+%else %let filterds=&libds;
+
 /**
   * Validate the filter query
   */
-%mp_filtercheck(work.filterquery,targetds=&libds,abort=YES)
+%mp_filtercheck(work.filterquery,targetds=&filterds,abort=YES)
 
 /**
   * Prepare the query
@@ -162,14 +178,14 @@ run;
 
 /* cannot %inc in a sql where clause, only data step, so - use a view */
 data work.vw_vals/view=work.vw_vals;
-  set &libds;
+  set &filterds;
   where %inc myfilter;;
 run;
 
 proc sql;
 create view work.vw_vals_sorted as
   select distinct
-      put(&col2,%mf_getVarFormat(&libds,&col2,force=1)) as formatted,
+      put(&col2,%mf_getVarFormat(work.vw_vals,&col2,force=1)) as formatted,
       &col2 as unformatted
   from work.vw_vals;
 
@@ -194,7 +210,7 @@ proc sql noprint;
 select count(*) into: nobs from work.vw_vals_sorted;
 data meta;
   column="&col2";
-  sasformat="%mf_getVarFormat(&libds,&col2)";
+  sasformat="%mf_getVarFormat(work.vw_vals,&col2)";
   startrow=&startrow;
   rows=&rows;
   nobs=&nobs;
diff --git a/sas/sasjs/services/public/getcolvals.test.1.sas b/sas/sasjs/services/public/getcolvals.test.1.sas
index 575e357..b66554f 100644
--- a/sas/sasjs/services/public/getcolvals.test.1.sas
+++ b/sas/sasjs/services/public/getcolvals.test.1.sas
@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
   @li mx_execute.sas
   @li mf_getuniquefileref.sas
 
@@ -36,3 +37,53 @@ run;
   desc=DCLIB found in unfiltered getgolvals response,
   test=ANYVAL
 )
+
+/**
+  * derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
+    ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
+    ,variable_nm='DERIVED_KEY'
+    ,variable_type='C'
+    ,variable_length=20
+    ,variable_format=''
+    ,variable_logic='cats(primary_key_field)'
+    ,active=1;
+quit;
+
+%let f4=%mf_getuniquefileref();
+data _null_;
+  file &f4 termstr=crlf;
+  put 'LIBDS:$43. COL:$32.';
+  put "&dclib..MPE_X_TEST,DERIVED_KEY";
+run;
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputfiles=&f4:iwant,
+  outlib=web4
+)
+
+data work.vals4;
+  set web4.vals;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.vals4,
+  desc=Values are returned for a derived column,
+  test=HASOBS
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where variable_nm='DERIVED_KEY'
+    and upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
+quit;
diff --git a/sas/sasjs/services/public/validatefilter.sas b/sas/sasjs/services/public/validatefilter.sas
index e54c42a..2085b64 100644
--- a/sas/sasjs/services/public/validatefilter.sas
+++ b/sas/sasjs/services/public/validatefilter.sas
@@ -26,8 +26,10 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_filterstore.sas
+  @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li removecolsfromwork.sas
 
@@ -69,8 +71,20 @@ run;
 
 %dc_assignlib(WRITE,%scan(&ds,1,.))
 
+/* derived variables exist only in a view, so validate against that */
+%mpe_derived(libds=&ds,
+  dclib=&dc_libref,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%if &derived_nobs>0 %then %let filterds=work.vw_derived;
+%else %let filterds=&ds;
+
 %mp_filterstore(
   libds=&ds,
+  targetds=&filterds,
   queryds=work.filterquery,
   filter_summary=&dc_libref..mpe_filteranytable,
   filter_detail=&dc_libref..mpe_filtersource,
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index 8b3cd10..6fc1570 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -77,3 +77,65 @@ run;
   desc=Test2 result filter,
   test=EQUALS 1
 )
+
+/**
+  * Test 3 - filter on a derived variable
+  */
+
+proc sql;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,variable_nm='DERIVED_NOTE'
+    ,variable_type='C'
+    ,variable_length=41
+    ,variable_format=''
+    ,variable_logic="cats(libref,'.',dsn)"
+    ,active=1;
+quit;
+
+data work.iwant;
+  filter_table="&dclib..MPE_TABLES";
+run;
+data work.filterquery;
+  GROUP_LOGIC='AND';
+  SUBGROUP_LOGIC='AND';
+  SUBGROUP_ID=1;
+  VARIABLE_NM='DERIVED_NOTE';
+  OPERATOR_NM='CONTAINS';
+  RAW_VALUE="'MPE_LOCK'";
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.iwant work.filterquery,
+  outlib=web3,
+  outref=web3ref
+)
+data _null_;
+  infile web3ref;
+  input;
+  putlog _infile_;
+run;
+data result;
+  set web3.result;
+  putlog (_all_)(=);
+run;
+%mp_assertdsobs(work.result,
+  desc=Test3 derived filter,
+  test=EQUALS 1
+)
+
+/* tidy up the config so repeated runs start clean */
+proc sql;
+delete from &dc_libref..mpe_derived
+  where variable_nm='DERIVED_NOTE'
+    and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+    and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
+quit;

From b4fae88ae13658607e4209449577cd42849d3a99 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Mon, 19 Oct 2026 23:18:46 +0000
Subject: [PATCH 02/48] feat(mocks): implement derived variables in the JS
 services

So the EDIT screen can be exercised before a SAS server is available.

- dcMockUtils: mpeDerived() reads MPE_DERIVED for a table and mpeDeriveRows()
  adds the computed columns to the rows and to the schema, before the filter
  runs - the same order as the real service.
- compileSasExpression() covers the subset of SAS expression syntax the mock
  configs use (ifc, cats, catx, upcase, lowcase, strip, trim, length, substr,
  index, comparisons, AND/OR and IN). Anything else leaves the column missing,
  as a failing SAS expression would. The real evaluation is done by SAS.
- getdata / getcolvals apply the derivation before filtering, so a filter on a
  derived column works and dropdown values come from the derived data.
- makedata: MPE_DERIVED schema and rows, a new TESTDATA.DERIVED_DEMO table
  (REGION, LINE_VALUE) and its MPE_TABLES registration.
- cypress: derived-variables.cy.ts covers display and filtering on a character
  and a numeric derived column.

Also bumps @sasjs/core to 5.6.0, which provides the mp_filterstore targetds=
parameter the validatefilter change depends on.
---
 client/cypress/e2e/derived-variables.cy.ts    | 157 +++++++++++
 sas/mocks/sasjs/services/admin/makedata.js    |  68 ++++-
 sas/mocks/sasjs/services/dcMockUtils.js       | 245 ++++++++++++++++++
 sas/mocks/sasjs/services/editors/getdata.js   |  15 ++
 sas/mocks/sasjs/services/public/getcolvals.js |  18 +-
 sas/package-lock.json                         |   8 +-
 sas/package.json                              |   2 +-
 7 files changed, 506 insertions(+), 7 deletions(-)
 create mode 100644 client/cypress/e2e/derived-variables.cy.ts

diff --git a/client/cypress/e2e/derived-variables.cy.ts b/client/cypress/e2e/derived-variables.cy.ts
new file mode 100644
index 0000000..693d725
--- /dev/null
+++ b/client/cypress/e2e/derived-variables.cy.ts
@@ -0,0 +1,157 @@
+export {}
+//
+// Derived (computed) variables are configured per table in MPE_DERIVED and
+// created at read time, ahead of the filter, so that they can be filtered on.
+//
+// TESTDATA.DERIVED_DEMO carries two of them, neither of which is a physical
+// column of the table:
+//   REGION      ifc(STATE in ('CA','OR','WA'),'West','East')
+//   LINE_VALUE  QTY * PRICE
+//
+// The mock implements the derivation in getdata.js / getcolvals.js (see
+// mpeDerived in dcMockUtils.js), so the EDIT screen can be exercised without a
+// SAS server. The real evaluation is done by SAS.
+
+const hostUrl = Cypress.env('hosturl')
+const appLocation = Cypress.env('appLocation')
+const longerCommandTimeout = Cypress.env('longerCommandTimeout')
+
+context('derived variables: ', function () {
+  this.beforeAll(() => {
+    cy.visit(`${hostUrl}/SASLogon/logout`, { timeout: longerCommandTimeout })
+    cy.loginAndUpdateValidKey(true)
+  })
+
+  this.beforeEach(() => {
+    cy.viewport(1440, 900)
+    bootApp()
+  })
+
+  it('1 | derived columns are shown in the EDIT grid with computed values', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    getCellByHeaderAndRow(0, 'REGION').should('have.text', 'West')
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'East')
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '21')
+    getCellByHeaderAndRow(1, 'LINE_VALUE').should('have.text', '99')
+
+    cy.screenshot('derived-1-grid', { capture: 'viewport' })
+  })
+
+  it('2 | filters on a character derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('REGION')
+    setFilterValue('West')
+    submitFilter()
+
+    cy.get('.infoBar-text').should('contain.text', "REGION = 'West'")
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 2)
+    getCellByHeaderAndRow(1, 'REGION').should('have.text', 'West')
+
+    cy.screenshot('derived-2-filter-region', { capture: 'viewport' })
+  })
+
+  it('3 | filters on a numeric derived column', () => {
+    openEditor('TESTDATA.DERIVED_DEMO')
+
+    openFilterPopup()
+    setFilterVariable('LINE_VALUE')
+    setFilterOperator('>')
+    setFilterValue('50')
+    submitFilter()
+
+    cy.get('.ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length', 1)
+    getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '99')
+
+    cy.screenshot('derived-3-filter-value', { capture: 'viewport' })
+  })
+})
+
+/**
+ * Cypress clears cookies between tests, which drops the SASjs Server session
+ * and lands the app back on the evaluation agreement card. Accept it when it is
+ * shown - a no-op when the session survived.
+ */
+const bootApp = () => {
+  cy.visit(hostUrl, { timeout: longerCommandTimeout })
+  cy.get('body').then(($body: any) => {
+    if ($body.find('#TCS input[type="checkbox"]').length) {
+      cy.get('#TCS input[type="checkbox"]').check({ force: true })
+      cy.wait(4000)
+    }
+  })
+}
+
+/** Opens a table straight in the EDIT screen and waits for the grid. */
+const openEditor = (libMem: string) => {
+  cy.visit(`${hostUrl}${appLocation}/#/editor/${libMem}`, {
+    timeout: longerCommandTimeout
+  })
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).should('exist')
+  cy.wait(2500)
+}
+
+const openFilterPopup = () => {
+  cy.get('.filterSide', { timeout: longerCommandTimeout }).click()
+  cy.get('clr-dropdown-menu', { timeout: longerCommandTimeout })
+    .contains('Filter')
+    .click()
+  cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
+    'be.visible'
+  )
+}
+
+/**
+ * The soft-select inputs drop a transparent .overlay click-catcher over the
+ * modal while their suggestion list is open, so they need force.
+ */
+const setFilterVariable = (column: string) => {
+  cy.get('#vals_var_id0_0').clear({ force: true }).type(column, { force: true })
+  cy.get('#datalist_vals_var_id0_0 option').contains(column).click({
+    force: true
+  })
+  cy.get('#vals_var_id0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterValue = (value: string) => {
+  cy.get('#vals_0_0').clear({ force: true }).type(value, { force: true })
+  cy.get('#vals_0_0').trigger('keyup', { key: 'Escape', force: true })
+  cy.wait(400)
+}
+
+const setFilterOperator = (operator: string) => {
+  cy.get('.filter-modal .operator-col select').first().select(operator)
+  cy.wait(500)
+}
+
+const submitFilter = () => {
+  cy.contains('.filter-modal button', 'Ok').click()
+  cy.get('.app-loading', { timeout: longerCommandTimeout }).should('not.exist')
+  cy.wait(2500)
+}
+
+const getCellByHeaderAndRow = (rowIndex: number, headerText: string) => {
+  return cy
+    .get('.ht_clone_top .htCore thead tr th')
+    .should(($ths) => {
+      const texts = [...$ths].map((th) => (th as HTMLElement).innerText.trim())
+      expect(texts).to.include(headerText)
+    })
+    .then(($ths) => {
+      const index = [...$ths].findIndex(
+        (th) => (th as HTMLElement).innerText.trim() === headerText
+      )
+
+      return cy
+        .get('.ht_master tbody tr')
+        .then((rows: any) => rows[rowIndex].childNodes[index])
+        .then((cell) => cy.get(cell))
+    })
+}
diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index b1d8422..59e71d7 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -281,6 +281,36 @@ const tables = {
       DD_SENSITIVITY: 'Low'
     }
   ],
+  // Derived (computed) variables.  Values are display-time only - the logic is
+  // applied by mpe_derived in the mock's getdata/getcolvals, and must use the
+  // expression subset that compileSasExpression supports (the real evaluation
+  // is done by SAS).
+  MPE_DERIVED: [
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      variable_nm: 'REGION',
+      variable_type: 'C',
+      variable_length: 8,
+      variable_format: '',
+      variable_logic: "ifc(STATE in ('CA','OR','WA'),'West','East')",
+      active: 1
+    },
+    {
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      variable_nm: 'LINE_VALUE',
+      variable_type: 'N',
+      variable_length: 8,
+      variable_format: 'dollar12.2',
+      variable_logic: 'QTY * PRICE',
+      active: 1
+    }
+  ],
   MPE_DATALOADS: [],
   MPE_EMAILS: [],
   MPE_EXCEL_CONFIG: [],
@@ -3262,6 +3292,18 @@ const schema = {
       notnull: false
     }
   ],
+  MPE_DERIVED: [
+    { name: 'TX_FROM', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true },
+    { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
+    { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VARIABLE_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VARIABLE_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
+    { name: 'VARIABLE_LENGTH', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VARIABLE_FORMAT', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VARIABLE_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'ACTIVE', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'TX_TO', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true }
+  ],
   MPE_DATALOADS: [
     {
       name: 'LIBREF',
@@ -5349,7 +5391,15 @@ const testTables = {
     { ORDER_ID: 1003, CUSTOMER: 'Cobalt GmbH', REGION: 'EU', AMOUNT: 620, STATUS: 'CLOSED' },
     { ORDER_ID: 1004, CUSTOMER: 'Delta Systems', REGION: 'US', AMOUNT: 4500, STATUS: 'NEW' }
   ],
-  DEMO_MIRROR: []
+  DEMO_MIRROR: [],
+  // Derived variable demo - REGION and LINE_VALUE are not columns of this
+  // table, they are computed at read time from MPE_DERIVED rows.
+  DERIVED_DEMO: [
+    { ID: 1, STATE: 'CA', QTY: 2, PRICE: 10.5 },
+    { ID: 2, STATE: 'NY', QTY: 1, PRICE: 99 },
+    { ID: 3, STATE: 'OR', QTY: 5, PRICE: 3 },
+    { ID: 4, STATE: 'TX', QTY: 3, PRICE: 7.25 }
+  ]
 }
 
 // Schemas for the TESTDATA tables, matching the old inline getdata.js payload
@@ -5691,6 +5741,12 @@ const testSchema = {
       label: 'Last reviewed',
       notnull: false
     }
+  ],
+  DERIVED_DEMO: [
+    { name: 'ID', type: 'N', length: 8, format: '', label: 'ID', notnull: true },
+    { name: 'STATE', type: 'C', length: 2, format: '', label: 'State', notnull: false },
+    { name: 'QTY', type: 'N', length: 8, format: '', label: 'Quantity', notnull: false },
+    { name: 'PRICE', type: 'N', length: 8, format: 'dollar12.2', label: 'Price', notnull: false }
   ]
 }
 
@@ -5739,6 +5795,16 @@ tables.MPE_TABLES.push(
     buskey: 'ID',
     notes: 'Compact rule demo table (eight columns)'
   },
+  {
+    tx_from: 0,
+    tx_to: 127490111999,
+    libref: testDataLibref,
+    dsn: 'DERIVED_DEMO',
+    num_of_approvals_required: 1,
+    loadtype: 'UPDATE',
+    buskey: 'ID',
+    notes: 'Derived variable demo table'
+  },
   {
     tx_from: 0,
     tx_to: 127490111999,
diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index 15aed55..5f1671d 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1154,6 +1154,251 @@ function mpFilterStore(opts) {
     }
 }
 
+// ─── mpe_derived.sas ────────────────────────────────────────────────────────
+
+/**
+ * Compiles a SAS expression into a JS function - for the mock only.
+ *
+ * The real evaluation is done by SAS. This covers the subset used by the mock
+ * and demo configs, so the EDIT screen can be exercised (display, filtering,
+ * submission) without a SAS server:
+ *
+ *   - column names, numbers, quoted strings
+ *   - + - * / and comparisons  (= ne < > <= >=)
+ *   - AND OR, and `X IN (a, b, c)`
+ *   - ifc(), cats(), catx(), upcase(), lowcase(), strip(), trim(), length(),
+ *     substr(), index()
+ *
+ * Anything outside that subset throws when a row is derived, and the column is
+ * left missing for that row - the same way a failing SAS expression would.
+ */
+function compileSasExpression(expr, columnNames) {
+    // Tokenise first, so operators are only rewritten outside quoted strings
+    // (a literal such as 'OR' must never be read as an operator).
+    const tokens = []
+    const s = String(expr)
+    let i = 0
+    while (i < s.length) {
+        const c = s[i]
+        if (/\s/.test(c)) { i++; continue }
+        if (c === "'" || c === '"') {
+            const q = c
+            let j = i + 1
+            let val = ''
+            while (j < s.length) {
+                if (s[j] === q) {
+                    if (s[j + 1] === q) { val += q; j += 2; continue }
+                    break
+                }
+                val += s[j]
+                j++
+            }
+            tokens.push({ type: 'str', value: val })
+            i = j + 1
+            continue
+        }
+        if (/[0-9]/.test(c)) {
+            let j = i
+            while (j < s.length && /[0-9.]/.test(s[j])) j++
+            tokens.push({ type: 'num', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        if (/[A-Za-z_]/.test(c)) {
+            let j = i
+            while (j < s.length && /[A-Za-z0-9_]/.test(s[j])) j++
+            tokens.push({ type: 'word', value: s.slice(i, j) })
+            i = j
+            continue
+        }
+        const two = s.slice(i, i + 2)
+        if (['<=', '>=', '!=', '<>'].includes(two)) {
+            tokens.push({ type: 'punct', value: two })
+            i += 2
+            continue
+        }
+        tokens.push({ type: 'punct', value: c })
+        i++
+    }
+
+    // X IN (a, b, c)  ->  __in([a, b, c], X)
+    const out = []
+    for (let t = 0; t < tokens.length; t++) {
+        const tok = tokens[t]
+        if (
+            tok.type === 'word' &&
+            tok.value.toLowerCase() === 'in' &&
+            out.length > 0 &&
+            tokens[t + 1] &&
+            tokens[t + 1].value === '('
+        ) {
+            let depth = 0
+            let j = t + 1
+            for (; j < tokens.length; j++) {
+                if (tokens[j].value === '(') depth++
+                else if (tokens[j].value === ')') {
+                    depth--
+                    if (depth === 0) break
+                }
+            }
+            if (j < tokens.length) {
+                const left = out.pop()
+                const inner = tokens.slice(t + 2, j)
+                out.push(
+                    { type: 'word', value: '__in' },
+                    { type: 'punct', value: '(' },
+                    { type: 'punct', value: '[' },
+                    ...inner,
+                    { type: 'punct', value: ']' },
+                    { type: 'punct', value: ',' },
+                    left,
+                    { type: 'punct', value: ')' }
+                )
+                t = j
+                continue
+            }
+        }
+        out.push(tok)
+    }
+
+    const OPS = {
+        and: '&&', or: '||',
+        eq: '==', ne: '!=', lt: '<', gt: '>', le: '<=', ge: '>='
+    }
+    const js = out
+        .map(tok => {
+            if (tok.type === 'str') return JSON.stringify(tok.value)
+            if (tok.type === 'num') return tok.value
+            if (tok.type === 'word') return OPS[tok.value.toLowerCase()] || tok.value
+            if (tok.value === '=') return '=='
+            if (tok.value === '<>') return '!='
+            return tok.value
+        })
+        .join(' ')
+
+    const sasTrue = v => {
+        if (typeof v === 'boolean') return v
+        if (typeof v === 'number') return v !== 0 && !isNaN(v)
+        if (v === null || v === undefined) return false
+        const str = String(v).trim()
+        return str !== '' && str !== '0'
+    }
+    const toStr = v => {
+        if (v === null || v === undefined) return ''
+        if (typeof v === 'number') return isNaN(v) ? '' : String(v)
+        return String(v).trim()
+    }
+    const fns = {
+        ifc: (c, a, b) => (sasTrue(c) ? a : b),
+        cats: (...args) => args.map(toStr).join(''),
+        catx: (sep, ...args) => args.map(toStr).filter(v => v !== '').join(toStr(sep)),
+        upcase: v => toStr(v).toUpperCase(),
+        lowcase: v => toStr(v).toLowerCase(),
+        strip: v => toStr(v),
+        trim: v => toStr(v),
+        length: v => toStr(v).length,
+        substr: (v, p, n) => {
+            const str = toStr(v)
+            const start = Number(p) - 1
+            return n === undefined
+                ? str.substring(start)
+                : str.substring(start, start + Number(n))
+        },
+        index: (v, sub) => toStr(v).indexOf(toStr(sub)) + 1,
+        __in: (list, v) => list.some(item => String(item) === String(v) || Number(item) === Number(v))
+    }
+
+    const names = Object.keys(fns)
+    const fn = new Function(...columnNames, ...names, 'return (' + js + ');')
+    return row => fn(...columnNames.map(n => row[n]), ...names.map(n => fns[n]))
+}
+
+/**
+ * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
+ * variables with a compiled row function, ordered by name (the real macro
+ * sorts by variable name too).
+ *
+ * @param {Object} opts
+ * @param {string} opts.libref - target libref
+ * @param {string} opts.table - target dataset
+ * @param {string} opts.dataDir - data directory of the DC control library
+ * @param {Array<Object>} opts.columns - physical columns of the target table
+ */
+function mpeDerived(opts) {
+    const { libref, table, dataDir, columns } = opts
+    const loadTableData = makeTableLoader(dataDir)
+    const cfg = loadTableData('MPE_DERIVED')
+    if (!cfg || !Array.isArray(cfg.rows) || cfg.rows.length === 0) return []
+
+    const now = Date.now() / 1000
+    const lib = String(libref || '').toUpperCase()
+    const dsn = String(table || '').toUpperCase()
+    const columnNames = (columns || []).map(c => c.name)
+
+    const vars = cfg.rows
+        .filter(r =>
+            String(colVal(r, 'LIBREF') || '').toUpperCase() === lib &&
+            String(colVal(r, 'DSN') || '').toUpperCase() === dsn &&
+            Number(colVal(r, 'ACTIVE')) === 1 &&
+            (!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
+        )
+        .map(r => {
+            const logic = String(colVal(r, 'VARIABLE_LOGIC') || '').trim()
+            return {
+                name: String(colVal(r, 'VARIABLE_NM') || '').toUpperCase(),
+                type: String(colVal(r, 'VARIABLE_TYPE') || '').toUpperCase(),
+                length: Number(colVal(r, 'VARIABLE_LENGTH')) || 8,
+                format: String(colVal(r, 'VARIABLE_FORMAT') || ''),
+                logic,
+                label: 'DERIVED: ' + logic
+            }
+        })
+        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
+        .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+
+    for (const d of vars) {
+        try {
+            d.fn = compileSasExpression(d.logic, columnNames)
+        } catch (err) {
+            console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
+            d.fn = () => (d.type === 'N' ? null : '')
+        }
+    }
+    return vars
+}
+
+/**
+ * Adds the derived variables to every row and to the column list, so that the
+ * filter (which runs afterwards) and the grid both see them.
+ */
+function mpeDeriveRows(tableData, derived) {
+    if (!derived || derived.length === 0) return tableData
+    const rows = tableData.rows.map(row => {
+        const copy = Object.assign({}, row)
+        for (const d of derived) {
+            try {
+                copy[d.name] = d.fn(copy)
+            } catch (err) {
+                copy[d.name] = d.type === 'N' ? null : ''
+            }
+        }
+        return copy
+    })
+    const derivedCols = derived.map(d => ({
+        name: d.name,
+        type: d.type,
+        length: d.length,
+        format: d.format,
+        label: d.label,
+        notnull: false
+    }))
+    return {
+        name: tableData.name,
+        columns: (tableData.columns || []).concat(derivedCols),
+        rows
+    }
+}
+
 // ─── DC settings (services/settings.js) ──────────────────────────────────────
 
 /**
diff --git a/sas/mocks/sasjs/services/editors/getdata.js b/sas/mocks/sasjs/services/editors/getdata.js
index c2b6087..35997bb 100644
--- a/sas/mocks/sasjs/services/editors/getdata.js
+++ b/sas/mocks/sasjs/services/editors/getdata.js
@@ -77,6 +77,21 @@ if (mpeTablesData && mpeTablesData.rows) {
     tableReg = mpeTablesData.rows.find(r => r.libref === requestedLibref && r.dsn === requestedTable)
 }
 
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Applied before the filter, so a filter can reference them.  The derived
+// columns are added to the schema too, which is what puts them in COLS and in
+// the grid (read only, since the mock marks every column READ).
+
+const derived = mpeDerived({
+    libref: requestedLibref,
+    table: requestedTable,
+    dataDir: mpeDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── DQ rules (from MPE_VALIDATIONS + NOTNULL from schema) ──────────────────
 // Mirrors getdata.sas: dqrules = MPE_VALIDATIONS rows UNION dictionary.columns
 // NOTNULL constraints. MPE_VALIDATIONS lives in the control library.
diff --git a/sas/mocks/sasjs/services/public/getcolvals.js b/sas/mocks/sasjs/services/public/getcolvals.js
index 332f612..4dd489a 100644
--- a/sas/mocks/sasjs/services/public/getcolvals.js
+++ b/sas/mocks/sasjs/services/public/getcolvals.js
@@ -21,9 +21,25 @@ const tableName = libds.split('.')[1] || ''
 // ─── Load table data (searches all libref data folders) ──────────────────────
 
 const _loaded = loadTableAnyLib(tableName) || {}
-const tableData = _loaded.data
+let tableData = _loaded.data
 const tableLibref = _loaded.libref || libref
 
+// ─── Derived (computed) variables (mpe_derived.sas) ─────────────────────────
+// Present in the schema and in the rows before the filter is applied, so the
+// filter may reference them and the values below are computed from the same
+// row set the grid shows.
+
+const controlDataDir = libDataDir(dcLibref)
+const derived = mpeDerived({
+    libref: tableLibref,
+    table: tableName,
+    dataDir: controlDataDir,
+    columns: tableData ? tableData.columns : []
+})
+if (tableData && derived.length > 0) {
+    tableData = mpeDeriveRows(tableData, derived)
+}
+
 // ─── Resolve column metadata ─────────────────────────────────────────────────
 
 let colMeta = null
diff --git a/sas/package-lock.json b/sas/package-lock.json
index a3a8b5a..7539c9e 100644
--- a/sas/package-lock.json
+++ b/sas/package-lock.json
@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "4.20.9",
-        "@sasjs/core": "5.5.2"
+        "@sasjs/core": "5.6.0"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -240,9 +240,9 @@
       "license": "MIT"
     },
     "node_modules/@sasjs/core": {
-      "version": "5.5.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.5.2.tgz",
-      "integrity": "sha512-puojJhAKAKg5o0uHdGSkpQ3cXgzajMY9AXAm3R22l6bZ4GKvC03qRq1ueGjBBF3npNxviwIKCABTsNEtjs1LAw==",
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.6.0.tgz",
+      "integrity": "sha512-c8YGglQkQKdTrpKN463a6t5bJsRTBkgX1W53oCYW8zZOJVFXVLPI2t+ORFfLerMYlrcYFn3/OG4JLK2MATK63Q==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
diff --git a/sas/package.json b/sas/package.json
index 17d3504..6293d62 100644
--- a/sas/package.json
+++ b/sas/package.json
@@ -31,7 +31,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "4.20.9",
-    "@sasjs/core": "5.5.2"
+    "@sasjs/core": "5.6.0"
   },
   "overrides": {
     "nanoid": "3.3.18"

From 11ae41d626ff4b579e3cc79d258ebdab7ba37426 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Mon, 19 Oct 2026 23:38:57 +0000
Subject: [PATCH 03/48] test(cypress): open the EDIT screen filter from the
 editor, not the viewer

The editor has no .filterSide button - the filter is opened from the
btnView button above the grid.
---
 client/cypress/e2e/derived-variables.cy.ts | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/client/cypress/e2e/derived-variables.cy.ts b/client/cypress/e2e/derived-variables.cy.ts
index 693d725..087b578 100644
--- a/client/cypress/e2e/derived-variables.cy.ts
+++ b/client/cypress/e2e/derived-variables.cy.ts
@@ -93,15 +93,16 @@ const openEditor = (libMem: string) => {
   cy.visit(`${hostUrl}${appLocation}/#/editor/${libMem}`, {
     timeout: longerCommandTimeout
   })
-  cy.get('.filterSide', { timeout: longerCommandTimeout }).should('exist')
+  cy.get('.ht_clone_top .htCore thead tr th', {
+    timeout: longerCommandTimeout
+  }).should('exist')
   cy.wait(2500)
 }
 
 const openFilterPopup = () => {
-  cy.get('.filterSide', { timeout: longerCommandTimeout }).click()
-  cy.get('clr-dropdown-menu', { timeout: longerCommandTimeout })
-    .contains('Filter')
-    .click()
+  cy.contains('button.btnView', 'Filter', {
+    timeout: longerCommandTimeout
+  }).click()
   cy.get('.filter-modal', { timeout: longerCommandTimeout }).should(
     'be.visible'
   )

From 8e82a7c85779483f3e865f61c7a760f99a1bbd8b Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 00:03:04 +0000
Subject: [PATCH 04/48] ci: run the derived-variables cypress spec

The cypress step takes an explicit spec list, so a new spec has to be
added to it to run.
---
 .gitea/workflows/build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml
index 4145c05..b1573f2 100644
--- a/.gitea/workflows/build.yaml
+++ b/.gitea/workflows/build.yaml
@@ -198,7 +198,7 @@ jobs:
           # Start frontend and run cypress
           # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/derived-variables.cy.ts,cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

From 05ad5544e2fd3cc7a1b175195fd443a08ef18365 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 00:43:22 +0000
Subject: [PATCH 05/48] ci: run the derived-variables spec last

It applies a licence key via loginAndUpdateValidKey, and csv-limited
asserts the free-tier restriction - so it must not run before that spec.
Running last keeps the licence state every other spec sees unchanged.
---
 .gitea/workflows/build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml
index b1573f2..ddb867f 100644
--- a/.gitea/workflows/build.yaml
+++ b/.gitea/workflows/build.yaml
@@ -198,7 +198,7 @@ jobs:
           # Start frontend and run cypress
           # timeout 1800: SIGTERM after 30 min so Cypress can flush video/screenshots
           # before the outer timeout-minutes hard-kills the step (avoids silent multi-hour hangs)
-          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/derived-variables.cy.ts,cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts"
+          npx ng serve --host 0.0.0.0 --port 4200 & npx wait-on http://localhost:4200 && timeout 1800 npx cypress run --browser chrome --spec "cypress/e2e/csv-limited.cy.ts,cypress/e2e/hook-programs.cy.ts,cypress/e2e/liveness.cy.ts,cypress/e2e/editor.cy.ts,cypress/e2e/excel-multi-load.cy.ts,cypress/e2e/excel.cy.ts,cypress/e2e/csv.cy.ts,cypress/e2e/filtering.cy.ts,cypress/e2e/filter-panel.cy.ts,cypress/e2e/licensing.cy.ts,cypress/e2e/viewer-labels.cy.ts,cypress/e2e/full-table-search.cy.ts,cypress/e2e/viewbox.cy.ts,cypress/e2e/stage.cy.ts,cypress/e2e/browser-info.cy.ts,cypress/e2e/ddl-export.cy.ts,cypress/e2e/row-cell-limits.cy.ts,cypress/e2e/derived-variables.cy.ts"
 
       - name: Zip Cypress videos
         if: always()

From d3dec682aafeb143fe983d6546337cb6490f5620 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 01:24:50 +0000
Subject: [PATCH 06/48] fix(derived): enforce the documented limits, and handle
 derived filters in VIEW

Review follow-up.

- viewdata / getrawdata: apply the filter against the derived view when one
  exists, so a saved filter that references a derived column no longer aborts
  with "Variable ... is not on file" in the VIEW screen or a download.  The
  derived columns themselves are dropped from that output - they remain an
  EDIT screen feature.
- mpe_derived: reject a VARIABLE_FORMAT containing a semicolon, macro trigger
  or comment, and require it to look like a format name, so it cannot inject a
  statement into the generated data step.
- mpe_derived: reject a VARIABLE_LOGIC that references another derived
  variable (word-boundary match), which was documented but not enforced.
- mpe_derived: stmt length 4100 so a 4000-char expression plus the name and
  assignment cannot lose its trailing semicolon.
- mock tokenizer: allowlist the punctuation the expression subset uses, so
  anything else is rejected instead of being evaluated.
---
 sas/mocks/sasjs/services/dcMockUtils.js  |  5 ++++
 sas/sasjs/macros/mpe_derived.sas         | 33 ++++++++++++++++++++--
 sas/sasjs/services/public/getrawdata.sas | 27 +++++++++++++++++-
 sas/sasjs/services/public/viewdata.sas   | 35 +++++++++++++++++++++---
 4 files changed, 92 insertions(+), 8 deletions(-)

diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index 5f1671d..9f726a0 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1217,6 +1217,11 @@ function compileSasExpression(expr, columnNames) {
             i += 2
             continue
         }
+        // only the operators in the documented subset may reach the generated
+        // function - anything else is rejected rather than executed
+        if (!'+-*/(),<>='.includes(c)) {
+            throw new Error('Unsupported character in derived expression: ' + c)
+        }
         tokens.push({ type: 'punct', value: c })
         i++
     }
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 8c7dc7a..10aedc8 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -68,7 +68,7 @@
   outview=0
 );
 
-%local physlist derived_err;
+%local physlist derived_err derived_list;
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
@@ -115,6 +115,13 @@ proc sort data=work.derived_cfg out=work.derived_srt;
   by name;
 run;
 
+/* every derived name, for the self-containment check below */
+%let derived_list=;
+proc sql noprint;
+select name into: derived_list separated by ' '
+  from work.derived_srt;
+quit;
+
 /**
   * Reject anything that could break the generated data step, or that would
   * produce a variable we cannot describe to the client
@@ -123,8 +130,21 @@ run;
 
 data work.derived_bad;
   set work.derived_srt;
-  length reason $200;
+  length reason $200 other pat $64;
   call missing(reason);
+
+  /* a derived variable may not reference another one - the evaluation order
+    would depend on statement order, which is not configured */
+  do i=1 to countw("&derived_list");
+    other=scan("&derived_list",i);
+    if other ne name then do;
+      pat=cats('/(^|[^A-Z0-9_])',other,'([^A-Z0-9_]|$)/');
+      if prxmatch(pat,upcase(logic))>0 then
+        reason='VARIABLE_LOGIC must not reference another derived variable';
+    end;
+  end;
+
+  if missing(reason) then do;
   if name='' then reason='VARIABLE_NM is required';
   else if verify(name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
     then reason='VARIABLE_NM must be a valid SAS name';
@@ -143,6 +163,11 @@ data work.derived_bad;
     then reason='VARIABLE_LENGTH exceeds 32767 for a character variable';
   else if type='N' and len>8
     then reason='VARIABLE_LENGTH exceeds 8 for a numeric variable';
+  else if index(fmt,';')>0 or indexc(fmt,'&%')>0
+    or index(fmt,'/*')>0 or index(fmt,'*/')>0
+    then reason='VARIABLE_FORMAT must be a format name, without ; & % or //*';
+  else if verify(fmt,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VARIABLE_FORMAT must be a format name';
   else if missing(logic)
     then reason='VARIABLE_LOGIC is required';
   else if index(logic,';')>0
@@ -151,6 +176,8 @@ data work.derived_bad;
     then reason='VARIABLE_LOGIC must not contain macro triggers';
   else if index(logic,'/*')>0 or index(logic,'*/')>0
     then reason='VARIABLE_LOGIC must not contain comments';
+  end;
+
   if missing(reason)=0 then output;
 run;
 
@@ -186,7 +213,7 @@ run;
 data _null_;
   file &outref;
   set work.derived_srt;
-  length stmt $4000 label $256 fmt2 $49;
+  length stmt $4100 label $256 fmt2 $49;
   /* labels cannot contain single quotes, and are limited to 256 chars */
   label=cats('DERIVED: ',translate(logic,' ','0D0A'x));
   label=tranwrd(label,"'","''");
diff --git a/sas/sasjs/services/public/getrawdata.sas b/sas/sasjs/services/public/getrawdata.sas
index e0f20dd..a9d15d8 100644
--- a/sas/sasjs/services/public/getrawdata.sas
+++ b/sas/sasjs/services/public/getrawdata.sas
@@ -21,6 +21,8 @@
   @li mp_ds2csv.sas
   @li mp_streamfile.sas
   @li mp_validatecol.sas
+  @li mf_nobs.sas
+  @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li mpe_filtermaster.sas
 
@@ -143,10 +145,33 @@ data _null_;
   putlog _infile_;
 run;
 
+/* derived variables exist only in a view, so a filter that references one
+  must be applied against it.  They are dropped from the download - they are
+  an EDIT screen feature. */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+%let derived_nobs=%mf_nobs(work.derived_vars);
+%let derived_droplist=;
+%if &derived_nobs>0 %then %do;
+  %let filterds=work.vw_derived;
+  proc sql noprint;
+  select name into: derived_droplist separated by ' '
+    from work.derived_vars;
+  quit;
+%end;
+%else %let filterds=&libds;
+
 options obs=200000;/* stop limit */
 data staged(drop=&txfrom &txto);
-  set &libds;
+  set &filterds;
   where %inc filtref;;
+  %if %length(&derived_droplist)>0 %then %do;
+    drop &derived_droplist;
+  %end;
 run;
 options obs=max;
 
diff --git a/sas/sasjs/services/public/viewdata.sas b/sas/sasjs/services/public/viewdata.sas
index 384883c..bbeda72 100644
--- a/sas/sasjs/services/public/viewdata.sas
+++ b/sas/sasjs/services/public/viewdata.sas
@@ -59,6 +59,7 @@
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getversions.sas
   @li mpe_filtermaster.sas
@@ -218,6 +219,26 @@ run;
     putlog _infile_;
   run;
 
+  /* derived variables exist only in a view, so a filter that references one
+    must be applied against it.  The columns themselves are dropped from the
+    output - they are an EDIT screen feature. */
+  %mpe_derived(libds=&libds,
+    dclib=&mpelib,
+    outref=dercode,
+    outview=work.vw_derived,
+    outvars=work.derived_vars
+  )
+  %let derived_nobs=%mf_nobs(work.derived_vars);
+  %let derived_droplist=;
+  %if &derived_nobs>0 %then %do;
+    %let filterds=work.vw_derived;
+    proc sql noprint;
+    select name into: derived_droplist separated by ' '
+      from work.derived_vars;
+    quit;
+  %end;
+  %else %let filterds=&libds;
+
   %if &searchtype=NONE or "%trim(&searchtype) " = " " %then %do;
     /* get row count */
     filename rows temp;
@@ -227,7 +248,7 @@ run;
       input;
       if _n_=1 then do;
         put 'proc sql;';
-        put "select count(*) into: dsobs from &libds where";
+        put "select count(*) into: dsobs from &filterds where";
       end;
       put _infile_;
       if eof then put ';';
@@ -242,8 +263,11 @@ run;
 
     /* send actual data, filtered and row-capped */
     data work.viewdata;
-      set &libds;
+      set &filterds;
       where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
       if _n_>&DC_MAXOBS_WEBVIEW then stop;
     run;
     %if %mf_nobs(work.viewdata)=0 %then %do;
@@ -256,7 +280,7 @@ run;
   %end;
   %else %do;
     data work.vwsearch/view=work.vwsearch;
-      set &libds;
+      set &filterds;
       where %inc filtref;;
     run;
     %if %upcase(&searchtype)=CHAR %then %do;
@@ -277,13 +301,16 @@ run;
       %let dsobs=%mf_nobs(MPSEARCH.vwsearch);
       data viewdata;
         set MPSEARCH.vwsearch;
+        %if %length(&derived_droplist)>0 %then %do;
+          drop &derived_droplist;
+        %end;
         if _n_<=&DC_MAXOBS_WEBVIEW;
       run;
     %end;
     %else %do;
       %let dsobs=0;
       data viewdata;
-        set &libds;
+        set &filterds;
         stop;
       run;
     %end;

From 33d043ea92f007dca70c95d5363329d4447eeba9 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 02:09:37 +0000
Subject: [PATCH 07/48] feat(derived): VARIABLE_ORDER, so a derived variable
 can build on another

A derived variable may need to reference one defined before it (a label that
concatenates a derived region, for example), so the order has to be
configured rather than assumed.

- MPE_DERIVED: new VARIABLE_ORDER column (ddl, makedatamodel, migration).
- mpe_derived: statements are generated in VARIABLE_ORDER then name sequence.
  VARIABLE_ORDER is required and must be a positive integer. A reference to a
  derived variable with an equal or higher order is rejected, because SAS
  would otherwise treat it as uninitialised and return a missing value with no
  error; references to lower-order variables are allowed.
- mock: same ordering, each variable compiled against the physical columns
  plus those defined earlier, so the frontend can exercise a chained variable.
- makedata: VARIABLE_ORDER on both demo rows plus a chained REGION_LABEL.
- cypress: asserts the chained value in the grid.
- tests: the getdata test seeds a second variable built on the first.
- docs: VARIABLE_ORDER documented, with the ordered rule replacing the
  self-contained one.
---
 client/cypress/e2e/derived-variables.cy.ts    |  4 +
 sas/mocks/sasjs/services/admin/makedata.js    | 17 ++++
 sas/mocks/sasjs/services/dcMockUtils.js       | 25 ++++--
 sas/sasjs/db/datactrl/mpe_derived.ddl         |  1 +
 .../db/migrations/20261019_v7.19_release.sas  |  1 +
 sas/sasjs/macros/mpe_derived.sas              | 78 ++++++++++++-------
 sas/sasjs/macros/mpe_makedatamodel.sas        |  1 +
 sas/sasjs/services/editors/getdata.test.sas   | 24 ++++++
 .../services/public/getcolvals.test.1.sas     |  1 +
 .../services/public/validatefilter.test.sas   |  1 +
 10 files changed, 122 insertions(+), 31 deletions(-)

diff --git a/client/cypress/e2e/derived-variables.cy.ts b/client/cypress/e2e/derived-variables.cy.ts
index 087b578..26342f4 100644
--- a/client/cypress/e2e/derived-variables.cy.ts
+++ b/client/cypress/e2e/derived-variables.cy.ts
@@ -34,6 +34,10 @@ context('derived variables: ', function () {
     getCellByHeaderAndRow(1, 'REGION').should('have.text', 'East')
     getCellByHeaderAndRow(0, 'LINE_VALUE').should('have.text', '21')
     getCellByHeaderAndRow(1, 'LINE_VALUE').should('have.text', '99')
+    // REGION_LABEL is built on REGION, so it only resolves if the derived
+    // statements run in VARIABLE_ORDER sequence
+    getCellByHeaderAndRow(0, 'REGION_LABEL').should('have.text', 'West-CA')
+    getCellByHeaderAndRow(1, 'REGION_LABEL').should('have.text', 'East-NY')
 
     cy.screenshot('derived-1-grid', { capture: 'viewport' })
   })
diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index 59e71d7..6690503 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -292,6 +292,7 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       variable_nm: 'REGION',
+      variable_order: 10,
       variable_type: 'C',
       variable_length: 8,
       variable_format: '',
@@ -304,11 +305,26 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       variable_nm: 'LINE_VALUE',
+      variable_order: 20,
       variable_type: 'N',
       variable_length: 8,
       variable_format: 'dollar12.2',
       variable_logic: 'QTY * PRICE',
       active: 1
+    },
+    {
+      // built on REGION, so it proves the order is applied
+      tx_from: 0,
+      tx_to: 253717919999,
+      libref: 'TESTDATA',
+      dsn: 'DERIVED_DEMO',
+      variable_nm: 'REGION_LABEL',
+      variable_order: 30,
+      variable_type: 'C',
+      variable_length: 14,
+      variable_format: '',
+      variable_logic: "cats(REGION,'-',STATE)",
+      active: 1
     }
   ],
   MPE_DATALOADS: [],
@@ -3297,6 +3313,7 @@ const schema = {
     { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
     { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
     { name: 'VARIABLE_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VARIABLE_ORDER', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'VARIABLE_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
     { name: 'VARIABLE_LENGTH', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'VARIABLE_FORMAT', type: 'C', length: 32, format: '', label: '', notnull: false },
diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index 9f726a0..f98a54e 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1320,8 +1320,8 @@ function compileSasExpression(expr, columnNames) {
 
 /**
  * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
- * variables with a compiled row function, ordered by name (the real macro
- * sorts by variable name too).
+ * variables with a compiled row function, ordered by VARIABLE_ORDER (then
+ * name), the same sequence the real macro emits.
  *
  * @param {Object} opts
  * @param {string} opts.libref - target libref
@@ -1351,6 +1351,8 @@ function mpeDerived(opts) {
             const logic = String(colVal(r, 'VARIABLE_LOGIC') || '').trim()
             return {
                 name: String(colVal(r, 'VARIABLE_NM') || '').toUpperCase(),
+                // statement order - a variable may reference one with a lower order
+                order: Number(colVal(r, 'VARIABLE_ORDER')) || 0,
                 type: String(colVal(r, 'VARIABLE_TYPE') || '').toUpperCase(),
                 length: Number(colVal(r, 'VARIABLE_LENGTH')) || 8,
                 format: String(colVal(r, 'VARIABLE_FORMAT') || ''),
@@ -1358,16 +1360,29 @@ function mpeDerived(opts) {
                 label: 'DERIVED: ' + logic
             }
         })
-        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
-        .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+        .filter(d => {
+            if (!d.name || (d.type !== 'C' && d.type !== 'N') || !d.logic) return false
+            if (!(d.order >= 1)) {
+                console.log('derived variable ' + d.name + ' has no VARIABLE_ORDER - skipped')
+                return false
+            }
+            return true
+        })
+        .sort((a, b) => a.order - b.order || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
 
+    // compile each against the physical columns plus the variables defined
+    // earlier, so a derived variable can build on another one
+    const compileCols = columnNames.slice()
     for (const d of vars) {
         try {
-            d.fn = compileSasExpression(d.logic, columnNames)
+            // slice(): the compiled function captures the column list it was
+            // built with, not the array as it grows
+            d.fn = compileSasExpression(d.logic, compileCols.slice())
         } catch (err) {
             console.log('derived variable ' + d.name + ' could not be compiled: ' + err.message)
             d.fn = () => (d.type === 'N' ? null : '')
         }
+        compileCols.push(d.name)
     }
     return vars
 }
diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
index dab7706..344c2b5 100644
--- a/sas/sasjs/db/datactrl/mpe_derived.ddl
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -20,6 +20,7 @@ create table &curlib..MPE_DERIVED
     LIBREF varchar(8),
     DSN varchar(32),
     VARIABLE_NM varchar(32),
+    VARIABLE_ORDER int,
     VARIABLE_TYPE varchar(1),
     VARIABLE_LENGTH int,
     VARIABLE_FORMAT varchar(32),
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index 36b1380..a6b332f 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -23,6 +23,7 @@ create table &dclib..mpe_derived
     libref varchar(8),
     dsn varchar(32),
     variable_nm varchar(32),
+    variable_order num,
     variable_type varchar(1),
     variable_length num,
     variable_format varchar(32),
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 10aedc8..b0f2ccd 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -37,9 +37,14 @@
   deterministic.  The label defaults to the expression itself, prefixed with
   DERIVED: so that computed columns are obvious in the grid.
 
-  Expressions are self-contained: they may reference the physical columns of
-  the target table only, not other derived variables.  Semicolons, macro
-  triggers and comments are rejected, so a row can only ever be an expression.
+  Statements are generated in VARIABLE_ORDER sequence, so a derived variable
+  may reference another derived variable with a lower order.  A reference to a
+  variable that is not defined earlier is rejected, because SAS would otherwise
+  treat it as an uninitialised variable and return a missing value without an
+  error.
+
+  Semicolons, macro triggers and comments are rejected, so a row can only ever
+  be an expression.
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
@@ -68,7 +73,7 @@
   outview=0
 );
 
-%local physlist derived_err derived_list;
+%local physlist derived_err;
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
@@ -107,20 +112,20 @@ data work.derived_cfg;
   type=upcase(variable_type);
   len=variable_length;
   fmt=strip(upcase(variable_format));
+  ord=variable_order;
 run;
 
 %if %mf_nobs(work.derived_cfg)=0 %then %return;
 
+/* statement order is the configured order - a derived variable may reference
+  one with a lower VARIABLE_ORDER, so they must be assigned in this sequence */
 proc sort data=work.derived_cfg out=work.derived_srt;
-  by name;
+  by ord name;
 run;
 
-/* every derived name, for the self-containment check below */
-%let derived_list=;
-proc sql noprint;
-select name into: derived_list separated by ' '
-  from work.derived_srt;
-quit;
+proc sort data=work.derived_cfg out=work.derived_byname;
+  by name;
+run;
 
 /**
   * Reject anything that could break the generated data step, or that would
@@ -130,21 +135,9 @@ quit;
 
 data work.derived_bad;
   set work.derived_srt;
-  length reason $200 other pat $64;
+  length reason $200;
   call missing(reason);
 
-  /* a derived variable may not reference another one - the evaluation order
-    would depend on statement order, which is not configured */
-  do i=1 to countw("&derived_list");
-    other=scan("&derived_list",i);
-    if other ne name then do;
-      pat=cats('/(^|[^A-Z0-9_])',other,'([^A-Z0-9_]|$)/');
-      if prxmatch(pat,upcase(logic))>0 then
-        reason='VARIABLE_LOGIC must not reference another derived variable';
-    end;
-  end;
-
-  if missing(reason) then do;
   if name='' then reason='VARIABLE_NM is required';
   else if verify(name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
     then reason='VARIABLE_NM must be a valid SAS name';
@@ -155,6 +148,8 @@ data work.derived_bad;
   else if name in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
     ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
     then reason='VARIABLE_NM is reserved';
+  else if missing(ord) or ord ne int(ord) or ord<1
+    then reason='VARIABLE_ORDER must be a positive integer';
   else if type not in ('C','N')
     then reason='VARIABLE_TYPE must be C or N';
   else if missing(len) or len ne int(len) or len<1
@@ -176,13 +171,12 @@ data work.derived_bad;
     then reason='VARIABLE_LOGIC must not contain macro triggers';
   else if index(logic,'/*')>0 or index(logic,'*/')>0
     then reason='VARIABLE_LOGIC must not contain comments';
-  end;
 
   if missing(reason)=0 then output;
 run;
 
 data work.derived_dup;
-  set work.derived_srt;
+  set work.derived_byname;
   by name;
   length reason $200;
   if first.name and not last.name;
@@ -192,6 +186,38 @@ run;
 proc append base=work.derived_bad data=work.derived_dup;
 run;
 
+/* a derived variable may only reference derived variables defined earlier -
+  otherwise the reference silently resolves to a missing value */
+proc sql noprint;
+create table work.derived_orderbad_key as
+  select distinct a.name
+  from work.derived_srt a
+  inner join work.derived_srt b
+    on a.name ne b.name
+  where prxmatch(
+      cats('/(^|[^A-Z0-9_])',b.name,'([^A-Z0-9_]|$)/')
+      ,upcase(a.logic)
+    )>0
+    and b.ord>=a.ord
+  order by a.name;
+quit;
+
+proc sort data=work.derived_orderbad_key;
+  by name;
+run;
+
+data work.derived_orderbad;
+  merge work.derived_byname(in=in1) work.derived_orderbad_key(in=in2);
+  by name;
+  if in2;
+  length reason $200;
+  reason='VARIABLE_LOGIC references a derived variable that is not defined'
+    !!' earlier - lower the VARIABLE_ORDER of this one';
+run;
+
+proc append base=work.derived_bad data=work.derived_orderbad;
+run;
+
 %if %mf_nobs(work.derived_bad)>0 %then %do;
   data _null_;
     set work.derived_bad;
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index b006cb4..98e6258 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -700,6 +700,7 @@ create table &lib..MPE_DERIVED
     LIBREF varchar(8),
     DSN varchar(32),
     VARIABLE_NM varchar(32),
+    VARIABLE_ORDER num,
     VARIABLE_TYPE varchar(1),
     VARIABLE_LENGTH num,
     VARIABLE_FORMAT varchar(32),
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index a596a57..22a95c9 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -258,11 +258,24 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,variable_nm='DERIVED_NOTE'
+    ,variable_order=10
     ,variable_type='C'
     ,variable_length=41
     ,variable_format=''
     ,variable_logic="cats(libref,'.',dsn)"
     ,active=1;
+insert into &dc_libref..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref="%scan(&dclib..MPE_TABLES,1,.)"
+    ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
+    ,variable_nm='DERIVED_NOTE2'
+    ,variable_order=20
+    ,variable_type='C'
+    ,variable_length=45
+    ,variable_format=''
+    ,variable_logic="cats(DERIVED_NOTE,' (2)')"
+    ,active=1;
 quit;
 
 /* a filter may reference the derived variable, so validate against the view */
@@ -361,6 +374,17 @@ run;
   outds=work.test_results
 )
 
+/* the second variable is built on the first, so this proves statement order */
+data work.chk4d;
+  set work.sasdata4;
+  if derived_note2=cats(derived_note,' (2)');
+run;
+%mp_assertdsobs(work.chk4d,
+  desc=Test4 - a derived variable can reference an earlier one,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
 /* tidy up the config so repeated runs start clean */
 proc sql;
 delete from &dc_libref..mpe_derived
diff --git a/sas/sasjs/services/public/getcolvals.test.1.sas b/sas/sasjs/services/public/getcolvals.test.1.sas
index b66554f..a01ab3b 100644
--- a/sas/sasjs/services/public/getcolvals.test.1.sas
+++ b/sas/sasjs/services/public/getcolvals.test.1.sas
@@ -52,6 +52,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
     ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
     ,variable_nm='DERIVED_KEY'
+    ,variable_order=10
     ,variable_type='C'
     ,variable_length=20
     ,variable_format=''
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index 6fc1570..bfae64f 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -92,6 +92,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,variable_nm='DERIVED_NOTE'
+    ,variable_order=10
     ,variable_type='C'
     ,variable_length=41
     ,variable_format=''

From 9ef813d439b48e7fefd35c426e62b99e5943002a Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 02:49:24 +0000
Subject: [PATCH 08/48] fix(derived): close the RLS and search gaps on derived
 columns

Review follow-up.

- stagedata: build the derived droplist before the ULOAD filter, and drop the
  derived columns in the badrecords step, so the row-level-security check runs
  against physical columns only.  A submitted derived value can no longer
  satisfy an RLS clause that names a derived column.
- viewdata: the full-table search view now drops the derived columns too, so
  a free-text search cannot match a value in an EDIT-only column.
- mpe_derived: upcase the libds when matching MPE_DERIVED, so a lower-case
  table reference still finds its configuration instead of silently showing
  no derived columns.
---
 sas/sasjs/macros/mpe_derived.sas         |  4 +--
 sas/sasjs/services/editors/stagedata.sas | 43 ++++++++++++++----------
 sas/sasjs/services/public/viewdata.sas   |  3 ++
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index b0f2ccd..0fdf493 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -104,8 +104,8 @@ data work.derived_cfg;
   set &dclib..mpe_derived;
   where &dc_dttmtfmt lt tx_to
     and active=1
-    and upcase(libref)="%scan(&libds,1,.)"
-    and upcase(dsn)="%scan(&libds,2,.)";
+    and upcase(libref)="%upcase(%scan(&libds,1,.))"
+    and upcase(dsn)="%upcase(%scan(&libds,2,.))";
   length name $32 logic $4000 fmt $32;
   name=upcase(variable_nm);
   logic=strip(variable_logic);
diff --git a/sas/sasjs/services/editors/stagedata.sas b/sas/sasjs/services/editors/stagedata.sas
index 34f5cf2..0b9edb6 100755
--- a/sas/sasjs/services/editors/stagedata.sas
+++ b/sas/sasjs/services/editors/stagedata.sas
@@ -107,6 +107,27 @@ run;
 )
 
 %put Verify that the upload does not violate Row Level Security checks:;
+
+/* derived variables are display-only and must not be submitted, so drop them
+  from the data before the security check runs - otherwise a crafted
+  submission could satisfy an RLS clause that references a derived column */
+%mpe_derived(libds=&libds,
+  dclib=&mpelib,
+  outref=dercode,
+  outvars=work.derived_vars
+)
+%let derived_droplist=;
+%if %mf_nobs(work.derived_vars)>0 %then %do;
+  proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
+  run;
+  proc sql noprint;
+  select a.name into: derived_droplist separated by ' '
+    from work.derived_vars a
+    inner join work.jsdata_vars b
+    on a.name=b.name;
+  quit;
+%end;
+
 %mpe_filtermaster(ULOAD,&libds,
   dclib=&mpelib,
   outref=filtref,
@@ -131,6 +152,9 @@ run;
 /* apply the query */
 data work.badrecords;
   set work.jsdata;
+  %if %length(&derived_droplist)>0 %then %do;
+    drop &derived_droplist;
+  %end;
   %inc &tempref/source2;;
   putlog (_all_)(=);
 run;
@@ -222,24 +246,7 @@ data _null_;
 run;
 
 /* derived variables are display-only - they never reach the approval package
-  or the target table, so drop any that were submitted */
-%mpe_derived(libds=&libds,
-  dclib=&mpelib,
-  outref=dercode,
-  outvars=work.derived_vars
-)
-%let derived_droplist=;
-%if %mf_nobs(work.derived_vars)>0 %then %do;
-  proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
-  run;
-  proc sql noprint;
-  select a.name into: derived_droplist separated by ' '
-    from work.derived_vars a
-    inner join work.jsdata_vars b
-    on a.name=b.name;
-  quit;
-%end;
-
+  or the target table (the droplist was built before the security check) */
 data approve.jsdset;
   length _____DELETE__THIS__RECORD_____ $3;
   set jsdata;
diff --git a/sas/sasjs/services/public/viewdata.sas b/sas/sasjs/services/public/viewdata.sas
index bbeda72..a25504f 100644
--- a/sas/sasjs/services/public/viewdata.sas
+++ b/sas/sasjs/services/public/viewdata.sas
@@ -282,6 +282,9 @@ run;
     data work.vwsearch/view=work.vwsearch;
       set &filterds;
       where %inc filtref;;
+      %if %length(&derived_droplist)>0 %then %do;
+        drop &derived_droplist;
+      %end;
     run;
     %if %upcase(&searchtype)=CHAR %then %do;
       %mp_searchdata(lib=work

From 3ede98fb187613b15bcf52f6e8fff5785c2c368d Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 03:31:08 +0000
Subject: [PATCH 09/48] test(derived): cover mpe_derived directly

The macro carries the ordering and statement-generation logic, so it is
worth a test of its own:

- character, numeric and chained variables: statements, view, columns
- label and default/configured formats
- a filter on a derived column
- the no-config path: no view, no statements, empty outvars

The configuration checks call mp_abort, so they stay out of the suite.
---
 sas/sasjs/macros/mpe_derived.test.sas | 236 ++++++++++++++++++++++++++
 1 file changed, 236 insertions(+)
 create mode 100644 sas/sasjs/macros/mpe_derived.test.sas

diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
new file mode 100644
index 0000000..de37837
--- /dev/null
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -0,0 +1,236 @@
+/**
+  @file
+  @brief testing the mpe_derived macro
+
+  @details Covers the generated statements, the view, the metadata the callers
+  consume, and the ordering rules.
+
+  The configuration checks abort the run when they fail, so they are not
+  exercised here - this suite covers the behaviour that callers depend on.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertdsobs.sas
+  @li mpe_derived.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+/* a source table, and a table with no derived variables */
+data work.class;
+  set sashelp.class;
+run;
+data work.noconfig;
+  set sashelp.class;
+run;
+
+/* clean up any config left behind by an earlier run */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;
+
+/**
+  * Test 1 - statements, view, chaining
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,variable_nm='BAND'
+    ,variable_order=10
+    ,variable_type='C'
+    ,variable_length=8
+    ,variable_format=''
+    ,variable_logic="ifc(AGE>13,'TEEN','JUNIOR')"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,variable_nm='BAND_AGE'
+    ,variable_order=20
+    ,variable_type='C'
+    ,variable_length=16
+    ,variable_format=''
+    ,variable_logic="cats(BAND,'-',AGE)"
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,variable_nm='DOUBLE_AGE'
+    ,variable_order=30
+    ,variable_type='N'
+    ,variable_length=8
+    ,variable_format=''
+    ,variable_logic='AGE * 2'
+    ,active=1;
+quit;
+
+/* lower case libds on purpose - the config match must be case insensitive */
+%mpe_derived(libds=work.class,
+  dclib=&dclib,
+  outref=derref1,
+  outview=work.vw_derived,
+  outvars=work.derived_vars
+)
+
+%mp_assertdsobs(work.derived_vars,
+  desc=Test1 - one row per derived variable,
+  test=EQUALS 3,
+  outds=work.test_results
+)
+
+%mp_assertcols(work.vw_derived,
+  desc=Test1 - the view carries the derived columns,
+  cols=BAND BAND_AGE DOUBLE_AGE,
+  test=ALL,
+  outds=work.test_results
+)
+
+data work.chk1;
+  set work.vw_derived;
+  where BAND='TEEN';
+run;
+%mp_assertdsobs(work.chk1,
+  desc=Test1 - the filter on a derived column selects rows,
+  test=HASOBS,
+  outds=work.test_results
+)
+
+/* the chained value only resolves if the statements run in order */
+data work.chk1b;
+  set work.vw_derived;
+  where BAND='TEEN';
+  if BAND_AGE ne cats(BAND,'-',AGE) then output;
+run;
+%mp_assertdsobs(work.chk1b,
+  desc=Test1 - a derived variable can reference an earlier one,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+data work.chk1c;
+  set work.vw_derived;
+  if DOUBLE_AGE ne AGE*2 then output;
+run;
+%mp_assertdsobs(work.chk1c,
+  desc=Test1 - the numeric derived value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/* the label is derived from the expression, and the format from the length */
+proc contents noprint data=work.vw_derived
+  out=work.vw_cols(keep=name label format);
+run;
+data work.chk1d;
+  set work.vw_cols;
+  where name='BAND' and label=:'DERIVED:' and format='$8.';
+run;
+%mp_assertdsobs(work.chk1d,
+  desc=Test1 - the label and default format are applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - a configured format is used as-is
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,variable_nm='FLAG_DT'
+    ,variable_order=40
+    ,variable_type='N'
+    ,variable_length=8
+    ,variable_format='DATE9.'
+    ,variable_logic='mdy(1,1,2020)'
+    ,active=1;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref2,
+  outview=work.vw_derived2,
+  outvars=work.derived_vars2
+)
+
+proc contents noprint data=work.vw_derived2
+  out=work.vw_cols2(keep=name label format);
+run;
+data work.chk2;
+  set work.vw_cols2;
+  where name='FLAG_DT' and format='DATE9.';
+run;
+%mp_assertdsobs(work.chk2,
+  desc=Test2 - the configured format is applied,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+data work.chk2b;
+  set work.vw_derived2;
+  if flag_dt ne mdy(1,1,2020) then output;
+  stop;
+run;
+%mp_assertdsobs(work.chk2b,
+  desc=Test2 - the numeric value is correct,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - no configuration
+  */
+
+%mpe_derived(libds=WORK.NOCONFIG,
+  dclib=&dclib,
+  outref=derref3,
+  outview=work.vw_derived3,
+  outvars=work.derived_vars3
+)
+
+%mp_assertdsobs(work.derived_vars3,
+  desc=Test3 - no derived variables returned,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
+%mp_assert(iftrue=(%mf_existds(libds=work.vw_derived3)=0),
+  desc=Test3 - no view is built when there is no config,
+  outds=work.test_results
+)
+
+data _null_;
+  infile derref3 end=eof;
+  input;
+  if not eof or cats(_infile_) ne '' then do;
+    putlog 'ERR' 'OR: derived statement file is not empty';
+    call symputx('emptycheck','0','l');
+  end;
+run;
+%mp_assert(iftrue=(%symexist(emptycheck)=0),
+  desc=Test3 - no statements are generated,
+  outds=work.test_results
+)
+
+/* tidy up so repeated runs start clean */
+proc sql;
+delete from &dclib..mpe_derived where upcase(libref)='WORK';
+quit;

From 52803225dfc5ac33b18cdd4a1ed71d19d30e787e Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 07:57:49 +0000
Subject: [PATCH 10/48] refactor(derived): shorten the MPE_DERIVED column names
 to VAR_*

MPE_DERIVED columns:
  VARIABLE_NM     -> VAR_NM
  VARIABLE_ORDER  -> VAR_ORDER
  VARIABLE_TYPE   -> VAR_TYPE
  VARIABLE_LENGTH -> VAR_LEN
  VARIABLE_FORMAT -> VAR_FMT
  VARIABLE_LOGIC  -> VAR_LOGIC

Updated in the ddl, mpe_makedatamodel, the migration, the MPE_TABLES buskey,
the macro (reads and validation messages), the tests, the mocks and the docs.
---
 sas/mocks/sasjs/services/admin/makedata.js    | 48 ++++++++---------
 sas/mocks/sasjs/services/dcMockUtils.js       | 16 +++---
 sas/sasjs/db/datactrl/mpe_derived.ddl         | 14 ++---
 sas/sasjs/db/datactrl/mpe_tables.ddl          |  2 +-
 .../db/migrations/20261019_v7.19_release.sas  | 16 +++---
 sas/sasjs/macros/mpe_derived.sas              | 54 +++++++++----------
 sas/sasjs/macros/mpe_derived.test.sas         | 48 ++++++++---------
 sas/sasjs/macros/mpe_makedata.sas             |  2 +-
 sas/sasjs/macros/mpe_makedatamodel.sas        |  2 +-
 sas/sasjs/services/editors/getdata.test.sas   | 38 ++++++-------
 .../services/public/getcolvals.test.1.sas     | 14 ++---
 .../services/public/validatefilter.test.sas   | 20 +++----
 12 files changed, 137 insertions(+), 137 deletions(-)

diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index 6690503..3c548da 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -291,12 +291,12 @@ const tables = {
       tx_to: 253717919999,
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
-      variable_nm: 'REGION',
-      variable_order: 10,
-      variable_type: 'C',
-      variable_length: 8,
-      variable_format: '',
-      variable_logic: "ifc(STATE in ('CA','OR','WA'),'West','East')",
+      var_nm: 'REGION',
+      var_order: 10,
+      var_type: 'C',
+      var_len: 8,
+      var_fmt: '',
+      var_logic: "ifc(STATE in ('CA','OR','WA'),'West','East')",
       active: 1
     },
     {
@@ -304,12 +304,12 @@ const tables = {
       tx_to: 253717919999,
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
-      variable_nm: 'LINE_VALUE',
-      variable_order: 20,
-      variable_type: 'N',
-      variable_length: 8,
-      variable_format: 'dollar12.2',
-      variable_logic: 'QTY * PRICE',
+      var_nm: 'LINE_VALUE',
+      var_order: 20,
+      var_type: 'N',
+      var_len: 8,
+      var_fmt: 'dollar12.2',
+      var_logic: 'QTY * PRICE',
       active: 1
     },
     {
@@ -318,12 +318,12 @@ const tables = {
       tx_to: 253717919999,
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
-      variable_nm: 'REGION_LABEL',
-      variable_order: 30,
-      variable_type: 'C',
-      variable_length: 14,
-      variable_format: '',
-      variable_logic: "cats(REGION,'-',STATE)",
+      var_nm: 'REGION_LABEL',
+      var_order: 30,
+      var_type: 'C',
+      var_len: 14,
+      var_fmt: '',
+      var_logic: "cats(REGION,'-',STATE)",
       active: 1
     }
   ],
@@ -3312,12 +3312,12 @@ const schema = {
     { name: 'TX_FROM', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true },
     { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
     { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
-    { name: 'VARIABLE_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
-    { name: 'VARIABLE_ORDER', type: 'N', length: 8, format: '', label: '', notnull: false },
-    { name: 'VARIABLE_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
-    { name: 'VARIABLE_LENGTH', type: 'N', length: 8, format: '', label: '', notnull: false },
-    { name: 'VARIABLE_FORMAT', type: 'C', length: 32, format: '', label: '', notnull: false },
-    { name: 'VARIABLE_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'VAR_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_ORDER', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
+    { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
     { name: 'ACTIVE', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'TX_TO', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true }
   ],
diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index f98a54e..cbae51a 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1320,7 +1320,7 @@ function compileSasExpression(expr, columnNames) {
 
 /**
  * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
- * variables with a compiled row function, ordered by VARIABLE_ORDER (then
+ * variables with a compiled row function, ordered by VAR_ORDER (then
  * name), the same sequence the real macro emits.
  *
  * @param {Object} opts
@@ -1348,14 +1348,14 @@ function mpeDerived(opts) {
             (!Number(colVal(r, 'TX_TO')) || Number(colVal(r, 'TX_TO')) > now)
         )
         .map(r => {
-            const logic = String(colVal(r, 'VARIABLE_LOGIC') || '').trim()
+            const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
             return {
-                name: String(colVal(r, 'VARIABLE_NM') || '').toUpperCase(),
+                name: String(colVal(r, 'VAR_NM') || '').toUpperCase(),
                 // statement order - a variable may reference one with a lower order
-                order: Number(colVal(r, 'VARIABLE_ORDER')) || 0,
-                type: String(colVal(r, 'VARIABLE_TYPE') || '').toUpperCase(),
-                length: Number(colVal(r, 'VARIABLE_LENGTH')) || 8,
-                format: String(colVal(r, 'VARIABLE_FORMAT') || ''),
+                order: Number(colVal(r, 'VAR_ORDER')) || 0,
+                type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
+                length: Number(colVal(r, 'VAR_LEN')) || 8,
+                format: String(colVal(r, 'VAR_FMT') || ''),
                 logic,
                 label: 'DERIVED: ' + logic
             }
@@ -1363,7 +1363,7 @@ function mpeDerived(opts) {
         .filter(d => {
             if (!d.name || (d.type !== 'C' && d.type !== 'N') || !d.logic) return false
             if (!(d.order >= 1)) {
-                console.log('derived variable ' + d.name + ' has no VARIABLE_ORDER - skipped')
+                console.log('derived variable ' + d.name + ' has no VAR_ORDER - skipped')
                 return false
             }
             return true
diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
index 344c2b5..d37ec1e 100644
--- a/sas/sasjs/db/datactrl/mpe_derived.ddl
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -19,13 +19,13 @@ create table &curlib..MPE_DERIVED
     TX_FROM float not null format=datetime19.,
     LIBREF varchar(8),
     DSN varchar(32),
-    VARIABLE_NM varchar(32),
-    VARIABLE_ORDER int,
-    VARIABLE_TYPE varchar(1),
-    VARIABLE_LENGTH int,
-    VARIABLE_FORMAT varchar(32),
-    VARIABLE_LOGIC varchar(4000),
+    VAR_NM varchar(32),
+    VAR_ORDER int,
+    VAR_TYPE varchar(1),
+    VAR_LEN int,
+    VAR_FMT varchar(32),
+    VAR_LOGIC varchar(4000),
     ACTIVE int,
     TX_TO float not null format=datetime19.,
     constraint pk_mpe_derived
-        primary key(tx_from, libref, dsn, variable_nm));
+        primary key(tx_from, libref, dsn, var_nm));
diff --git a/sas/sasjs/db/datactrl/mpe_tables.ddl b/sas/sasjs/db/datactrl/mpe_tables.ddl
index 756be8c..50a319f 100644
--- a/sas/sasjs/db/datactrl/mpe_tables.ddl
+++ b/sas/sasjs/db/datactrl/mpe_tables.ddl
@@ -125,7 +125,7 @@ insert into &mpelib..mpe_tables
       ,dsn='MPE_DERIVED'
       ,num_of_approvals_required=1
       ,loadtype='TXTEMPORAL'
-      ,buskey='LIBREF DSN VARIABLE_NM'
+      ,buskey='LIBREF DSN VAR_NM'
       ,notes='Derived (computed) variables for the EDIT screen'
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index a6b332f..d04ecc9 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -22,12 +22,12 @@ create table &dclib..mpe_derived
     tx_from num not null format=datetime19.,
     libref varchar(8),
     dsn varchar(32),
-    variable_nm varchar(32),
-    variable_order num,
-    variable_type varchar(1),
-    variable_length num,
-    variable_format varchar(32),
-    variable_logic varchar(4000),
+    var_nm varchar(32),
+    var_order num,
+    var_type varchar(1),
+    var_len num,
+    var_fmt varchar(32),
+    var_logic varchar(4000),
     active num,
     tx_to num not null format=datetime19.
   );
@@ -36,7 +36,7 @@ quit;
 proc datasets lib=&dclib noprint;
   modify mpe_derived;
   index create
-    pk_mpe_derived=(tx_from libref dsn variable_nm)
+    pk_mpe_derived=(tx_from libref dsn var_nm)
     /nomiss unique;
 quit;
 
@@ -48,7 +48,7 @@ insert into &dclib..mpe_tables
     ,dsn='MPE_DERIVED'
     ,num_of_approvals_required=1
     ,loadtype='TXTEMPORAL'
-    ,buskey='LIBREF DSN VARIABLE_NM'
+    ,buskey='LIBREF DSN VAR_NM'
     ,notes='Derived (computed) variables for the EDIT screen'
     ,var_txfrom='TX_FROM'
     ,var_txto='TX_TO'
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 0fdf493..e69e178 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -37,7 +37,7 @@
   deterministic.  The label defaults to the expression itself, prefixed with
   DERIVED: so that computed columns are obvious in the grid.
 
-  Statements are generated in VARIABLE_ORDER sequence, so a derived variable
+  Statements are generated in VAR_ORDER sequence, so a derived variable
   may reference another derived variable with a lower order.  A reference to a
   variable that is not defined earlier is rejected, because SAS would otherwise
   treat it as an uninitialised variable and return a missing value without an
@@ -107,18 +107,18 @@ data work.derived_cfg;
     and upcase(libref)="%upcase(%scan(&libds,1,.))"
     and upcase(dsn)="%upcase(%scan(&libds,2,.))";
   length name $32 logic $4000 fmt $32;
-  name=upcase(variable_nm);
-  logic=strip(variable_logic);
-  type=upcase(variable_type);
-  len=variable_length;
-  fmt=strip(upcase(variable_format));
-  ord=variable_order;
+  name=upcase(var_nm);
+  logic=strip(var_logic);
+  type=upcase(var_type);
+  len=var_len;
+  fmt=strip(upcase(var_fmt));
+  ord=var_order;
 run;
 
 %if %mf_nobs(work.derived_cfg)=0 %then %return;
 
 /* statement order is the configured order - a derived variable may reference
-  one with a lower VARIABLE_ORDER, so they must be assigned in this sequence */
+  one with a lower VAR_ORDER, so they must be assigned in this sequence */
 proc sort data=work.derived_cfg out=work.derived_srt;
   by ord name;
 run;
@@ -138,39 +138,39 @@ data work.derived_bad;
   length reason $200;
   call missing(reason);
 
-  if name='' then reason='VARIABLE_NM is required';
+  if name='' then reason='VAR_NM is required';
   else if verify(name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
-    then reason='VARIABLE_NM must be a valid SAS name';
+    then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(name,1,1),'0123456789')=0
-    then reason='VARIABLE_NM must not start with a digit';
+    then reason='VAR_NM must not start with a digit';
   else if name in (%unquote(&physlist))
-    then reason='VARIABLE_NM collides with a column in the target table';
+    then reason='VAR_NM collides with a column in the target table';
   else if name in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
     ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
-    then reason='VARIABLE_NM is reserved';
+    then reason='VAR_NM is reserved';
   else if missing(ord) or ord ne int(ord) or ord<1
-    then reason='VARIABLE_ORDER must be a positive integer';
+    then reason='VAR_ORDER must be a positive integer';
   else if type not in ('C','N')
-    then reason='VARIABLE_TYPE must be C or N';
+    then reason='VAR_TYPE must be C or N';
   else if missing(len) or len ne int(len) or len<1
-    then reason='VARIABLE_LENGTH must be a positive integer';
+    then reason='VAR_LEN must be a positive integer';
   else if type='C' and len>32767
-    then reason='VARIABLE_LENGTH exceeds 32767 for a character variable';
+    then reason='VAR_LEN exceeds 32767 for a character variable';
   else if type='N' and len>8
-    then reason='VARIABLE_LENGTH exceeds 8 for a numeric variable';
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
   else if index(fmt,';')>0 or indexc(fmt,'&%')>0
     or index(fmt,'/*')>0 or index(fmt,'*/')>0
-    then reason='VARIABLE_FORMAT must be a format name, without ; & % or //*';
+    then reason='VAR_FMT must be a format name, without ; & % or //*';
   else if verify(fmt,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
-    then reason='VARIABLE_FORMAT must be a format name';
+    then reason='VAR_FMT must be a format name';
   else if missing(logic)
-    then reason='VARIABLE_LOGIC is required';
+    then reason='VAR_LOGIC is required';
   else if index(logic,';')>0
-    then reason='VARIABLE_LOGIC must be a single expression - remove the ;';
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
   else if indexc(logic,'&%')>0
-    then reason='VARIABLE_LOGIC must not contain macro triggers';
+    then reason='VAR_LOGIC must not contain macro triggers';
   else if index(logic,'/*')>0 or index(logic,'*/')>0
-    then reason='VARIABLE_LOGIC must not contain comments';
+    then reason='VAR_LOGIC must not contain comments';
 
   if missing(reason)=0 then output;
 run;
@@ -180,7 +180,7 @@ data work.derived_dup;
   by name;
   length reason $200;
   if first.name and not last.name;
-  reason='VARIABLE_NM is defined more than once';
+  reason='VAR_NM is defined more than once';
 run;
 
 proc append base=work.derived_bad data=work.derived_dup;
@@ -211,8 +211,8 @@ data work.derived_orderbad;
   by name;
   if in2;
   length reason $200;
-  reason='VARIABLE_LOGIC references a derived variable that is not defined'
-    !!' earlier - lower the VARIABLE_ORDER of this one';
+  reason='VAR_LOGIC references a derived variable that is not defined'
+    !!' earlier - lower the VAR_ORDER of this one';
 run;
 
 proc append base=work.derived_bad data=work.derived_orderbad;
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index de37837..48809aa 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -45,36 +45,36 @@ insert into &dclib..mpe_derived set
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref='WORK'
     ,dsn='CLASS'
-    ,variable_nm='BAND'
-    ,variable_order=10
-    ,variable_type='C'
-    ,variable_length=8
-    ,variable_format=''
-    ,variable_logic="ifc(AGE>13,'TEEN','JUNIOR')"
+    ,var_nm='BAND'
+    ,var_order=10
+    ,var_type='C'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic="ifc(AGE>13,'TEEN','JUNIOR')"
     ,active=1;
 insert into &dclib..mpe_derived set
     tx_from=0
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref='WORK'
     ,dsn='CLASS'
-    ,variable_nm='BAND_AGE'
-    ,variable_order=20
-    ,variable_type='C'
-    ,variable_length=16
-    ,variable_format=''
-    ,variable_logic="cats(BAND,'-',AGE)"
+    ,var_nm='BAND_AGE'
+    ,var_order=20
+    ,var_type='C'
+    ,var_len=16
+    ,var_fmt=''
+    ,var_logic="cats(BAND,'-',AGE)"
     ,active=1;
 insert into &dclib..mpe_derived set
     tx_from=0
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref='WORK'
     ,dsn='CLASS'
-    ,variable_nm='DOUBLE_AGE'
-    ,variable_order=30
-    ,variable_type='N'
-    ,variable_length=8
-    ,variable_format=''
-    ,variable_logic='AGE * 2'
+    ,var_nm='DOUBLE_AGE'
+    ,var_order=30
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE * 2'
     ,active=1;
 quit;
 
@@ -155,12 +155,12 @@ insert into &dclib..mpe_derived set
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref='WORK'
     ,dsn='CLASS'
-    ,variable_nm='FLAG_DT'
-    ,variable_order=40
-    ,variable_type='N'
-    ,variable_length=8
-    ,variable_format='DATE9.'
-    ,variable_logic='mdy(1,1,2020)'
+    ,var_nm='FLAG_DT'
+    ,var_order=40
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt='DATE9.'
+    ,var_logic='mdy(1,1,2020)'
     ,active=1;
 quit;
 
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index 7c41c41..cb905ba 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -1502,7 +1502,7 @@ insert into &lib..mpe_selectbox set
       ,dsn='MPE_DERIVED'
       ,num_of_approvals_required=1
       ,loadtype='TXTEMPORAL'
-      ,buskey='LIBREF DSN VARIABLE_NM'
+      ,buskey='LIBREF DSN VAR_NM'
       ,notes='Derived (computed) variables for the EDIT screen'
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index 98e6258..163c2df 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -711,7 +711,7 @@ create table &lib..MPE_DERIVED
 proc datasets lib=&lib noprint;
   modify mpe_derived;
   index create
-    pk_mpe_derived=(tx_from libref dsn variable_nm)
+    pk_mpe_derived=(tx_from libref dsn var_nm)
     /nomiss unique;
 quit;
 proc sql;
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index 22a95c9..6698663 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -26,11 +26,11 @@
 
 data work.inquery1;
   length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8
-    VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
+    VAR_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
   GROUP_LOGIC='AND';
   SUBGROUP_LOGIC='AND';
   SUBGROUP_ID=1;
-  VARIABLE_NM='LIBREF';
+  VAR_NM='LIBREF';
   OPERATOR_NM='=';
   RAW_VALUE="'&dclib'";
   output;
@@ -90,7 +90,7 @@ run;
 
 %mp_assertcols(work.query,
   desc=Query dataset carries the filter columns,
-  cols=VARIABLE_NM RAW_VALUE SUBGROUP_ID,
+  cols=VAR_NM RAW_VALUE SUBGROUP_ID,
   test=ALL,
   outds=work.test_results
 )
@@ -108,7 +108,7 @@ run;
 /* first, make filter */
 data work.inquery;
   infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,LIBREF,CONTAINS,"'DC'"
@@ -188,7 +188,7 @@ run;
 /* now, make filter */
 data work.inquery3;
   infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
   RAW_VALUE="'&fmtname'";
 datalines4;
@@ -257,24 +257,24 @@ insert into &dc_libref..mpe_derived set
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
-    ,variable_nm='DERIVED_NOTE'
-    ,variable_order=10
-    ,variable_type='C'
-    ,variable_length=41
-    ,variable_format=''
-    ,variable_logic="cats(libref,'.',dsn)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_order=10
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
     ,active=1;
 insert into &dc_libref..mpe_derived set
     tx_from=0
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
-    ,variable_nm='DERIVED_NOTE2'
-    ,variable_order=20
-    ,variable_type='C'
-    ,variable_length=45
-    ,variable_format=''
-    ,variable_logic="cats(DERIVED_NOTE,' (2)')"
+    ,var_nm='DERIVED_NOTE2'
+    ,var_order=20
+    ,var_type='C'
+    ,var_len=45
+    ,var_fmt=''
+    ,var_logic="cats(DERIVED_NOTE,' (2)')"
     ,active=1;
 quit;
 
@@ -288,7 +288,7 @@ quit;
 
 data work.inquery4;
   infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
@@ -388,7 +388,7 @@ run;
 /* tidy up the config so repeated runs start clean */
 proc sql;
 delete from &dc_libref..mpe_derived
-  where variable_nm='DERIVED_NOTE'
+  where var_nm='DERIVED_NOTE'
     and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
     and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
 quit;
diff --git a/sas/sasjs/services/public/getcolvals.test.1.sas b/sas/sasjs/services/public/getcolvals.test.1.sas
index a01ab3b..b324406 100644
--- a/sas/sasjs/services/public/getcolvals.test.1.sas
+++ b/sas/sasjs/services/public/getcolvals.test.1.sas
@@ -51,12 +51,12 @@ insert into &dc_libref..mpe_derived set
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
     ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
-    ,variable_nm='DERIVED_KEY'
-    ,variable_order=10
-    ,variable_type='C'
-    ,variable_length=20
-    ,variable_format=''
-    ,variable_logic='cats(primary_key_field)'
+    ,var_nm='DERIVED_KEY'
+    ,var_order=10
+    ,var_type='C'
+    ,var_len=20
+    ,var_fmt=''
+    ,var_logic='cats(primary_key_field)'
     ,active=1;
 quit;
 
@@ -84,7 +84,7 @@ run;
 /* tidy up the config so repeated runs start clean */
 proc sql;
 delete from &dc_libref..mpe_derived
-  where variable_nm='DERIVED_KEY'
+  where var_nm='DERIVED_KEY'
     and upcase(libref)="%scan(&dclib..MPE_X_TEST,1,.)"
     and upcase(dsn)="%scan(&dclib..MPE_X_TEST,2,.)";
 quit;
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index bfae64f..cefe4e8 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -26,7 +26,7 @@ data _null_;
   input;
   put _infile_;
 datalines4;
-GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. OPERATOR_NM:$10. RAW_VALUE:$4000.
+GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32. OPERATOR_NM:$10. RAW_VALUE:$4000.
 AND,AND,1,LIBREF,CONTAINS,"'DC'"
 AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
 ;;;;
@@ -53,7 +53,7 @@ data work.filterquery;
 GROUP_LOGIC='AND';
 SUBGROUP_LOGIC='AND';
 SUBGROUP_ID=1;
-VARIABLE_NM='FMTNAME';
+VAR_NM='FMTNAME';
 OPERATOR_NM='CONTAINS';
 RAW_VALUE="'MORD'";
 ;;;;
@@ -91,12 +91,12 @@ insert into &dc_libref..mpe_derived set
     ,tx_to='31DEC5999:23:59:59'dt
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
-    ,variable_nm='DERIVED_NOTE'
-    ,variable_order=10
-    ,variable_type='C'
-    ,variable_length=41
-    ,variable_format=''
-    ,variable_logic="cats(libref,'.',dsn)"
+    ,var_nm='DERIVED_NOTE'
+    ,var_order=10
+    ,var_type='C'
+    ,var_len=41
+    ,var_fmt=''
+    ,var_logic="cats(libref,'.',dsn)"
     ,active=1;
 quit;
 
@@ -107,7 +107,7 @@ data work.filterquery;
   GROUP_LOGIC='AND';
   SUBGROUP_LOGIC='AND';
   SUBGROUP_ID=1;
-  VARIABLE_NM='DERIVED_NOTE';
+  VAR_NM='DERIVED_NOTE';
   OPERATOR_NM='CONTAINS';
   RAW_VALUE="'MPE_LOCK'";
   output;
@@ -136,7 +136,7 @@ run;
 /* tidy up the config so repeated runs start clean */
 proc sql;
 delete from &dc_libref..mpe_derived
-  where variable_nm='DERIVED_NOTE'
+  where var_nm='DERIVED_NOTE'
     and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
     and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
 quit;

From 0b010f2c65ddada70d62ea83c706943c6c45756f Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 08:20:52 +0000
Subject: [PATCH 11/48] feat(derived): validate MPE_DERIVED at submit time

A post edit hook on MPE_DERIVED applies the configuration rules when a row
is submitted, rather than only when the EDIT screen next builds the view.

It normalises LIBREF, DSN, VAR_NM, VAR_TYPE and VAR_FMT, and rejects:
  - a missing LIBREF, DSN or VAR_NM
  - an invalid or reserved name, or one starting with a digit
  - a VAR_ORDER that is not a positive integer
  - a VAR_TYPE other than C or N
  - a VAR_LEN outside 1-32767 (character) or 1-8 (numeric)
  - a VAR_FMT that is not a plain format name
  - a VAR_LOGIC that is not a single expression
  - the same VAR_NM twice for one table
  - a reference to a derived variable that is not defined earlier

The name collision with a physical column is left to %mpe_derived, which runs
with the target library assigned; the hook does not have it.
---
 sas/sasjs/db/datactrl/mpe_tables.ddl          |   1 +
 .../db/migrations/20261019_v7.19_release.sas  |   1 +
 sas/sasjs/macros/mpe_makedata.sas             |   1 +
 .../services/hooks/mpe_derived_postedit.sas   | 171 ++++++++++++++++++
 4 files changed, 174 insertions(+)
 create mode 100644 sas/sasjs/services/hooks/mpe_derived_postedit.sas

diff --git a/sas/sasjs/db/datactrl/mpe_tables.ddl b/sas/sasjs/db/datactrl/mpe_tables.ddl
index 50a319f..aa68788 100644
--- a/sas/sasjs/db/datactrl/mpe_tables.ddl
+++ b/sas/sasjs/db/datactrl/mpe_tables.ddl
@@ -126,6 +126,7 @@ insert into &mpelib..mpe_tables
       ,num_of_approvals_required=1
       ,loadtype='TXTEMPORAL'
       ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
       ,notes='Derived (computed) variables for the EDIT screen'
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index d04ecc9..32a6dac 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -49,6 +49,7 @@ insert into &dclib..mpe_tables
     ,num_of_approvals_required=1
     ,loadtype='TXTEMPORAL'
     ,buskey='LIBREF DSN VAR_NM'
+    ,post_edit_hook='services/hooks/mpe_derived_postedit'
     ,notes='Derived (computed) variables for the EDIT screen'
     ,var_txfrom='TX_FROM'
     ,var_txto='TX_TO'
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index cb905ba..36570b3 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -1503,6 +1503,7 @@ insert into &lib..mpe_selectbox set
       ,num_of_approvals_required=1
       ,loadtype='TXTEMPORAL'
       ,buskey='LIBREF DSN VAR_NM'
+      ,post_edit_hook='services/hooks/mpe_derived_postedit'
       ,notes='Derived (computed) variables for the EDIT screen'
       ,var_txfrom='TX_FROM'
       ,var_txto='TX_TO'
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
new file mode 100644
index 0000000..66e5e9d
--- /dev/null
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -0,0 +1,171 @@
+/**
+  @file
+  @brief Post Edit Hook script for the MPE_DERIVED table
+  @details Post edit hooks provide additional backend validation for user
+  provided data.  The incoming dataset is named `work.staging_ds` and is
+  provided in mpe_loader.sas.
+
+  Available macro variables:
+    @li DC_LIBREF - The DC control library
+    @li LIBREF - The library of the dataset being edited (is assigned)
+    @li DS - The dataset being edited
+
+  This hook normalises the incoming rows and applies the configuration rules
+  at submit time, so a bad derived variable is rejected while the submitter is
+  still watching - rather than breaking the EDIT screen later, when the view
+  is built.
+
+  The target table itself is not assigned in this context, so the collision
+  between VAR_NM and a physical column is left to %mpe_derived, which runs
+  with the target library assigned.
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mpe_derived.sas
+  @li mpe_loader.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%let errflag=0;
+%let errmsg=;
+
+/* normalise the incoming values */
+data work.staging_ds;
+  set work.staging_ds;
+  if upcase(_____DELETE__THIS__RECORD_____) ne 'YES' then do;
+    libref=upcase(strip(libref));
+    dsn=upcase(strip(dsn));
+    var_nm=upcase(strip(var_nm));
+    var_type=upcase(strip(var_type));
+    var_fmt=upcase(strip(var_fmt));
+    var_logic=strip(var_logic);
+  end;
+run;
+
+/**
+  * Field level checks - the same rules %mpe_derived applies at read time
+  */
+data work.derived_check;
+  set work.staging_ds;
+  length reason $200;
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' then delete;
+  call missing(reason);
+
+  if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
+  else if var_nm='' then reason='VAR_NM is required';
+  else if verify(var_nm,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+    then reason='VAR_NM must be a valid SAS name';
+  else if verify(substr(var_nm,1,1),'0123456789')=0
+    then reason='VAR_NM must not start with a digit';
+  else if var_nm in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
+    ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
+    then reason='VAR_NM is reserved';
+  else if missing(var_order) or var_order ne int(var_order) or var_order<1
+    then reason='VAR_ORDER must be a positive integer';
+  else if var_type not in ('C','N')
+    then reason='VAR_TYPE must be C or N';
+  else if missing(var_len) or var_len ne int(var_len) or var_len<1
+    then reason='VAR_LEN must be a positive integer';
+  else if var_type='C' and var_len>32767
+    then reason='VAR_LEN exceeds 32767 for a character variable';
+  else if var_type='N' and var_len>8
+    then reason='VAR_LEN exceeds 8 for a numeric variable';
+  else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
+    or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
+    then reason='VAR_FMT must be a format name, without ; & % or //*';
+  else if verify(var_fmt,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+    then reason='VAR_FMT must be a format name';
+  else if missing(var_logic)
+    then reason='VAR_LOGIC is required';
+  else if index(var_logic,';')>0
+    then reason='VAR_LOGIC must be a single expression - remove the ;';
+  else if indexc(var_logic,'&%')>0
+    then reason='VAR_LOGIC must not contain macro triggers';
+  else if index(var_logic,'/*')>0 or index(var_logic,'*/')>0
+    then reason='VAR_LOGIC must not contain comments';
+
+  if not missing(reason) then output;
+run;
+
+%if %mf_nobs(work.derived_check)>0 %then %do;
+  data _null_;
+    set work.derived_check(obs=1);
+    call symputx('errmsg',cats('MPE_DERIVED: ',var_nm,': ',reason));
+    call symputx('errflag',1);
+  run;
+%end;
+
+%mp_abort(iftrue= (&errflag=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+/**
+  * A variable may only be defined once per table
+  */
+proc sql noprint;
+create table work.derived_dups as
+  select libref, dsn, var_nm, count(*) as cnt
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  group by 1,2,3
+  having cnt>1;
+
+%let errmsg=;
+select catx('.',libref,dsn,var_nm) into: errmsg
+  from work.derived_dups(obs=1);
+quit;
+
+%mp_abort(iftrue= (%mf_nobs(work.derived_dups)>0)
+  ,mac=mpe_derived_postedit
+  ,msg=%str(VAR_NM is defined more than once: &errmsg)
+)
+
+/**
+  * Both references must resolve in the resulting configuration, and only to a
+  * variable with a lower VAR_ORDER.  The resulting set is the staged rows plus
+  * the current rows the submission does not replace.
+  */
+proc sql noprint;
+create table work.derived_combined as
+  select libref, dsn, var_nm, var_order
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  union all
+  select libref, dsn, var_nm, var_order
+  from &DC_LIBREF..MPE_DERIVED
+  where &dc_dttmtfmt. lt tx_to
+    and catx('.',libref,dsn,var_nm) not in
+      (select catx('.',libref,dsn,var_nm) from work.staging_ds);
+
+create table work.derived_orderbad as
+  select distinct a.libref, a.dsn, a.var_nm
+  from work.staging_ds a
+  inner join work.derived_combined b
+    on a.libref=b.libref
+    and a.dsn=b.dsn
+  where upcase(a._____DELETE__THIS__RECORD_____) ne 'YES'
+    and a.var_nm ne b.var_nm
+    and prxmatch(
+      cats('/(^|[^A-Z0-9_])',b.var_nm,'([^A-Z0-9_]|$)/')
+      ,upcase(a.var_logic)
+    )>0
+    and b.var_order>=a.var_order;
+
+%let errmsg=;
+select catx('.',libref,dsn,var_nm) into: errmsg
+  from work.derived_orderbad(obs=1);
+quit;
+
+%mp_abort(iftrue= (%mf_nobs(work.derived_orderbad)>0)
+  ,mac=mpe_derived_postedit
+  ,msg=%str(VAR_LOGIC references a derived variable that is not defined earlier - lower the VAR_ORDER of &errmsg)
+)

From aa6d50e1ea5fc2fe6be8aa9fa859b92d0a1061c5 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 09:01:55 +0000
Subject: [PATCH 12/48] feat(derived): validation rules and a VAR_TYPE dropdown
 for MPE_DERIVED

Client side rules on the table itself, so mistakes show up in the grid:
  - NOTNULL on LIBREF, DSN, VAR_NM, VAR_ORDER, VAR_TYPE, VAR_LEN, VAR_LOGIC
  - UPCASE on LIBREF, DSN, VAR_NM, VAR_TYPE, VAR_FMT
  - MINVAL 1 / MAXVAL 999 on VAR_ORDER, MINVAL 1 / MAXVAL 32767 on VAR_LEN
  - HARDREGEX on VAR_NM, VAR_FMT and VAR_LOGIC
  - HARDSELECT on VAR_TYPE, with C and N in MPE_SELECTBOX

Seeded in mpe_makedata and in the v7.19 migration. The post edit hook stays
the server side enforcement, since these rules are evaluated in the browser.

Also fixes the MPE_DERIVED create in mpe_makedatamodel, which still used the
old VARIABLE_* column names.
---
 .../db/migrations/20261019_v7.19_release.sas  | 115 ++++++++++++++++++
 sas/sasjs/macros/mpe_makedata.sas             | 103 ++++++++++++++++
 sas/sasjs/macros/mpe_makedatamodel.sas        |  12 +-
 3 files changed, 224 insertions(+), 6 deletions(-)

diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index 32a6dac..a28e288 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -55,3 +55,118 @@ insert into &dclib..mpe_tables
     ,var_txto='TX_TO'
   ;
 quit;
+
+/* Dropdown values for VAR_TYPE */
+proc sql noprint;
+  select max(selectbox_rk) into: maxrk from &dclib..mpe_selectbox;
+quit;
+
+%let rk1=%eval(&maxrk+1);
+%let rk2=%eval(&maxrk+2);
+
+proc sql;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk1
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='C'
+    ,selectbox_order=1
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+insert into &dclib..mpe_selectbox
+  set selectbox_rk=&rk2
+    ,ver_from_dttm=0
+    ,select_lib="&dclib"
+    ,select_ds="MPE_DERIVED"
+    ,base_column="VAR_TYPE"
+    ,selectbox_value='N'
+    ,selectbox_order=2
+    ,ver_to_dttm='31DEC5999:23:59:59'dt;
+quit;
+
+/* Field rules for the EDIT screen.  These are client side - the server side
+  * enforcement is the post edit hook registered above. */
+proc sql;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
+    ,rule_type='MAXVAL',rule_value='999',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+quit;
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index 36570b3..c76f1fb 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -679,6 +679,24 @@ insert into &lib..mpe_security set
       ,selectbox_value='UNLOCKED'
       ,selectbox_order=2
       ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='C'
+      ,selectbox_order=1
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
+  insert into &lib..mpe_selectbox set
+      selectbox_rk=%mf_increment(rk)
+      ,ver_from_dttm=0
+      ,select_lib="&lib"
+      ,select_ds="MPE_DERIVED"
+      ,base_column="VAR_TYPE"
+      ,selectbox_value='N'
+      ,selectbox_order=2
+      ,ver_to_dttm='31DEC5999:23:59:59'dt;
   insert into &lib..mpe_selectbox set
       selectbox_rk=%mf_increment(rk)
       ,ver_from_dttm=0
@@ -1957,6 +1975,91 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_value="services/validations/columns_in_libds"
     ,rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
+
+/* MPE_DERIVED - the field rules are also enforced server-side, in
+  * services/hooks/mpe_derived_postedit.sas.  These rows give the user
+  * immediate feedback in the EDIT screen. */
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="LIBREF"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="DSN"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_NM"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
+    ,rule_type='MAXVAL',rule_value='999',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
+    ,rule_type='HARDSELECT',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LEN"
+    ,rule_type='MAXVAL',rule_value='32767',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
+    ,rule_type='HARDREGEX'
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?)?$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
+    ,rule_type='HARDREGEX',rule_value='^[^;&%]*$',rule_active=1
+    ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0
     ,base_lib="&lib"
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index 163c2df..15558fa 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -699,12 +699,12 @@ create table &lib..MPE_DERIVED
     TX_FROM num &notnull format=datetime19.3,
     LIBREF varchar(8),
     DSN varchar(32),
-    VARIABLE_NM varchar(32),
-    VARIABLE_ORDER num,
-    VARIABLE_TYPE varchar(1),
-    VARIABLE_LENGTH num,
-    VARIABLE_FORMAT varchar(32),
-    VARIABLE_LOGIC varchar(4000),
+    VAR_NM varchar(32),
+    VAR_ORDER num,
+    VAR_TYPE varchar(1),
+    VAR_LEN num,
+    VAR_FMT varchar(32),
+    VAR_LOGIC varchar(4000),
     ACTIVE num,
     TX_TO num &notnull format=datetime19.3
 );quit;

From 27ddcfb5759d6eb8f9880adc7c271a408e7741ad Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 09:07:18 +0000
Subject: [PATCH 13/48] fix(derived): raise the VAR_ORDER ceiling to 10000

The client side MAXVAL cannot express 'integer', so integrality stays in the
macro and the hook. The ceiling is only there to catch a typo.
---
 sas/sasjs/db/migrations/20261019_v7.19_release.sas | 2 +-
 sas/sasjs/macros/mpe_makedata.sas                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index a28e288..37765f6 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -126,7 +126,7 @@ insert into &dclib..MPE_VALIDATIONS set
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='MAXVAL',rule_value='999',rule_active=1
+    ,rule_type='MAXVAL',rule_value='10000',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index c76f1fb..ed09c4e 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -2017,7 +2017,7 @@ insert into &lib..MPE_VALIDATIONS set
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='MAXVAL',rule_value='999',rule_active=1
+    ,rule_type='MAXVAL',rule_value='10000',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"

From 8bae21230e9a8ea8c6a177a958a1600fbb2a4e72 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 09:56:52 +0000
Subject: [PATCH 14/48] refactor(derived): drop the duplicate name check from
 the hook

mpe_loader already sorts work.staging_ds by the business key
(buskey='LIBREF DSN VAR_NM') and aborts on duplicates before the post edit
hook runs, so the same check in the hook could never fire.

The check in %mpe_derived stays: it reads the control table, which can also
be edited outside the app.
---
 .../services/hooks/mpe_derived_postedit.sas   | 25 +++----------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 66e5e9d..d9525bd 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -17,7 +17,9 @@
 
   The target table itself is not assigned in this context, so the collision
   between VAR_NM and a physical column is left to %mpe_derived, which runs
-  with the target library assigned.
+  with the target library assigned.  Uniqueness on the business key
+  (LIBREF DSN VAR_NM) is already enforced by mpe_loader, so it is not
+  repeated here.
 
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
@@ -108,27 +110,6 @@ run;
   ,msg=%superq(errmsg)
 )
 
-/**
-  * A variable may only be defined once per table
-  */
-proc sql noprint;
-create table work.derived_dups as
-  select libref, dsn, var_nm, count(*) as cnt
-  from work.staging_ds
-  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
-  group by 1,2,3
-  having cnt>1;
-
-%let errmsg=;
-select catx('.',libref,dsn,var_nm) into: errmsg
-  from work.derived_dups(obs=1);
-quit;
-
-%mp_abort(iftrue= (%mf_nobs(work.derived_dups)>0)
-  ,mac=mpe_derived_postedit
-  ,msg=%str(VAR_NM is defined more than once: &errmsg)
-)
-
 /**
   * Both references must resolve in the resulting configuration, and only to a
   * variable with a lower VAR_ORDER.  The resulting set is the staged rows plus

From 4d9ad38df1fa89b0d5a12214d703d9c1fe5295f2 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 11:51:58 +0000
Subject: [PATCH 15/48] fix(derived): verify on stripped values, and label with
 the variable name

Found running the macro on a real Viya compute server:

- verify() was applied to fixed width variables, so the blank padding was
  always outside the allowed character list and every name was rejected.  The
  check now runs on strip(name) and strip(fmt).
- the empty outvars dataset used 'length name $32; stop;', which the Viya
  compute server rejects with 'Variable name is uninitialized'.  It now
  assigns the variable and subsets with 'if 0;'.
- the label is now <VAR_NM>=<expression> rather than 'DERIVED: <expression>',
  so the grid shows the variable name as well as the logic.
---
 sas/sasjs/macros/mpe_derived.sas                  | 15 ++++++++-------
 sas/sasjs/macros/mpe_derived.test.sas             |  4 ++--
 sas/sasjs/services/hooks/mpe_derived_postedit.sas |  4 ++--
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index e69e178..5dc3870 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -30,12 +30,13 @@
 
   Each statement block is of the form:
 
-      attrib REGION length=$21 label='DERIVED: <expression>' format=$21.;
+      attrib REGION length=$21 label='REGION=<expression>' format=$21.;
       REGION = <expression>;
 
   Type and length are declared by the administrator, so the result is
-  deterministic.  The label defaults to the expression itself, prefixed with
-  DERIVED: so that computed columns are obvious in the grid.
+  deterministic.  The label is the variable name, an equals sign and the
+  expression, so the grid shows both what the column is and where it comes
+  from.
 
   Statements are generated in VAR_ORDER sequence, so a derived variable
   may reference another derived variable with a lower order.  A reference to a
@@ -92,7 +93,7 @@ data _null_;
 run;
 data &outvars;
   length name $32;
-  stop;
+  if 0 then name='';
 run;
 
 %if %mf_existds(libds=&dclib..mpe_derived) ne 1 %then %do;
@@ -139,7 +140,7 @@ data work.derived_bad;
   call missing(reason);
 
   if name='' then reason='VAR_NM is required';
-  else if verify(name,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+  else if verify(strip(name),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
     then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(name,1,1),'0123456789')=0
     then reason='VAR_NM must not start with a digit';
@@ -161,7 +162,7 @@ data work.derived_bad;
   else if index(fmt,';')>0 or indexc(fmt,'&%')>0
     or index(fmt,'/*')>0 or index(fmt,'*/')>0
     then reason='VAR_FMT must be a format name, without ; & % or //*';
-  else if verify(fmt,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+  else if verify(strip(fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
     then reason='VAR_FMT must be a format name';
   else if missing(logic)
     then reason='VAR_LOGIC is required';
@@ -241,7 +242,7 @@ data _null_;
   set work.derived_srt;
   length stmt $4100 label $256 fmt2 $49;
   /* labels cannot contain single quotes, and are limited to 256 chars */
-  label=cats('DERIVED: ',translate(logic,' ','0D0A'x));
+  label=cats(name,'=',translate(logic,' ','0D0A'x));
   label=tranwrd(label,"'","''");
   label=substr(label,1,256);
   if not missing(fmt) and index(fmt,'.')=0 then fmt=cats(fmt,'.');
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 48809aa..42548ab 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -131,13 +131,13 @@ run;
   outds=work.test_results
 )
 
-/* the label is derived from the expression, and the format from the length */
+/* the label is the variable name plus the expression, and the format from the length */
 proc contents noprint data=work.vw_derived
   out=work.vw_cols(keep=name label format);
 run;
 data work.chk1d;
   set work.vw_cols;
-  where name='BAND' and label=:'DERIVED:' and format='$8.';
+  where name='BAND' and label=:'BAND=' and format='$8.';
 run;
 %mp_assertdsobs(work.chk1d,
   desc=Test1 - the label and default format are applied,
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index d9525bd..ec2d27f 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -63,7 +63,7 @@ data work.derived_check;
 
   if missing(libref) or missing(dsn) then reason='LIBREF and DSN are required';
   else if var_nm='' then reason='VAR_NM is required';
-  else if verify(var_nm,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
+  else if verify(strip(var_nm),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
     then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(var_nm,1,1),'0123456789')=0
     then reason='VAR_NM must not start with a digit';
@@ -83,7 +83,7 @@ data work.derived_check;
   else if index(var_fmt,';')>0 or indexc(var_fmt,'&%')>0
     or index(var_fmt,'/*')>0 or index(var_fmt,'*/')>0
     then reason='VAR_FMT must be a format name, without ; & % or //*';
-  else if verify(var_fmt,'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
+  else if verify(strip(var_fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
     then reason='VAR_FMT must be a format name';
   else if missing(var_logic)
     then reason='VAR_LOGIC is required';

From 813bce685691ad252043eeab31eb5460de81b756 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 16:51:08 +0000
Subject: [PATCH 16/48] feat(derived): declare VAR_DEPENDENCIES instead of
 VAR_ORDER

A sequence number has to be maintained by the administrator: inserting a
variable means renumbering everything after it, and a wrong number fails
silently - the derived value comes back missing, not wrong.

VAR_DEPENDENCIES names every input the expression uses, physical columns and
derived variables alike, eg 'REGION STATE'.  The statements are emitted in
dependency order, so a variable can build on another one with no sequence to
maintain, and a cycle is rejected.

The declaration is checked against the expression in both directions: a
derived variable referenced by VAR_LOGIC must be declared, and a declared
name that the expression does not reference is an error; a declared name must
resolve to a column of the target table or to a derived variable of the same
table.

Changes:
- MPE_DERIVED: VAR_ORDER (num) becomes VAR_DEPENDENCIES (varchar(4000)), in
  the ddl, mpe_makedatamodel and the v7.19 migration
- mpe_derived: dependency validation and a topological sort replace the
  VAR_ORDER sort and the forward-reference check
- mpe_derived_postedit: checks self reference, undeclared derived references
  and cycles at submit time, across the incoming rows plus the current rows
  they do not replace
- MPE_VALIDATIONS: the VAR_ORDER MINVAL/MAXVAL rules are replaced by UPCASE
  and a character pattern on VAR_DEPENDENCIES
- mocks: the dependency order is applied in JS, so a chained variable works
  even when the rows are loaded child first
- docs and tests updated
---
 sas/mocks/sasjs/services/admin/makedata.js    |  10 +-
 sas/mocks/sasjs/services/dcMockUtils.js       |  48 ++--
 sas/sasjs/db/datactrl/mpe_derived.ddl         |   2 +-
 .../db/migrations/20261019_v7.19_release.sas  |  15 +-
 sas/sasjs/macros/mpe_derived.sas              | 252 ++++++++++++++----
 sas/sasjs/macros/mpe_derived.test.sas         |   8 +-
 sas/sasjs/macros/mpe_makedata.sas             |  13 +-
 sas/sasjs/macros/mpe_makedatamodel.sas        |   2 +-
 sas/sasjs/services/editors/getdata.test.sas   |   4 +-
 .../services/hooks/mpe_derived_postedit.sas   | 190 ++++++++++---
 .../services/public/getcolvals.test.1.sas     |   2 +-
 .../services/public/validatefilter.test.sas   |   2 +-
 12 files changed, 410 insertions(+), 138 deletions(-)

diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index 3c548da..be1f683 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -292,7 +292,7 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       var_nm: 'REGION',
-      var_order: 10,
+      var_dependencies: 'STATE',
       var_type: 'C',
       var_len: 8,
       var_fmt: '',
@@ -305,7 +305,7 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       var_nm: 'LINE_VALUE',
-      var_order: 20,
+      var_dependencies: 'QTY PRICE',
       var_type: 'N',
       var_len: 8,
       var_fmt: 'dollar12.2',
@@ -313,13 +313,13 @@ const tables = {
       active: 1
     },
     {
-      // built on REGION, so it proves the order is applied
+      // built on REGION, so it proves the dependency order is applied
       tx_from: 0,
       tx_to: 253717919999,
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       var_nm: 'REGION_LABEL',
-      var_order: 30,
+      var_dependencies: 'REGION STATE',
       var_type: 'C',
       var_len: 14,
       var_fmt: '',
@@ -3313,7 +3313,7 @@ const schema = {
     { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
     { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
     { name: 'VAR_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
-    { name: 'VAR_ORDER', type: 'N', length: 8, format: '', label: '', notnull: false },
+    { name: 'VAR_DEPENDENCIES', type: 'C', length: 4000, format: '', label: '', notnull: false },
     { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
     { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index cbae51a..e1bc4a4 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1320,8 +1320,8 @@ function compileSasExpression(expr, columnNames) {
 
 /**
  * Reads MPE_DERIVED for a table, mirroring %mpe_derived.  Returns the active
- * variables with a compiled row function, ordered by VAR_ORDER (then
- * name), the same sequence the real macro emits.
+ * variables with a compiled row function, in dependency order (ties by name),
+ * the same sequence the real macro emits.
  *
  * @param {Object} opts
  * @param {string} opts.libref - target libref
@@ -1349,26 +1349,44 @@ function mpeDerived(opts) {
         )
         .map(r => {
             const logic = String(colVal(r, 'VAR_LOGIC') || '').trim()
+            const nm = String(colVal(r, 'VAR_NM') || '').toUpperCase()
             return {
-                name: String(colVal(r, 'VAR_NM') || '').toUpperCase(),
-                // statement order - a variable may reference one with a lower order
-                order: Number(colVal(r, 'VAR_ORDER')) || 0,
+                name: nm,
+                // every input the expression uses - columns and derived vars
+                deps: String(colVal(r, 'VAR_DEPENDENCIES') || '')
+                    .toUpperCase()
+                    .split(/\s+/)
+                    .filter(Boolean),
                 type: String(colVal(r, 'VAR_TYPE') || '').toUpperCase(),
                 length: Number(colVal(r, 'VAR_LEN')) || 8,
                 format: String(colVal(r, 'VAR_FMT') || ''),
                 logic,
-                label: 'DERIVED: ' + logic
+                label: nm + '=' + logic
             }
         })
-        .filter(d => {
-            if (!d.name || (d.type !== 'C' && d.type !== 'N') || !d.logic) return false
-            if (!(d.order >= 1)) {
-                console.log('derived variable ' + d.name + ' has no VAR_ORDER - skipped')
-                return false
-            }
-            return true
-        })
-        .sort((a, b) => a.order - b.order || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+        .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
+
+    // dependency order: repeatedly take the variables whose derived inputs are
+    // already placed, alphabetically within a round
+    const byName = new Map(vars.map(v => [v.name, v]))
+    const placed = []
+    const done = new Set()
+    while (placed.length < vars.length) {
+        const ready = vars
+            .filter(v => !done.has(v.name))
+            .filter(v => v.deps.every(d => !byName.has(d) || done.has(d)))
+            .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+        if (ready.length === 0) {
+            console.log('derived variables for ' + lib + '.' + dsn + ' contain a cycle')
+            return []
+        }
+        for (const v of ready) {
+            placed.push(v)
+            done.add(v.name)
+        }
+    }
+    vars.length = 0
+    vars.push.apply(vars, placed)
 
     // compile each against the physical columns plus the variables defined
     // earlier, so a derived variable can build on another one
diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
index d37ec1e..a0f983a 100644
--- a/sas/sasjs/db/datactrl/mpe_derived.ddl
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -20,7 +20,7 @@ create table &curlib..MPE_DERIVED
     LIBREF varchar(8),
     DSN varchar(32),
     VAR_NM varchar(32),
-    VAR_ORDER int,
+    VAR_DEPENDENCIES varchar(4000),
     VAR_TYPE varchar(1),
     VAR_LEN int,
     VAR_FMT varchar(32),
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index 37765f6..f2b5506 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -23,7 +23,7 @@ create table &dclib..mpe_derived
     libref varchar(8),
     dsn varchar(32),
     var_nm varchar(32),
-    var_order num,
+    var_dependencies varchar(4000),
     var_type varchar(1),
     var_len num,
     var_fmt varchar(32),
@@ -117,17 +117,16 @@ insert into &dclib..MPE_VALIDATIONS set
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='MAXVAL',rule_value='10000',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &dclib..MPE_VALIDATIONS set
+insert into &dclib..MPE_VALIDATIONS set
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 5dc3870..ade8be5 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -35,14 +35,17 @@
 
   Type and length are declared by the administrator, so the result is
   deterministic.  The label is the variable name, an equals sign and the
-  expression, so the grid shows both what the column is and where it comes
-  from.
+  expression.
 
-  Statements are generated in VAR_ORDER sequence, so a derived variable
-  may reference another derived variable with a lower order.  A reference to a
-  variable that is not defined earlier is rejected, because SAS would otherwise
-  treat it as an uninitialised variable and return a missing value without an
-  error.
+  VAR_DEPENDENCIES lists every input the expression uses - physical columns
+  and derived variables alike.  The statements are generated in dependency
+  order, so a derived variable may build on another one without a sequence
+  number being maintained.  A cycle is rejected.
+
+  The declaration is checked against the expression: a derived variable that
+  is referenced must be declared, a declared name that is not referenced is an
+  error, and a declared name must be a column of the target table or a derived
+  variable defined for it.
 
   Semicolons, macro triggers and comments are rejected, so a row can only ever
   be an expression.
@@ -74,7 +77,7 @@
   outview=0
 );
 
-%local physlist derived_err;
+%local physlist derived_err resv_list i w left nready;
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
@@ -91,9 +94,10 @@ filename &outref temp;
 data _null_;
   file &outref;
 run;
-data &outvars;
+data &outvars(keep=name);
   length name $32;
-  if 0 then name='';
+  name='';
+  if 0;
 run;
 
 %if %mf_existds(libds=&dclib..mpe_derived) ne 1 %then %do;
@@ -107,35 +111,32 @@ data work.derived_cfg;
     and active=1
     and upcase(libref)="%upcase(%scan(&libds,1,.))"
     and upcase(dsn)="%upcase(%scan(&libds,2,.))";
-  length name $32 logic $4000 fmt $32;
-  name=upcase(var_nm);
+  length name $32 logic $4000 fmt $32 deps $4000;
+  name=upcase(strip(var_nm));
   logic=strip(var_logic);
-  type=upcase(var_type);
+  type=upcase(strip(var_type));
   len=var_len;
   fmt=strip(upcase(var_fmt));
-  ord=var_order;
+  deps=upcase(strip(var_dependencies));
 run;
 
 %if %mf_nobs(work.derived_cfg)=0 %then %return;
 
-/* statement order is the configured order - a derived variable may reference
-  one with a lower VAR_ORDER, so they must be assigned in this sequence */
-proc sort data=work.derived_cfg out=work.derived_srt;
-  by ord name;
-run;
-
 proc sort data=work.derived_cfg out=work.derived_byname;
   by name;
 run;
 
-/**
-  * Reject anything that could break the generated data step, or that would
-  * produce a variable we cannot describe to the client
-  */
 %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
 
+/* names that may never be used - DC's own metadata columns */
+%let resv_list=%str("_____DELETE__THIS__RECORD_____","TX_FROM","TX_TO","PROCESSED_DTTM","VAR_TXFROM","VAR_TXTO","VAR_PROCESSED");
+
+/**
+  * Reject anything that could break the generated data step, or that would
+  * produce a variable we cannot describe to the caller
+  */
 data work.derived_bad;
-  set work.derived_srt;
+  set work.derived_byname;
   length reason $200;
   call missing(reason);
 
@@ -146,11 +147,8 @@ data work.derived_bad;
     then reason='VAR_NM must not start with a digit';
   else if name in (%unquote(&physlist))
     then reason='VAR_NM collides with a column in the target table';
-  else if name in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
-    ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
+  else if name in (%unquote(&resv_list))
     then reason='VAR_NM is reserved';
-  else if missing(ord) or ord ne int(ord) or ord<1
-    then reason='VAR_ORDER must be a positive integer';
   else if type not in ('C','N')
     then reason='VAR_TYPE must be C or N';
   else if missing(len) or len ne int(len) or len<1
@@ -187,37 +185,109 @@ run;
 proc append base=work.derived_bad data=work.derived_dup;
 run;
 
-/* a derived variable may only reference derived variables defined earlier -
-  otherwise the reference silently resolves to a missing value */
+/**
+  * The declared dependencies, one row per word
+  */
+data work.derived_dep;
+  set work.derived_byname;
+  length dep_nm $32;
+  do i=1 to countw(deps,' ');
+    dep_nm=scan(deps,i,' ');
+    if not missing(dep_nm) then output;
+  end;
+  keep name dep_nm;
+run;
+
+/**
+  * The identifiers the expression actually references - quoted strings are
+  * removed first, so a name inside a literal is not mistaken for a reference
+  */
+data work.derived_ref;
+  set work.derived_byname;
+  length l $4000 tok $32;
+  l=prxchange("s/'([^']|'')*'//",-1,logic);
+  l=prxchange('s/"([^"]|"")*"//',-1,l);
+  l=prxchange('s/[^A-Za-z0-9_]/ /',-1,upcase(l));
+  do i=1 to countw(l,' ');
+    tok=scan(l,i,' ');
+    if not missing(tok) then output;
+  end;
+  keep name tok;
+run;
+
+/* a declared name must resolve - to a column of the target, or to a variable
+  in this configuration */
 proc sql noprint;
-create table work.derived_orderbad_key as
-  select distinct a.name
-  from work.derived_srt a
-  inner join work.derived_srt b
-    on a.name ne b.name
-  where prxmatch(
-      cats('/(^|[^A-Z0-9_])',b.name,'([^A-Z0-9_]|$)/')
-      ,upcase(a.logic)
-    )>0
-    and b.ord>=a.ord
-  order by a.name;
+create table work.derived_bad_dep as
+  select d.name, d.dep_nm
+  from work.derived_dep d
+  where d.dep_nm not in (%unquote(&physlist))
+    and d.dep_nm not in (select name from work.derived_byname);
+
+/* every declared name must actually appear in the expression */
+create table work.derived_stale as
+  select d.name, d.dep_nm
+  from work.derived_dep d
+  where not exists (
+    select 1 from work.derived_ref r
+    where r.name=d.name and r.tok=d.dep_nm
+  );
+
+/* a derived variable referenced by the expression must be declared, otherwise
+  the statement order would be wrong and the value would be silently missing */
+create table work.derived_undeclared as
+  select distinct r.name, r.tok as dep_nm
+  from work.derived_ref r
+  where r.tok in (select name from work.derived_byname)
+    and r.tok ne r.name
+    and not exists (
+      select 1 from work.derived_dep d
+      where d.name=r.name and d.dep_nm=r.tok
+    );
+
+/* a variable may not depend on itself */
+create table work.derived_self as
+  select name, dep_nm from work.derived_dep
+  where name=dep_nm;
 quit;
 
-proc sort data=work.derived_orderbad_key;
-  by name;
-run;
+%if %mf_nobs(work.derived_bad_dep)>0 %then %do;
+  data _null_;
+    set work.derived_bad_dep(obs=1);
+    call symputx('derived_err'
+      ,cats(name,': VAR_DEPENDENCIES lists ',dep_nm
+        ,', which is not a column of the table or a derived variable'));
+  run;
+  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
+%end;
 
-data work.derived_orderbad;
-  merge work.derived_byname(in=in1) work.derived_orderbad_key(in=in2);
-  by name;
-  if in2;
-  length reason $200;
-  reason='VAR_LOGIC references a derived variable that is not defined'
-    !!' earlier - lower the VAR_ORDER of this one';
-run;
+%if %mf_nobs(work.derived_self)>0 %then %do;
+  data _null_;
+    set work.derived_self(obs=1);
+    call symputx('derived_err',cats(name,': VAR_DEPENDENCIES may not list itself'));
+  run;
+  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
+%end;
 
-proc append base=work.derived_bad data=work.derived_orderbad;
-run;
+%if %mf_nobs(work.derived_undeclared)>0 %then %do;
+  data _null_;
+    set work.derived_undeclared(obs=1);
+    call symputx('derived_err'
+      ,cats(name,': VAR_LOGIC references ',dep_nm
+        ,', which is not in VAR_DEPENDENCIES'));
+  run;
+  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
+%end;
+
+%if %mf_nobs(work.derived_stale)>0 %then %do;
+  data _null_;
+    set work.derived_stale(obs=1);
+    call symputx('derived_err'
+      ,cats(name,': VAR_DEPENDENCIES lists ',dep_nm
+        ,', which VAR_LOGIC does not reference'));
+  run;
+  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
+%end;
 
 %if %mf_nobs(work.derived_bad)>0 %then %do;
   data _null_;
@@ -234,6 +304,78 @@ run;
   )
 %end;
 
+/**
+  * Dependency order.  The edges are between derived variables only - a
+  * physical column is available from the start.  Repeatedly take the variables
+  * whose derived inputs have all been placed, alphabetically within a round;
+  * if a round places nothing, what is left is a cycle.
+  */
+proc sql;
+create table work.derived_edge as
+  select d.name as child, d.dep_nm as parent
+  from work.derived_dep d
+  where d.dep_nm in (select name from work.derived_byname);
+quit;
+
+data work.derived_todo(keep=name);
+  set work.derived_byname;
+run;
+
+data work.derived_placed(keep=name);
+  length name $32;
+  name='';
+  if 0;
+run;
+
+%let left=%mf_nobs(work.derived_todo);
+
+%do %while (&left>0);
+  proc sql;
+  create table work.derived_ready as
+    select t.name
+    from work.derived_todo t
+    where not exists (
+      select 1 from work.derived_edge e
+      inner join work.derived_todo t2 on e.parent=t2.name
+      where e.child=t.name
+    )
+    order by t.name;
+  quit;
+
+  %let nready=%mf_nobs(work.derived_ready);
+  %if &nready=0 %then %do;
+    data _null_;
+      length list $4000;
+      do until (eof);
+        set work.derived_todo end=eof;
+        list=catx(', ',list,name);
+      end;
+      call symputx('derived_err',list);
+    run;
+    %mp_abort(iftrue=1
+      ,mac=&sysmacroname
+      ,msg=%str(VAR_DEPENDENCIES contains a cycle - unresolved: &derived_err)
+    )
+  %end;
+
+  proc append base=work.derived_placed data=work.derived_ready force;
+  run;
+
+  proc sql;
+  delete from work.derived_todo
+    where name in (select name from work.derived_ready);
+  quit;
+
+  %let left=%mf_nobs(work.derived_todo);
+%end;
+
+data work.derived_srt;
+  merge work.derived_placed(in=in1) work.derived_byname(in=in2);
+  by name;
+  if in1 and in2;
+  seq=_n_;
+run;
+
 /**
   * Generate the attrib + assignment statements
   */
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 42548ab..70cebae 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -46,7 +46,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='BAND'
-    ,var_order=10
+    ,var_dependencies='AGE'
     ,var_type='C'
     ,var_len=8
     ,var_fmt=''
@@ -58,7 +58,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='BAND_AGE'
-    ,var_order=20
+    ,var_dependencies='BAND AGE'
     ,var_type='C'
     ,var_len=16
     ,var_fmt=''
@@ -70,7 +70,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='DOUBLE_AGE'
-    ,var_order=30
+    ,var_dependencies='AGE'
     ,var_type='N'
     ,var_len=8
     ,var_fmt=''
@@ -156,7 +156,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='FLAG_DT'
-    ,var_order=40
+    ,var_dependencies='AGE'
     ,var_type='N'
     ,var_len=8
     ,var_fmt='DATE9.'
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index ed09c4e..c9d673a 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -2008,17 +2008,16 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='NOTNULL',rule_value=' ',rule_active=1
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='MINVAL',rule_value='1',rule_active=1
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_ORDER"
-    ,rule_type='MAXVAL',rule_value='10000',rule_active=1
-    ,tx_to='31DEC5999:23:59:59'dt;
+insert into &lib..MPE_VALIDATIONS set
+insert into &lib..MPE_VALIDATIONS set
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index 15558fa..39e2a2c 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -700,7 +700,7 @@ create table &lib..MPE_DERIVED
     LIBREF varchar(8),
     DSN varchar(32),
     VAR_NM varchar(32),
-    VAR_ORDER num,
+    VAR_DEPENDENCIES varchar(4000),
     VAR_TYPE varchar(1),
     VAR_LEN num,
     VAR_FMT varchar(32),
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index 6698663..cdf32c5 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -258,7 +258,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE'
-    ,var_order=10
+    ,var_dependencies='STATE'
     ,var_type='C'
     ,var_len=41
     ,var_fmt=''
@@ -270,7 +270,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE2'
-    ,var_order=20
+    ,var_dependencies='REGION STATE'
     ,var_type='C'
     ,var_len=45
     ,var_fmt=''
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index ec2d27f..c33a170 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -15,11 +15,18 @@
   still watching - rather than breaking the EDIT screen later, when the view
   is built.
 
-  The target table itself is not assigned in this context, so the collision
-  between VAR_NM and a physical column is left to %mpe_derived, which runs
-  with the target library assigned.  Uniqueness on the business key
-  (LIBREF DSN VAR_NM) is already enforced by mpe_loader, so it is not
-  repeated here.
+  VAR_DEPENDENCIES must list every input the expression uses, physical columns
+  and derived variables alike.  The hook checks the part that does not need
+  the target library: a derived variable referenced by an expression must be
+  declared, a variable may not depend on itself, and the resulting graph must
+  be acyclic - across the incoming rows and the current rows they do not
+  replace.
+
+  The target table is not assigned in this context, so the collision between
+  VAR_NM and a physical column, and the resolution of a physical dependency,
+  are left to %mpe_derived, which runs with the target library assigned.
+  Uniqueness on the business key (LIBREF DSN VAR_NM) is already enforced by
+  mpe_loader, so it is not repeated here.
 
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
@@ -38,6 +45,7 @@
 
 %let errflag=0;
 %let errmsg=;
+%local left nready;
 
 /* normalise the incoming values */
 data work.staging_ds;
@@ -48,6 +56,7 @@ data work.staging_ds;
     var_nm=upcase(strip(var_nm));
     var_type=upcase(strip(var_type));
     var_fmt=upcase(strip(var_fmt));
+    var_dependencies=upcase(strip(var_dependencies));
     var_logic=strip(var_logic);
   end;
 run;
@@ -70,8 +79,6 @@ data work.derived_check;
   else if var_nm in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
     ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
     then reason='VAR_NM is reserved';
-  else if missing(var_order) or var_order ne int(var_order) or var_order<1
-    then reason='VAR_ORDER must be a positive integer';
   else if var_type not in ('C','N')
     then reason='VAR_TYPE must be C or N';
   else if missing(var_len) or var_len ne int(var_len) or var_len<1
@@ -111,42 +118,149 @@ run;
 )
 
 /**
-  * Both references must resolve in the resulting configuration, and only to a
-  * variable with a lower VAR_ORDER.  The resulting set is the staged rows plus
-  * the current rows the submission does not replace.
+  * The graph has to hold for the resulting configuration: the incoming rows
+  * plus the current rows of the same tables that this submission does not
+  * replace.
   */
-proc sql noprint;
-create table work.derived_combined as
-  select libref, dsn, var_nm, var_order
+proc sql;
+create table work.derived_rows as
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_dependencies)) as deps
+    ,strip(var_logic) as logic
   from work.staging_ds
   where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
   union all
-  select libref, dsn, var_nm, var_order
-  from &DC_LIBREF..MPE_DERIVED
+  select libref, dsn, upcase(strip(var_nm)) as name
+    ,upcase(strip(var_dependencies)) as deps
+    ,strip(var_logic) as logic
+  from &DC_LIBREF..mpe_derived
   where &dc_dttmtfmt. lt tx_to
     and catx('.',libref,dsn,var_nm) not in
-      (select catx('.',libref,dsn,var_nm) from work.staging_ds);
-
-create table work.derived_orderbad as
-  select distinct a.libref, a.dsn, a.var_nm
-  from work.staging_ds a
-  inner join work.derived_combined b
-    on a.libref=b.libref
-    and a.dsn=b.dsn
-  where upcase(a._____DELETE__THIS__RECORD_____) ne 'YES'
-    and a.var_nm ne b.var_nm
-    and prxmatch(
-      cats('/(^|[^A-Z0-9_])',b.var_nm,'([^A-Z0-9_]|$)/')
-      ,upcase(a.var_logic)
-    )>0
-    and b.var_order>=a.var_order;
-
-%let errmsg=;
-select catx('.',libref,dsn,var_nm) into: errmsg
-  from work.derived_orderbad(obs=1);
+      (select catx('.',libref,dsn,var_nm) from work.staging_ds)
+    and catx('.',libref,dsn) in
+      (select catx('.',libref,dsn) from work.staging_ds);
 quit;
 
-%mp_abort(iftrue= (%mf_nobs(work.derived_orderbad)>0)
-  ,mac=mpe_derived_postedit
-  ,msg=%str(VAR_LOGIC references a derived variable that is not defined earlier - lower the VAR_ORDER of &errmsg)
-)
+data work.derived_dep;
+  set work.derived_rows;
+  length dep_nm $32;
+  do i=1 to countw(deps,' ');
+    dep_nm=scan(deps,i,' ');
+    if not missing(dep_nm) then output;
+  end;
+  keep libref dsn name dep_nm;
+run;
+
+data work.derived_ref;
+  set work.derived_rows;
+  length l $4000 tok $32;
+  l=prxchange("s/'([^']|'')*'//",-1,logic);
+  l=prxchange('s/"([^"]|"")*"//',-1,l);
+  l=prxchange('s/[^A-Za-z0-9_]/ /',-1,upcase(l));
+  do i=1 to countw(l,' ');
+    tok=scan(l,i,' ');
+    if not missing(tok) then output;
+  end;
+  keep libref dsn name tok;
+run;
+
+proc sql;
+create table work.derived_self as
+  select libref, dsn, name, dep_nm
+  from work.derived_dep
+  where name=dep_nm;
+
+create table work.derived_undeclared as
+  select r.libref, r.dsn, r.name, r.tok as dep_nm
+  from work.derived_ref r
+  where exists (
+      select 1 from work.derived_rows n
+      where n.libref=r.libref and n.dsn=r.dsn and n.name=r.tok
+    )
+    and r.tok ne r.name
+    and not exists (
+      select 1 from work.derived_dep d
+      where d.libref=r.libref and d.dsn=r.dsn and d.name=r.name
+        and d.dep_nm=r.tok
+    );
+quit;
+
+%if %mf_nobs(work.derived_self)>0 %then %do;
+  data _null_;
+    set work.derived_self(obs=1);
+    call symputx('errmsg'
+      ,cats('MPE_DERIVED: ',name,': VAR_DEPENDENCIES may not list itself'));
+  run;
+  %mp_abort(iftrue=1
+    ,mac=mpe_derived_postedit
+    ,msg=%superq(errmsg)
+  )
+%end;
+
+%if %mf_nobs(work.derived_undeclared)>0 %then %do;
+  data _null_;
+    set work.derived_undeclared(obs=1);
+    call symputx('errmsg'
+      ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
+        ,', which is not in VAR_DEPENDENCIES'));
+  run;
+  %mp_abort(iftrue=1
+    ,mac=mpe_derived_postedit
+    ,msg=%superq(errmsg)
+  )
+%end;
+
+/**
+  * Cycle check - repeatedly take the variables whose derived inputs have all
+  * been placed; a round that places nothing leaves a cycle
+  */
+proc sql;
+create table work.derived_edge as
+  select d.libref, d.dsn, d.name as child, d.dep_nm as parent
+  from work.derived_dep d
+  where d.dep_nm in (select name from work.derived_rows);
+quit;
+
+data work.derived_todo;
+  set work.derived_rows(keep=libref dsn name);
+run;
+
+%let left=%mf_nobs(work.derived_todo);
+
+%do %while (&left>0);
+  proc sql;
+  create table work.derived_ready as
+    select t.libref, t.dsn, t.name
+    from work.derived_todo t
+    where not exists (
+      select 1 from work.derived_edge e
+      inner join work.derived_todo t2
+        on e.libref=t2.libref and e.dsn=t2.dsn and e.parent=t2.name
+      where e.libref=t.libref and e.dsn=t.dsn and e.child=t.name
+    );
+  quit;
+
+  %let nready=%mf_nobs(work.derived_ready);
+  %if &nready=0 %then %do;
+    data _null_;
+      length list $4000;
+      do until (eof);
+        set work.derived_todo end=eof;
+        list=catx(', ',list,catx('.',libref,dsn,name));
+      end;
+      call symputx('errmsg',cats('VAR_DEPENDENCIES contains a cycle - unresolved: ',list));
+    run;
+    %mp_abort(iftrue=1
+      ,mac=mpe_derived_postedit
+      ,msg=%superq(errmsg)
+    )
+  %end;
+
+  proc sql;
+  delete from work.derived_todo
+    where catx('.',libref,dsn,name) in
+      (select catx('.',libref,dsn,name) from work.derived_ready);
+  quit;
+
+  %let left=%mf_nobs(work.derived_todo);
+%end;
diff --git a/sas/sasjs/services/public/getcolvals.test.1.sas b/sas/sasjs/services/public/getcolvals.test.1.sas
index b324406..25ba3cf 100644
--- a/sas/sasjs/services/public/getcolvals.test.1.sas
+++ b/sas/sasjs/services/public/getcolvals.test.1.sas
@@ -52,7 +52,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
     ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
     ,var_nm='DERIVED_KEY'
-    ,var_order=10
+    ,var_dependencies='STATE'
     ,var_type='C'
     ,var_len=20
     ,var_fmt=''
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index cefe4e8..80c02c4 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -92,7 +92,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE'
-    ,var_order=10
+    ,var_dependencies='STATE'
     ,var_type='C'
     ,var_len=41
     ,var_fmt=''

From 5f6569b9e5ccc6ad6574409ea43532afeced9d08 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 17:06:59 +0000
Subject: [PATCH 17/48] test(derived): declare the real dependencies in the SAS
 tests

The test fixtures now state the inputs their expressions actually use:
LIBREF DSN, DERIVED_NOTE, PRIMARY_KEY_FIELD, and none for a literal-only
expression.  The getdata label assertion follows the new <VAR_NM>=<expr>
label.
---
 sas/sasjs/macros/mpe_derived.test.sas             | 2 +-
 sas/sasjs/services/editors/getdata.test.sas       | 6 +++---
 sas/sasjs/services/public/getcolvals.test.1.sas   | 2 +-
 sas/sasjs/services/public/validatefilter.test.sas | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 70cebae..9407070 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -156,7 +156,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='FLAG_DT'
-    ,var_dependencies='AGE'
+    ,var_dependencies=''
     ,var_type='N'
     ,var_len=8
     ,var_fmt='DATE9.'
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index cdf32c5..bd5d7fe 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -258,7 +258,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE'
-    ,var_dependencies='STATE'
+    ,var_dependencies='LIBREF DSN'
     ,var_type='C'
     ,var_len=41
     ,var_fmt=''
@@ -270,7 +270,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE2'
-    ,var_dependencies='REGION STATE'
+    ,var_dependencies='DERIVED_NOTE'
     ,var_type='C'
     ,var_len=45
     ,var_fmt=''
@@ -350,7 +350,7 @@ run;
 
 data work.chk4b;
   set work.cols4;
-  where name='DERIVED_NOTE' and label=:'DERIVED:';
+  where name='DERIVED_NOTE' and label=:'DERIVED_NOTE=';
 run;
 %mp_assertdsobs(work.chk4b,
   desc=Test4 - derived column is labelled,
diff --git a/sas/sasjs/services/public/getcolvals.test.1.sas b/sas/sasjs/services/public/getcolvals.test.1.sas
index 25ba3cf..353ee45 100644
--- a/sas/sasjs/services/public/getcolvals.test.1.sas
+++ b/sas/sasjs/services/public/getcolvals.test.1.sas
@@ -52,7 +52,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
     ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
     ,var_nm='DERIVED_KEY'
-    ,var_dependencies='STATE'
+    ,var_dependencies='PRIMARY_KEY_FIELD'
     ,var_type='C'
     ,var_len=20
     ,var_fmt=''
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index 80c02c4..2a1230e 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -92,7 +92,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE'
-    ,var_dependencies='STATE'
+    ,var_dependencies='LIBREF DSN'
     ,var_type='C'
     ,var_len=41
     ,var_fmt=''

From cee1a954ee97e17f297b7cf43ef0e5cfc6e393a6 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 17:52:04 +0000
Subject: [PATCH 18/48] feat(derived): report the dependency cycle path

The cycle check was already in the post edit hook and in mpe_derived - the
topological sort places a variable only once its derived inputs are placed,
and if a round places nothing the remainder is a cycle.  What it reported was
just the set of names that could not be placed, which does not tell the
administrator what to change.

The message now walks the edges from the first unresolved variable and names
the loop, eg:

  VAR_DEPENDENCIES contains a cycle: A -> B -> C -> A

Names are collected in a dotted seen list, so a lead-in to the loop is shown
as well and the walk stops at the first repeat.
---
 sas/sasjs/macros/mpe_derived.sas              | 34 +++++++++++++----
 .../services/hooks/mpe_derived_postedit.sas   | 37 +++++++++++++++----
 2 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index ade8be5..a02c728 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -77,7 +77,8 @@
   outview=0
 );
 
-%local physlist derived_err resv_list i w left nready;
+%local physlist derived_err resv_list i w left nready
+  cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i;
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
@@ -345,16 +346,33 @@ run;
   %let nready=%mf_nobs(work.derived_ready);
   %if &nready=0 %then %do;
     data _null_;
-      length list $4000;
-      do until (eof);
-        set work.derived_todo end=eof;
-        list=catx(', ',list,name);
-      end;
-      call symputx('derived_err',list);
+      set work.derived_todo(obs=1);
+      call symputx('cyc_start',name);
     run;
+    %let cyc_path=&cyc_start;
+    %let cyc_cur=&cyc_start;
+    %let cyc_seen=%str(.&cyc_start..);
+    %do cyc_i=1 %to &left;
+      %let cyc_next=;
+      proc sql noprint;
+        select e.parent into: cyc_next
+        from work.derived_edge e
+        where e.child="&cyc_cur"
+          and e.parent in (select name from work.derived_todo);
+      quit;
+      %if %length(&cyc_next)=0 %then %let cyc_i=&left;
+      %else %do;
+        %let cyc_path=&cyc_path. -> &cyc_next;
+        %if %index(&cyc_seen,%str(.&cyc_next..))>0 %then %let cyc_i=&left;
+        %else %do;
+          %let cyc_seen=&cyc_seen.&cyc_next..;
+          %let cyc_cur=&cyc_next;
+        %end;
+      %end;
+    %end;
     %mp_abort(iftrue=1
       ,mac=&sysmacroname
-      ,msg=%str(VAR_DEPENDENCIES contains a cycle - unresolved: &derived_err)
+      ,msg=%str(VAR_DEPENDENCIES contains a cycle: &cyc_path)
     )
   %end;
 
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index c33a170..6dcfb35 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -45,7 +45,7 @@
 
 %let errflag=0;
 %let errmsg=;
-%local left nready;
+%local left nready cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i cyc_n;
 
 /* normalise the incoming values */
 data work.staging_ds;
@@ -242,17 +242,38 @@ run;
 
   %let nready=%mf_nobs(work.derived_ready);
   %if &nready=0 %then %do;
+    /* walk the edges, so the message names the cycle rather than just the
+      names that could not be placed */
+    %let cyc_n=%mf_nobs(work.derived_todo);
     data _null_;
-      length list $4000;
-      do until (eof);
-        set work.derived_todo end=eof;
-        list=catx(', ',list,catx('.',libref,dsn,name));
-      end;
-      call symputx('errmsg',cats('VAR_DEPENDENCIES contains a cycle - unresolved: ',list));
+      set work.derived_todo(obs=1);
+      call symputx('cyc_start',name);
     run;
+    %let cyc_path=&cyc_start;
+    %let cyc_cur=&cyc_start;
+    %let cyc_seen=%str(.&cyc_start..);
+    %do cyc_i=1 %to &cyc_n;
+      %let cyc_next=;
+      proc sql noprint;
+        select e.parent into: cyc_next
+        from work.derived_edge e
+        where e.child="&cyc_cur"
+          and catx('.',e.libref,e.dsn,e.parent) in
+            (select catx('.',libref,dsn,name) from work.derived_todo);
+      quit;
+      %if %length(&cyc_next)=0 %then %let cyc_i=&cyc_n;
+      %else %do;
+        %let cyc_path=&cyc_path. -> &cyc_next;
+        %if %index(&cyc_seen,%str(.&cyc_next..))>0 %then %let cyc_i=&cyc_n;
+        %else %do;
+          %let cyc_seen=&cyc_seen.&cyc_next..;
+          %let cyc_cur=&cyc_next;
+        %end;
+      %end;
+    %end;
     %mp_abort(iftrue=1
       ,mac=mpe_derived_postedit
-      ,msg=%superq(errmsg)
+      ,msg=%str(VAR_DEPENDENCIES contains a cycle: &cyc_path)
     )
   %end;
 

From ee5d7c678e5d98cbe8029c458f9b4ea3c6cc2784 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 18:33:24 +0000
Subject: [PATCH 19/48] refactor(derived): VAR_DEPS instead of VAR_DEPENDENCIES

Shorter column name, same meaning: every input the expression uses,
physical columns and derived variables alike.
---
 sas/mocks/sasjs/services/admin/makedata.js         |  8 ++++----
 sas/mocks/sasjs/services/dcMockUtils.js            |  2 +-
 sas/sasjs/db/datactrl/mpe_derived.ddl              |  2 +-
 sas/sasjs/db/migrations/20261019_v7.19_release.sas |  6 +++---
 sas/sasjs/macros/mpe_derived.sas                   | 14 +++++++-------
 sas/sasjs/macros/mpe_derived.test.sas              |  8 ++++----
 sas/sasjs/macros/mpe_makedata.sas                  |  4 ++--
 sas/sasjs/macros/mpe_makedatamodel.sas             |  2 +-
 sas/sasjs/services/editors/getdata.test.sas        |  4 ++--
 sas/sasjs/services/hooks/mpe_derived_postedit.sas  | 14 +++++++-------
 sas/sasjs/services/public/getcolvals.test.1.sas    |  2 +-
 sas/sasjs/services/public/validatefilter.test.sas  |  2 +-
 12 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index be1f683..95c79d9 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -292,7 +292,7 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       var_nm: 'REGION',
-      var_dependencies: 'STATE',
+      var_deps: 'STATE',
       var_type: 'C',
       var_len: 8,
       var_fmt: '',
@@ -305,7 +305,7 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       var_nm: 'LINE_VALUE',
-      var_dependencies: 'QTY PRICE',
+      var_deps: 'QTY PRICE',
       var_type: 'N',
       var_len: 8,
       var_fmt: 'dollar12.2',
@@ -319,7 +319,7 @@ const tables = {
       libref: 'TESTDATA',
       dsn: 'DERIVED_DEMO',
       var_nm: 'REGION_LABEL',
-      var_dependencies: 'REGION STATE',
+      var_deps: 'REGION STATE',
       var_type: 'C',
       var_len: 14,
       var_fmt: '',
@@ -3313,7 +3313,7 @@ const schema = {
     { name: 'LIBREF', type: 'C', length: 8, format: '', label: '', notnull: false },
     { name: 'DSN', type: 'C', length: 32, format: '', label: '', notnull: false },
     { name: 'VAR_NM', type: 'C', length: 32, format: '', label: '', notnull: false },
-    { name: 'VAR_DEPENDENCIES', type: 'C', length: 4000, format: '', label: '', notnull: false },
+    { name: 'VAR_DEPS', type: 'C', length: 4000, format: '', label: '', notnull: false },
     { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
     { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index e1bc4a4..9054e36 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1353,7 +1353,7 @@ function mpeDerived(opts) {
             return {
                 name: nm,
                 // every input the expression uses - columns and derived vars
-                deps: String(colVal(r, 'VAR_DEPENDENCIES') || '')
+                deps: String(colVal(r, 'VAR_DEPS') || '')
                     .toUpperCase()
                     .split(/\s+/)
                     .filter(Boolean),
diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
index a0f983a..8c3ff53 100644
--- a/sas/sasjs/db/datactrl/mpe_derived.ddl
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -20,7 +20,7 @@ create table &curlib..MPE_DERIVED
     LIBREF varchar(8),
     DSN varchar(32),
     VAR_NM varchar(32),
-    VAR_DEPENDENCIES varchar(4000),
+    VAR_DEPS varchar(4000),
     VAR_TYPE varchar(1),
     VAR_LEN int,
     VAR_FMT varchar(32),
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index f2b5506..78517fa 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -23,7 +23,7 @@ create table &dclib..mpe_derived
     libref varchar(8),
     dsn varchar(32),
     var_nm varchar(32),
-    var_dependencies varchar(4000),
+    var_deps varchar(4000),
     var_type varchar(1),
     var_len num,
     var_fmt varchar(32),
@@ -117,11 +117,11 @@ insert into &dclib..MPE_VALIDATIONS set
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index a02c728..3bfddb8 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -37,7 +37,7 @@
   deterministic.  The label is the variable name, an equals sign and the
   expression.
 
-  VAR_DEPENDENCIES lists every input the expression uses - physical columns
+  VAR_DEPS lists every input the expression uses - physical columns
   and derived variables alike.  The statements are generated in dependency
   order, so a derived variable may build on another one without a sequence
   number being maintained.  A cycle is rejected.
@@ -118,7 +118,7 @@ data work.derived_cfg;
   type=upcase(strip(var_type));
   len=var_len;
   fmt=strip(upcase(var_fmt));
-  deps=upcase(strip(var_dependencies));
+  deps=upcase(strip(var_deps));
 run;
 
 %if %mf_nobs(work.derived_cfg)=0 %then %return;
@@ -256,7 +256,7 @@ quit;
   data _null_;
     set work.derived_bad_dep(obs=1);
     call symputx('derived_err'
-      ,cats(name,': VAR_DEPENDENCIES lists ',dep_nm
+      ,cats(name,': VAR_DEPS lists ',dep_nm
         ,', which is not a column of the table or a derived variable'));
   run;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
@@ -265,7 +265,7 @@ quit;
 %if %mf_nobs(work.derived_self)>0 %then %do;
   data _null_;
     set work.derived_self(obs=1);
-    call symputx('derived_err',cats(name,': VAR_DEPENDENCIES may not list itself'));
+    call symputx('derived_err',cats(name,': VAR_DEPS may not list itself'));
   run;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
@@ -275,7 +275,7 @@ quit;
     set work.derived_undeclared(obs=1);
     call symputx('derived_err'
       ,cats(name,': VAR_LOGIC references ',dep_nm
-        ,', which is not in VAR_DEPENDENCIES'));
+        ,', which is not in VAR_DEPS'));
   run;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
@@ -284,7 +284,7 @@ quit;
   data _null_;
     set work.derived_stale(obs=1);
     call symputx('derived_err'
-      ,cats(name,': VAR_DEPENDENCIES lists ',dep_nm
+      ,cats(name,': VAR_DEPS lists ',dep_nm
         ,', which VAR_LOGIC does not reference'));
   run;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
@@ -372,7 +372,7 @@ run;
     %end;
     %mp_abort(iftrue=1
       ,mac=&sysmacroname
-      ,msg=%str(VAR_DEPENDENCIES contains a cycle: &cyc_path)
+      ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
     )
   %end;
 
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 9407070..2130844 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -46,7 +46,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='BAND'
-    ,var_dependencies='AGE'
+    ,var_deps='AGE'
     ,var_type='C'
     ,var_len=8
     ,var_fmt=''
@@ -58,7 +58,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='BAND_AGE'
-    ,var_dependencies='BAND AGE'
+    ,var_deps='BAND AGE'
     ,var_type='C'
     ,var_len=16
     ,var_fmt=''
@@ -70,7 +70,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='DOUBLE_AGE'
-    ,var_dependencies='AGE'
+    ,var_deps='AGE'
     ,var_type='N'
     ,var_len=8
     ,var_fmt=''
@@ -156,7 +156,7 @@ insert into &dclib..mpe_derived set
     ,libref='WORK'
     ,dsn='CLASS'
     ,var_nm='FLAG_DT'
-    ,var_dependencies=''
+    ,var_deps=''
     ,var_type='N'
     ,var_len=8
     ,var_fmt='DATE9.'
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index c9d673a..17279f0 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -2008,11 +2008,11 @@ insert into &lib..MPE_VALIDATIONS set
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z_][A-Za-z0-9_]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='CASE',rule_value='UPCASE',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
-    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPENDENCIES"
+    tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index 39e2a2c..a8ba7ea 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -700,7 +700,7 @@ create table &lib..MPE_DERIVED
     LIBREF varchar(8),
     DSN varchar(32),
     VAR_NM varchar(32),
-    VAR_DEPENDENCIES varchar(4000),
+    VAR_DEPS varchar(4000),
     VAR_TYPE varchar(1),
     VAR_LEN num,
     VAR_FMT varchar(32),
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index bd5d7fe..b0a0519 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -258,7 +258,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE'
-    ,var_dependencies='LIBREF DSN'
+    ,var_deps='LIBREF DSN'
     ,var_type='C'
     ,var_len=41
     ,var_fmt=''
@@ -270,7 +270,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE2'
-    ,var_dependencies='DERIVED_NOTE'
+    ,var_deps='DERIVED_NOTE'
     ,var_type='C'
     ,var_len=45
     ,var_fmt=''
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 6dcfb35..6dfb632 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -15,7 +15,7 @@
   still watching - rather than breaking the EDIT screen later, when the view
   is built.
 
-  VAR_DEPENDENCIES must list every input the expression uses, physical columns
+  VAR_DEPS must list every input the expression uses, physical columns
   and derived variables alike.  The hook checks the part that does not need
   the target library: a derived variable referenced by an expression must be
   declared, a variable may not depend on itself, and the resulting graph must
@@ -56,7 +56,7 @@ data work.staging_ds;
     var_nm=upcase(strip(var_nm));
     var_type=upcase(strip(var_type));
     var_fmt=upcase(strip(var_fmt));
-    var_dependencies=upcase(strip(var_dependencies));
+    var_deps=upcase(strip(var_deps));
     var_logic=strip(var_logic);
   end;
 run;
@@ -125,13 +125,13 @@ run;
 proc sql;
 create table work.derived_rows as
   select libref, dsn, upcase(strip(var_nm)) as name
-    ,upcase(strip(var_dependencies)) as deps
+    ,upcase(strip(var_deps)) as deps
     ,strip(var_logic) as logic
   from work.staging_ds
   where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
   union all
   select libref, dsn, upcase(strip(var_nm)) as name
-    ,upcase(strip(var_dependencies)) as deps
+    ,upcase(strip(var_deps)) as deps
     ,strip(var_logic) as logic
   from &DC_LIBREF..mpe_derived
   where &dc_dttmtfmt. lt tx_to
@@ -189,7 +189,7 @@ quit;
   data _null_;
     set work.derived_self(obs=1);
     call symputx('errmsg'
-      ,cats('MPE_DERIVED: ',name,': VAR_DEPENDENCIES may not list itself'));
+      ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
   run;
   %mp_abort(iftrue=1
     ,mac=mpe_derived_postedit
@@ -202,7 +202,7 @@ quit;
     set work.derived_undeclared(obs=1);
     call symputx('errmsg'
       ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
-        ,', which is not in VAR_DEPENDENCIES'));
+        ,', which is not in VAR_DEPS'));
   run;
   %mp_abort(iftrue=1
     ,mac=mpe_derived_postedit
@@ -273,7 +273,7 @@ run;
     %end;
     %mp_abort(iftrue=1
       ,mac=mpe_derived_postedit
-      ,msg=%str(VAR_DEPENDENCIES contains a cycle: &cyc_path)
+      ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
     )
   %end;
 
diff --git a/sas/sasjs/services/public/getcolvals.test.1.sas b/sas/sasjs/services/public/getcolvals.test.1.sas
index 353ee45..355403e 100644
--- a/sas/sasjs/services/public/getcolvals.test.1.sas
+++ b/sas/sasjs/services/public/getcolvals.test.1.sas
@@ -52,7 +52,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_X_TEST,1,.)"
     ,dsn="%scan(&dclib..MPE_X_TEST,2,.)"
     ,var_nm='DERIVED_KEY'
-    ,var_dependencies='PRIMARY_KEY_FIELD'
+    ,var_deps='PRIMARY_KEY_FIELD'
     ,var_type='C'
     ,var_len=20
     ,var_fmt=''
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index 2a1230e..f2e9558 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -92,7 +92,7 @@ insert into &dc_libref..mpe_derived set
     ,libref="%scan(&dclib..MPE_TABLES,1,.)"
     ,dsn="%scan(&dclib..MPE_TABLES,2,.)"
     ,var_nm='DERIVED_NOTE'
-    ,var_dependencies='LIBREF DSN'
+    ,var_deps='LIBREF DSN'
     ,var_type='C'
     ,var_len=41
     ,var_fmt=''

From be50cb5d082e788c6ae106bc1d02e73a8c7b9646 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 21:24:03 +0000
Subject: [PATCH 20/48] fix(derived): carry the placement sequence in
 mpe_derived

Same defect as the core review found in mp_derivedvars: the final assemble
merged the placement table by name, but that table is built in dependency
rounds and is not name ordered, so the BY-merge failed whenever the rounds
were not alphabetical, and seq was a BY-group counter rather than the
dependency order.

Each round now stamps seq=_n_+nplaced, the merge runs on the name-sorted
tables, and the result is sorted by seq.  Also guards the physical-column
in-list for a zero-column table, and checks libds exists before running.
---
 sas/sasjs/macros/mpe_derived.sas | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 3bfddb8..28cf0d3 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -77,7 +77,7 @@
   outview=0
 );
 
-%local physlist derived_err resv_list i w left nready
+%local physlist derived_err resv_list i w left nready nplaced
   cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i;
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
@@ -88,6 +88,10 @@
   ,mac=&sysmacroname
   ,msg=%str(Please provide an outref for the generated statements)
 )
+%mp_abort(iftrue= (%mf_existds(libds=&libds) ne 1)
+  ,mac=&sysmacroname
+  ,msg=%str(Target table not found: &libds)
+)
 
 filename &outref temp;
 
@@ -146,8 +150,10 @@ data work.derived_bad;
     then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(name,1,1),'0123456789')=0
     then reason='VAR_NM must not start with a digit';
+  %if %length(&physlist)>0 %then %do;
   else if name in (%unquote(&physlist))
     then reason='VAR_NM collides with a column in the target table';
+  %end;
   else if name in (%unquote(&resv_list))
     then reason='VAR_NM is reserved';
   else if type not in ('C','N')
@@ -322,13 +328,15 @@ data work.derived_todo(keep=name);
   set work.derived_byname;
 run;
 
-data work.derived_placed(keep=name);
+data work.derived_placed(keep=name seq);
   length name $32;
   name='';
+  seq=0;
   if 0;
 run;
 
 %let left=%mf_nobs(work.derived_todo);
+%let nplaced=0;
 
 %do %while (&left>0);
   proc sql;
@@ -376,8 +384,15 @@ run;
     )
   %end;
 
+  /* seq carries the dependency order - the placement rounds are not name
+    ordered, so the final merge must not rely on by name alone */
+  data work.derived_ready;
+    set work.derived_ready;
+    seq=_n_+&nplaced;
+  run;
   proc append base=work.derived_placed data=work.derived_ready force;
   run;
+  %let nplaced=%eval(&nplaced+&nready);
 
   proc sql;
   delete from work.derived_todo
@@ -387,11 +402,18 @@ run;
   %let left=%mf_nobs(work.derived_todo);
 %end;
 
+proc sort data=work.derived_placed;
+  by name;
+run;
+
 data work.derived_srt;
   merge work.derived_placed(in=in1) work.derived_byname(in=in2);
   by name;
   if in1 and in2;
-  seq=_n_;
+run;
+
+proc sort data=work.derived_srt;
+  by seq;
 run;
 
 /**

From d2de3ce88abe58328197fb2f52918c1a5bea773c Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 21:26:30 +0000
Subject: [PATCH 21/48] test(derived): pin the macro's scope in
 mpe_derived.test.sas

---
 sas/sasjs/macros/mpe_derived.test.sas | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 2130844..f7d7141 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -3,7 +3,7 @@
   @brief testing the mpe_derived macro
 
   @details Covers the generated statements, the view, the metadata the callers
-  consume, and the ordering rules.
+  consume, and the dependency rules.
 
   The configuration checks abort the run when they fail, so they are not
   exercised here - this suite covers the behaviour that callers depend on.
@@ -13,6 +13,7 @@
   @li mp_assert.sas
   @li mp_assertcols.sas
   @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
   @li mpe_derived.sas
 
   @author 4GL Apps Ltd
@@ -21,6 +22,8 @@
     4GL Apps Ltd.
 
 **/
+%mp_assertscope(SNAPSHOT)
+
 
 /* a source table, and a table with no derived variables */
 data work.class;
@@ -234,3 +237,5 @@ run;
 proc sql;
 delete from &dclib..mpe_derived where upcase(libref)='WORK';
 quit;
+
+%mp_assertscope(COMPARE, outds=work.test_results)

From 1055a0bc4a2a3ce5b3aa5863948c3b909f5e7d1f Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 21:34:21 +0000
Subject: [PATCH 22/48] chore(deps): follow-redirects 1.16.1

Two high advisories were published against 1.16.0 in the last few hours
(GHSA-q3jh-wrc6-jgq8, GHSA-8r9p-f939-6h3c), and the 'Check audit' step fails
on any vulnerability, so every run after 18:33 UTC today is red - including
the ones on this branch that had passed earlier.

Lockfile only, no source change: 1.16.1 satisfies the ^1.15.6 range axios
asks for.  Worth cherry-picking to main, since it blocks every PR.
---
 client/package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/package-lock.json b/client/package-lock.json
index a9fb988..e677d39 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -15420,9 +15420,9 @@
       "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
-      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
+      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
       "funding": [
         {
           "type": "individual",

From 12beeff69518376a042fbbc58332acc40d89d132 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 22:25:16 +0000
Subject: [PATCH 23/48] feat(derived): allow a variable to shadow a physical
 column

SUMMER = SUMMER + 1 is a legitimate definition: the name matches an existing
column, the expression reads the physical value, and the view carries the
derived one under the same name.

- mpe_derived now takes the physical column list from mp_getcols, so it has
  type and length in one pass; a shadowed name must match the column's type,
  and its length may not exceed it (numeric lengths must be equal), as attrib
  cannot change either on an existing variable
- a self reference is only an error when the name is not a physical column
- self references are not graph edges, so the dependency sort is unaffected
- the post edit hook no longer rejects a self reference, since it cannot see
  the target library; it still rejects cycles between distinct names
- mpe_derived now uses mf_getuniquename for its working tables, matching the
  core macro, and carries the same placement sequence fix
- test 4 covers shadowing, docs cover the rule
---
 sas/sasjs/macros/mpe_derived.sas              | 210 +++++++++++-------
 sas/sasjs/macros/mpe_derived.test.sas         |  43 ++++
 .../services/hooks/mpe_derived_postedit.sas   |  28 +--
 3 files changed, 184 insertions(+), 97 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 28cf0d3..79b3201 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -34,13 +34,16 @@
       REGION = <expression>;
 
   Type and length are declared by the administrator, so the result is
-  deterministic.  The label is the variable name, an equals sign and the
-  expression.
+  deterministic, and a derived variable may shadow a physical column - the
+  expression then reads the physical value, eg SUMMER = SUMMER + 1 - in which
+  case VAR_TYPE and VAR_LEN must match the column, as attrib cannot change
+  either on an existing variable.  The label is the variable name, an equals
+  sign and the expression.
 
-  VAR_DEPS lists every input the expression uses - physical columns
-  and derived variables alike.  The statements are generated in dependency
-  order, so a derived variable may build on another one without a sequence
-  number being maintained.  A cycle is rejected.
+  VAR_DEPS lists every input the expression uses - physical columns and
+  derived variables alike.  The statements are generated in dependency order,
+  so a derived variable may build on another one without a sequence number
+  being maintained.  A cycle is rejected.
 
   The declaration is checked against the expression: a derived variable that
   is referenced must be declared, a declared name that is not referenced is an
@@ -52,9 +55,10 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
-  @li mf_getvarlist.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_getcols.sas
 
   @param [in] libds= The target table (libref.dataset, lib must be assigned)
   @param [in] dclib= The Data Controller control library
@@ -78,7 +82,29 @@
 );
 
 %local physlist derived_err resv_list i w left nready nplaced
-  cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i;
+  cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
+  ds_cfg ds_byname ds_bad ds_baddep ds_dup ds_dep ds_ref ds_stale
+  ds_undecl ds_self ds_shadow ds_cols ds_edge ds_todo ds_ready ds_placed ds_srt;
+
+/* every working table gets its own name, so a caller with tables of the same
+  name in WORK cannot collide with this macro's intermediates */
+%let ds_cfg=%mf_getuniquename(prefix=dvcfg);
+%let ds_byname=%mf_getuniquename(prefix=dvbyn);
+%let ds_bad=%mf_getuniquename(prefix=dvbad);
+%let ds_baddep=%mf_getuniquename(prefix=dvbdp);
+%let ds_dup=%mf_getuniquename(prefix=dvdup);
+%let ds_dep=%mf_getuniquename(prefix=dvdep);
+%let ds_ref=%mf_getuniquename(prefix=dvref);
+%let ds_stale=%mf_getuniquename(prefix=dvstl);
+%let ds_undecl=%mf_getuniquename(prefix=dvund);
+%let ds_self=%mf_getuniquename(prefix=dvsfl);
+%let ds_shadow=%mf_getuniquename(prefix=dvshw);
+%let ds_cols=%mf_getuniquename(prefix=dvcol);
+%let ds_edge=%mf_getuniquename(prefix=dvedg);
+%let ds_todo=%mf_getuniquename(prefix=dvtd);
+%let ds_ready=%mf_getuniquename(prefix=dvrdy);
+%let ds_placed=%mf_getuniquename(prefix=dvpld);
+%let ds_srt=%mf_getuniquename(prefix=dvsrt);
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
@@ -110,7 +136,7 @@ run;
   %return;
 %end;
 
-data work.derived_cfg;
+data work.&ds_cfg.;
   set &dclib..mpe_derived;
   where &dc_dttmtfmt lt tx_to
     and active=1
@@ -125,13 +151,24 @@ data work.derived_cfg;
   deps=upcase(strip(var_deps));
 run;
 
-%if %mf_nobs(work.derived_cfg)=0 %then %return;
+%if %mf_nobs(work.&ds_cfg.)=0 %then %return;
 
-proc sort data=work.derived_cfg out=work.derived_byname;
+proc sort data=work.&ds_cfg. out=work.&ds_byname.;
   by name;
 run;
 
-%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
+/* one contents pass gives the names, types and lengths.  The type and length
+  matter when a derived variable shadows a physical column. */
+%mp_getcols(&libds, outds=work.&ds_cols)
+
+%let physlist=;
+proc sql noprint;
+select quote(name) into: physlist separated by ','
+  from work.&ds_cols;
+quit;
+/* a table with no columns still has to compile into a valid in-list */
+%if %length(&physlist)=0 %then %let physlist=%str(%'%');
+%let physlist=%upcase(&physlist);
 
 /* names that may never be used - DC's own metadata columns */
 %let resv_list=%str("_____DELETE__THIS__RECORD_____","TX_FROM","TX_TO","PROCESSED_DTTM","VAR_TXFROM","VAR_TXTO","VAR_PROCESSED");
@@ -140,8 +177,8 @@ run;
   * Reject anything that could break the generated data step, or that would
   * produce a variable we cannot describe to the caller
   */
-data work.derived_bad;
-  set work.derived_byname;
+data work.&ds_bad.;
+  set work.&ds_byname.;
   length reason $200;
   call missing(reason);
 
@@ -150,10 +187,6 @@ data work.derived_bad;
     then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(name,1,1),'0123456789')=0
     then reason='VAR_NM must not start with a digit';
-  %if %length(&physlist)>0 %then %do;
-  else if name in (%unquote(&physlist))
-    then reason='VAR_NM collides with a column in the target table';
-  %end;
   else if name in (%unquote(&resv_list))
     then reason='VAR_NM is reserved';
   else if type not in ('C','N')
@@ -181,22 +214,22 @@ data work.derived_bad;
   if missing(reason)=0 then output;
 run;
 
-data work.derived_dup;
-  set work.derived_byname;
+data work.&ds_dup.;
+  set work.&ds_byname.;
   by name;
   length reason $200;
   if first.name and not last.name;
   reason='VAR_NM is defined more than once';
 run;
 
-proc append base=work.derived_bad data=work.derived_dup;
+proc append base=work.&ds_bad. data=work.&ds_dup.;
 run;
 
 /**
   * The declared dependencies, one row per word
   */
-data work.derived_dep;
-  set work.derived_byname;
+data work.&ds_dep.;
+  set work.&ds_byname.;
   length dep_nm $32;
   do i=1 to countw(deps,' ');
     dep_nm=scan(deps,i,' ');
@@ -209,8 +242,8 @@ run;
   * The identifiers the expression actually references - quoted strings are
   * removed first, so a name inside a literal is not mistaken for a reference
   */
-data work.derived_ref;
-  set work.derived_byname;
+data work.&ds_ref.;
+  set work.&ds_byname.;
   length l $4000 tok $32;
   l=prxchange("s/'([^']|'')*'//",-1,logic);
   l=prxchange('s/"([^"]|"")*"//',-1,l);
@@ -225,42 +258,54 @@ run;
 /* a declared name must resolve - to a column of the target, or to a variable
   in this configuration */
 proc sql noprint;
-create table work.derived_bad_dep as
+create table work.&ds_baddep. as
   select d.name, d.dep_nm
-  from work.derived_dep d
+  from work.&ds_dep. d
   where d.dep_nm not in (%unquote(&physlist))
-    and d.dep_nm not in (select name from work.derived_byname);
+    and d.dep_nm not in (select name from work.&ds_byname.);
 
 /* every declared name must actually appear in the expression */
-create table work.derived_stale as
+create table work.&ds_stale. as
   select d.name, d.dep_nm
-  from work.derived_dep d
+  from work.&ds_dep. d
   where not exists (
-    select 1 from work.derived_ref r
+    select 1 from work.&ds_ref. r
     where r.name=d.name and r.tok=d.dep_nm
   );
 
 /* a derived variable referenced by the expression must be declared, otherwise
   the statement order would be wrong and the value would be silently missing */
-create table work.derived_undeclared as
+create table work.&ds_undecl. as
   select distinct r.name, r.tok as dep_nm
-  from work.derived_ref r
-  where r.tok in (select name from work.derived_byname)
+  from work.&ds_ref. r
+  where r.tok in (select name from work.&ds_byname.)
     and r.tok ne r.name
     and not exists (
-      select 1 from work.derived_dep d
+      select 1 from work.&ds_dep. d
       where d.name=r.name and d.dep_nm=r.tok
     );
 
 /* a variable may not depend on itself */
-create table work.derived_self as
-  select name, dep_nm from work.derived_dep
-  where name=dep_nm;
+create table work.&ds_self. as
+  select name, dep_nm from work.&ds_dep.
+  where name=dep_nm
+    and name not in (%unquote(&physlist));
+
+/* a derived variable may shadow a physical column, but the two must agree on
+  type and length - attrib cannot change either on an existing variable */
+create table work.&ds_shadow. as
+  select c.name, c.type, c.len, p.type as ptype, p.length as plen
+  from work.&ds_byname. c
+  inner join work.&ds_cols. p
+    on c.name=p.name
+  where c.type ne p.type
+    or (c.type='C' and c.len gt p.length)
+    or (c.type='N' and c.len ne p.length);
 quit;
 
-%if %mf_nobs(work.derived_bad_dep)>0 %then %do;
+%if %mf_nobs(work.&ds_baddep.)>0 %then %do;
   data _null_;
-    set work.derived_bad_dep(obs=1);
+    set work.&ds_baddep.(obs=1);
     call symputx('derived_err'
       ,cats(name,': VAR_DEPS lists ',dep_nm
         ,', which is not a column of the table or a derived variable'));
@@ -268,17 +313,27 @@ quit;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
 
-%if %mf_nobs(work.derived_self)>0 %then %do;
+%if %mf_nobs(work.&ds_shadow.)>0 %then %do;
   data _null_;
-    set work.derived_self(obs=1);
+    set work.&ds_shadow.(obs=1);
+    call symputx('derived_err'
+      ,cats(name,': VAR_NM shadows a column of the table, so VAR_TYPE and'
+        ,' VAR_LEN must match it - the column is ',ptype,'(',plen,')'));
+  run;
+  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
+%end;
+
+%if %mf_nobs(work.&ds_self.)>0 %then %do;
+  data _null_;
+    set work.&ds_self.(obs=1);
     call symputx('derived_err',cats(name,': VAR_DEPS may not list itself'));
   run;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
 
-%if %mf_nobs(work.derived_undeclared)>0 %then %do;
+%if %mf_nobs(work.&ds_undecl.)>0 %then %do;
   data _null_;
-    set work.derived_undeclared(obs=1);
+    set work.&ds_undecl.(obs=1);
     call symputx('derived_err'
       ,cats(name,': VAR_LOGIC references ',dep_nm
         ,', which is not in VAR_DEPS'));
@@ -286,9 +341,9 @@ quit;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
 
-%if %mf_nobs(work.derived_stale)>0 %then %do;
+%if %mf_nobs(work.&ds_stale.)>0 %then %do;
   data _null_;
-    set work.derived_stale(obs=1);
+    set work.&ds_stale.(obs=1);
     call symputx('derived_err'
       ,cats(name,': VAR_DEPS lists ',dep_nm
         ,', which VAR_LOGIC does not reference'));
@@ -296,13 +351,13 @@ quit;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
 
-%if %mf_nobs(work.derived_bad)>0 %then %do;
+%if %mf_nobs(work.&ds_bad.)>0 %then %do;
   data _null_;
-    set work.derived_bad;
+    set work.&ds_bad.;
     putlog 'ERR' 'OR: derived variable config: ' name= reason=;
   run;
   data _null_;
-    set work.derived_bad(obs=1);
+    set work.&ds_bad.(obs=1);
     call symputx('derived_err',cats(name,': ',reason));
   run;
   %mp_abort(iftrue=1
@@ -318,43 +373,46 @@ quit;
   * if a round places nothing, what is left is a cycle.
   */
 proc sql;
-create table work.derived_edge as
+create table work.&ds_edge. as
   select d.name as child, d.dep_nm as parent
-  from work.derived_dep d
-  where d.dep_nm in (select name from work.derived_byname);
+  from work.&ds_dep. d
+  where d.dep_nm in (select name from work.&ds_byname.)
+    and d.dep_nm ne d.name;
 quit;
 
-data work.derived_todo(keep=name);
-  set work.derived_byname;
+data work.&ds_todo.(keep=name);
+  set work.&ds_byname.;
 run;
 
-data work.derived_placed(keep=name seq);
+data work.&ds_placed.(keep=name seq);
   length name $32;
   name='';
   seq=0;
   if 0;
 run;
 
-%let left=%mf_nobs(work.derived_todo);
+%let left=%mf_nobs(work.&ds_todo.);
 %let nplaced=0;
 
 %do %while (&left>0);
   proc sql;
-  create table work.derived_ready as
+  create table work.&ds_ready. as
     select t.name
-    from work.derived_todo t
+    from work.&ds_todo. t
     where not exists (
-      select 1 from work.derived_edge e
-      inner join work.derived_todo t2 on e.parent=t2.name
+      select 1 from work.&ds_edge. e
+      inner join work.&ds_todo. t2 on e.parent=t2.name
       where e.child=t.name
     )
     order by t.name;
   quit;
 
-  %let nready=%mf_nobs(work.derived_ready);
+  %let nready=%mf_nobs(work.&ds_ready.);
   %if &nready=0 %then %do;
+    /* walk the edges, so the message names the cycle rather than just the
+      names that could not be placed */
     data _null_;
-      set work.derived_todo(obs=1);
+      set work.&ds_todo.(obs=1);
       call symputx('cyc_start',name);
     run;
     %let cyc_path=&cyc_start;
@@ -364,9 +422,9 @@ run;
       %let cyc_next=;
       proc sql noprint;
         select e.parent into: cyc_next
-        from work.derived_edge e
+        from work.&ds_edge. e
         where e.child="&cyc_cur"
-          and e.parent in (select name from work.derived_todo);
+          and e.parent in (select name from work.&ds_todo.);
       quit;
       %if %length(&cyc_next)=0 %then %let cyc_i=&left;
       %else %do;
@@ -386,33 +444,33 @@ run;
 
   /* seq carries the dependency order - the placement rounds are not name
     ordered, so the final merge must not rely on by name alone */
-  data work.derived_ready;
-    set work.derived_ready;
+  data work.&ds_ready.;
+    set work.&ds_ready.;
     seq=_n_+&nplaced;
   run;
-  proc append base=work.derived_placed data=work.derived_ready force;
+  proc append base=work.&ds_placed. data=work.&ds_ready. force;
   run;
   %let nplaced=%eval(&nplaced+&nready);
 
   proc sql;
-  delete from work.derived_todo
-    where name in (select name from work.derived_ready);
+  delete from work.&ds_todo.
+    where name in (select name from work.&ds_ready.);
   quit;
 
-  %let left=%mf_nobs(work.derived_todo);
+  %let left=%mf_nobs(work.&ds_todo.);
 %end;
 
-proc sort data=work.derived_placed;
+proc sort data=work.&ds_placed.;
   by name;
 run;
 
-data work.derived_srt;
-  merge work.derived_placed(in=in1) work.derived_byname(in=in2);
+data work.&ds_srt.;
+  merge work.&ds_placed.(in=in1) work.&ds_byname.(in=in2);
   by name;
   if in1 and in2;
 run;
 
-proc sort data=work.derived_srt;
+proc sort data=work.&ds_srt.;
   by seq;
 run;
 
@@ -421,7 +479,7 @@ run;
   */
 data _null_;
   file &outref;
-  set work.derived_srt;
+  set work.&ds_srt.;
   length stmt $4100 label $256 fmt2 $49;
   /* labels cannot contain single quotes, and are limited to 256 chars */
   label=cats(name,'=',translate(logic,' ','0D0A'x));
@@ -451,7 +509,7 @@ run;
 )
 
 data &outvars(keep=name);
-  set work.derived_srt;
+  set work.&ds_srt.;
 run;
 
 %if %length(&outview)>0 and &outview ne 0 %then %do;
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index f7d7141..491c528 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -233,6 +233,49 @@ run;
   outds=work.test_results
 )
 
+/**
+  * Test 4 - a derived variable may shadow a physical column
+  */
+
+data work.class_orig;
+  set sashelp.class;
+run;
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='AGE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 1'
+    ,active=1;
+quit;
+
+%mpe_derived(libds=work.class,
+  dclib=&dclib,
+  outref=dercode4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.chk4shadow;
+  merge work.vw_derived4(keep=name age rename=(age=age_shadow))
+    work.class_orig(keep=name age);
+  by name;
+  if age_shadow ne age + 1 then output;
+run;
+
+%mp_assertdsobs(work.chk4shadow,
+  desc=Test4 - a derived variable can shadow a physical column,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
 /* tidy up so repeated runs start clean */
 proc sql;
 delete from &dclib..mpe_derived where upcase(libref)='WORK';
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 6dfb632..5833303 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -18,9 +18,9 @@
   VAR_DEPS must list every input the expression uses, physical columns
   and derived variables alike.  The hook checks the part that does not need
   the target library: a derived variable referenced by an expression must be
-  declared, a variable may not depend on itself, and the resulting graph must
-  be acyclic - across the incoming rows and the current rows they do not
-  replace.
+  declared, and the resulting graph must be acyclic - across the incoming
+  rows and the current rows they do not replace.  A self reference is allowed,
+  since it may legitimately read a physical column of the same name.
 
   The target table is not assigned in this context, so the collision between
   VAR_NM and a physical column, and the resolution of a physical dependency,
@@ -165,11 +165,6 @@ data work.derived_ref;
 run;
 
 proc sql;
-create table work.derived_self as
-  select libref, dsn, name, dep_nm
-  from work.derived_dep
-  where name=dep_nm;
-
 create table work.derived_undeclared as
   select r.libref, r.dsn, r.name, r.tok as dep_nm
   from work.derived_ref r
@@ -185,18 +180,6 @@ create table work.derived_undeclared as
     );
 quit;
 
-%if %mf_nobs(work.derived_self)>0 %then %do;
-  data _null_;
-    set work.derived_self(obs=1);
-    call symputx('errmsg'
-      ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
-  run;
-  %mp_abort(iftrue=1
-    ,mac=mpe_derived_postedit
-    ,msg=%superq(errmsg)
-  )
-%end;
-
 %if %mf_nobs(work.derived_undeclared)>0 %then %do;
   data _null_;
     set work.derived_undeclared(obs=1);
@@ -215,10 +198,13 @@ quit;
   * been placed; a round that places nothing leaves a cycle
   */
 proc sql;
+/* a self reference is not a graph edge - it reads the physical column of the
+  same name, which the macro validates where the target library is assigned */
 create table work.derived_edge as
   select d.libref, d.dsn, d.name as child, d.dep_nm as parent
   from work.derived_dep d
-  where d.dep_nm in (select name from work.derived_rows);
+  where d.dep_nm in (select name from work.derived_rows)
+    and d.dep_nm ne d.name;
 quit;
 
 data work.derived_todo;

From 2d5de9a324a07a36edc0a4f74c8fad09fde07cbf Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 22:27:19 +0000
Subject: [PATCH 24/48] fix(derived): do not drop a shadowed column from
 staging or downloads

With shadowing allowed, a derived name can also be a physical column.  The
derived droplist was built from every derived name, so such a column would
have been dropped on submission (losing the user's edit) and from the view
and raw downloads (losing a real column).

stagedata, viewdata and getrawdata now subtract the target's own columns
from the droplist, so only genuinely virtual columns are dropped.
---
 sas/sasjs/services/editors/stagedata.sas | 7 ++++++-
 sas/sasjs/services/public/getrawdata.sas | 7 ++++++-
 sas/sasjs/services/public/viewdata.sas   | 7 ++++++-
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/sas/sasjs/services/editors/stagedata.sas b/sas/sasjs/services/editors/stagedata.sas
index 0b9edb6..adec8e8 100755
--- a/sas/sasjs/services/editors/stagedata.sas
+++ b/sas/sasjs/services/editors/stagedata.sas
@@ -20,6 +20,7 @@
   @li mf_verifymacvars.sas
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
+  @li mf_getvarlist.sas
   @li mpe_loader.sas
   @li mpe_filtermaster.sas
   @li mpe_derived.sas
@@ -120,11 +121,15 @@ run;
 %if %mf_nobs(work.derived_vars)>0 %then %do;
   proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
   run;
+  /* a derived variable may shadow a physical column - that column is still
+    editable, so it must not be dropped */
+  %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
   proc sql noprint;
   select a.name into: derived_droplist separated by ' '
     from work.derived_vars a
     inner join work.jsdata_vars b
-    on a.name=b.name;
+    on a.name=b.name
+    where a.name not in (%unquote(&physlist));
   quit;
 %end;
 
diff --git a/sas/sasjs/services/public/getrawdata.sas b/sas/sasjs/services/public/getrawdata.sas
index a9d15d8..131adfd 100644
--- a/sas/sasjs/services/public/getrawdata.sas
+++ b/sas/sasjs/services/public/getrawdata.sas
@@ -22,6 +22,7 @@
   @li mp_streamfile.sas
   @li mp_validatecol.sas
   @li mf_nobs.sas
+  @li mf_getvarlist.sas
   @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li mpe_filtermaster.sas
@@ -158,9 +159,13 @@ run;
 %let derived_droplist=;
 %if &derived_nobs>0 %then %do;
   %let filterds=work.vw_derived;
+  /* a derived variable may shadow a physical column - that column carries
+    real data, so it stays in the output */
+  %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
   proc sql noprint;
   select name into: derived_droplist separated by ' '
-    from work.derived_vars;
+    from work.derived_vars
+    where name not in (%unquote(&physlist));
   quit;
 %end;
 %else %let filterds=&libds;
diff --git a/sas/sasjs/services/public/viewdata.sas b/sas/sasjs/services/public/viewdata.sas
index a25504f..69d52d1 100644
--- a/sas/sasjs/services/public/viewdata.sas
+++ b/sas/sasjs/services/public/viewdata.sas
@@ -59,6 +59,7 @@
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
+  @li mf_getvarlist.sas
   @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getversions.sas
@@ -232,9 +233,13 @@ run;
   %let derived_droplist=;
   %if &derived_nobs>0 %then %do;
     %let filterds=work.vw_derived;
+    /* a derived variable may shadow a physical column - that column carries
+      real data, so it stays in the output */
+    %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
     proc sql noprint;
     select name into: derived_droplist separated by ' '
-      from work.derived_vars;
+      from work.derived_vars
+      where name not in (%unquote(&physlist));
     quit;
   %end;
   %else %let filterds=&libds;

From 6a1128a46352f77c95dd02df9309f0984a1db2d1 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 22:28:44 +0000
Subject: [PATCH 25/48] fix(derived): do not duplicate a shadowed column in the
 grid payload

cols1 already holds the physical column, so appending the derived row for the
same name would send it twice.  The derived row is now appended only when the
name is new; a shadowed column keeps the physical label and format, while its
value comes from the view.
---
 sas/sasjs/services/editors/getdata.sas | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index 8d9858e..423a3a8 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -544,10 +544,14 @@ select upcase(loadtype)
 %if &derived_nobs>0 %then %do;
   %mp_getcols(work.vw_derived, outds=derived_cols_all)
   proc sql;
+  /* a derived variable may shadow a physical column - that column is already
+    in cols1, and adding it again would duplicate it in the grid.  The
+    physical row keeps its label and format; the value comes from the view. */
   create table work.derived_cols as
     select a.*
     from work.derived_cols_all a
-    where a.NAME in (select NAME from work.derived_vars);
+    where a.NAME in (select NAME from work.derived_vars)
+      and a.NAME not in (select NAME from work.cols1);
   quit;
   data work.cols1;
     set work.cols1 work.derived_cols;

From 5af8e6fc359221586af74a685159dfd75a40ec45 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 23:00:01 +0000
Subject: [PATCH 26/48] revert(derived): no shadowing, and no self reference

Allowing a derived variable to reuse a physical column's name was wrong: an
intra-row calculation (SUMMER = SUMMER + 1) implies a retained value, which
depends on the order rows are read.  The same argument rules out a variable
listing itself as a dependency.

Reverted:
- the name collision is an error again ('VAR_NM collides with a column in the
  target table')
- VAR_DEPS may not list the variable itself
- the physical column list is back to mf_getvarlist - the type/length check
  only existed to support shadowing
- the droplist guards in stagedata / viewdata / getrawdata, and the duplicate
  suppression in getdata, are no longer needed
- the docs and both test suites drop the shadowing case
---
 sas/sasjs/macros/mpe_derived.sas              | 61 +++++--------------
 sas/sasjs/macros/mpe_derived.test.sas         | 43 -------------
 sas/sasjs/services/editors/getdata.sas        |  6 +-
 sas/sasjs/services/editors/stagedata.sas      |  7 +--
 .../services/hooks/mpe_derived_postedit.sas   | 28 ++++++---
 sas/sasjs/services/public/getrawdata.sas      |  7 +--
 sas/sasjs/services/public/viewdata.sas        |  7 +--
 7 files changed, 39 insertions(+), 120 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 79b3201..dfc5d84 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -34,11 +34,8 @@
       REGION = <expression>;
 
   Type and length are declared by the administrator, so the result is
-  deterministic, and a derived variable may shadow a physical column - the
-  expression then reads the physical value, eg SUMMER = SUMMER + 1 - in which
-  case VAR_TYPE and VAR_LEN must match the column, as attrib cannot change
-  either on an existing variable.  The label is the variable name, an equals
-  sign and the expression.
+  deterministic.  The label is the variable name, an equals sign and the
+  expression.
 
   VAR_DEPS lists every input the expression uses - physical columns and
   derived variables alike.  The statements are generated in dependency order,
@@ -51,14 +48,16 @@
   variable defined for it.
 
   Semicolons, macro triggers and comments are rejected, so a row can only ever
-  be an expression.
+  be an expression.  A derived variable may not reuse the name of a physical
+  column, and may not reference itself: an intra-row calculation would imply a
+  retained value, which depends on the order rows happen to arrive in.
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_nobs.sas
   @li mp_abort.sas
-  @li mp_getcols.sas
 
   @param [in] libds= The target table (libref.dataset, lib must be assigned)
   @param [in] dclib= The Data Controller control library
@@ -84,7 +83,7 @@
 %local physlist derived_err resv_list i w left nready nplaced
   cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
   ds_cfg ds_byname ds_bad ds_baddep ds_dup ds_dep ds_ref ds_stale
-  ds_undecl ds_self ds_shadow ds_cols ds_edge ds_todo ds_ready ds_placed ds_srt;
+  ds_undecl ds_self ds_edge ds_todo ds_ready ds_placed ds_srt;
 
 /* every working table gets its own name, so a caller with tables of the same
   name in WORK cannot collide with this macro's intermediates */
@@ -98,8 +97,6 @@
 %let ds_stale=%mf_getuniquename(prefix=dvstl);
 %let ds_undecl=%mf_getuniquename(prefix=dvund);
 %let ds_self=%mf_getuniquename(prefix=dvsfl);
-%let ds_shadow=%mf_getuniquename(prefix=dvshw);
-%let ds_cols=%mf_getuniquename(prefix=dvcol);
 %let ds_edge=%mf_getuniquename(prefix=dvedg);
 %let ds_todo=%mf_getuniquename(prefix=dvtd);
 %let ds_ready=%mf_getuniquename(prefix=dvrdy);
@@ -157,18 +154,7 @@ proc sort data=work.&ds_cfg. out=work.&ds_byname.;
   by name;
 run;
 
-/* one contents pass gives the names, types and lengths.  The type and length
-  matter when a derived variable shadows a physical column. */
-%mp_getcols(&libds, outds=work.&ds_cols)
-
-%let physlist=;
-proc sql noprint;
-select quote(name) into: physlist separated by ','
-  from work.&ds_cols;
-quit;
-/* a table with no columns still has to compile into a valid in-list */
-%if %length(&physlist)=0 %then %let physlist=%str(%'%');
-%let physlist=%upcase(&physlist);
+%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
 
 /* names that may never be used - DC's own metadata columns */
 %let resv_list=%str("_____DELETE__THIS__RECORD_____","TX_FROM","TX_TO","PROCESSED_DTTM","VAR_TXFROM","VAR_TXTO","VAR_PROCESSED");
@@ -187,6 +173,10 @@ data work.&ds_bad.;
     then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(name,1,1),'0123456789')=0
     then reason='VAR_NM must not start with a digit';
+  %if %length(&physlist)>0 %then %do;
+  else if name in (%unquote(&physlist))
+    then reason='VAR_NM collides with a column in the target table';
+  %end;
   else if name in (%unquote(&resv_list))
     then reason='VAR_NM is reserved';
   else if type not in ('C','N')
@@ -288,19 +278,7 @@ create table work.&ds_undecl. as
 /* a variable may not depend on itself */
 create table work.&ds_self. as
   select name, dep_nm from work.&ds_dep.
-  where name=dep_nm
-    and name not in (%unquote(&physlist));
-
-/* a derived variable may shadow a physical column, but the two must agree on
-  type and length - attrib cannot change either on an existing variable */
-create table work.&ds_shadow. as
-  select c.name, c.type, c.len, p.type as ptype, p.length as plen
-  from work.&ds_byname. c
-  inner join work.&ds_cols. p
-    on c.name=p.name
-  where c.type ne p.type
-    or (c.type='C' and c.len gt p.length)
-    or (c.type='N' and c.len ne p.length);
+  where name=dep_nm;
 quit;
 
 %if %mf_nobs(work.&ds_baddep.)>0 %then %do;
@@ -313,16 +291,6 @@ quit;
   %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
 %end;
 
-%if %mf_nobs(work.&ds_shadow.)>0 %then %do;
-  data _null_;
-    set work.&ds_shadow.(obs=1);
-    call symputx('derived_err'
-      ,cats(name,': VAR_NM shadows a column of the table, so VAR_TYPE and'
-        ,' VAR_LEN must match it - the column is ',ptype,'(',plen,')'));
-  run;
-  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
-%end;
-
 %if %mf_nobs(work.&ds_self.)>0 %then %do;
   data _null_;
     set work.&ds_self.(obs=1);
@@ -376,8 +344,7 @@ proc sql;
 create table work.&ds_edge. as
   select d.name as child, d.dep_nm as parent
   from work.&ds_dep. d
-  where d.dep_nm in (select name from work.&ds_byname.)
-    and d.dep_nm ne d.name;
+  where d.dep_nm in (select name from work.&ds_byname.);
 quit;
 
 data work.&ds_todo.(keep=name);
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 491c528..f7d7141 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -233,49 +233,6 @@ run;
   outds=work.test_results
 )
 
-/**
-  * Test 4 - a derived variable may shadow a physical column
-  */
-
-data work.class_orig;
-  set sashelp.class;
-run;
-
-proc sql;
-insert into &dclib..mpe_derived set
-    tx_from=0
-    ,tx_to='31DEC5999:23:59:59'dt
-    ,libref='WORK'
-    ,dsn='CLASS'
-    ,var_nm='AGE'
-    ,var_deps='AGE'
-    ,var_type='N'
-    ,var_len=8
-    ,var_fmt=''
-    ,var_logic='AGE + 1'
-    ,active=1;
-quit;
-
-%mpe_derived(libds=work.class,
-  dclib=&dclib,
-  outref=dercode4,
-  outview=work.vw_derived4,
-  outvars=work.derived_vars4
-)
-
-data work.chk4shadow;
-  merge work.vw_derived4(keep=name age rename=(age=age_shadow))
-    work.class_orig(keep=name age);
-  by name;
-  if age_shadow ne age + 1 then output;
-run;
-
-%mp_assertdsobs(work.chk4shadow,
-  desc=Test4 - a derived variable can shadow a physical column,
-  test=EQUALS 0,
-  outds=work.test_results
-)
-
 /* tidy up so repeated runs start clean */
 proc sql;
 delete from &dclib..mpe_derived where upcase(libref)='WORK';
diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index 423a3a8..8d9858e 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -544,14 +544,10 @@ select upcase(loadtype)
 %if &derived_nobs>0 %then %do;
   %mp_getcols(work.vw_derived, outds=derived_cols_all)
   proc sql;
-  /* a derived variable may shadow a physical column - that column is already
-    in cols1, and adding it again would duplicate it in the grid.  The
-    physical row keeps its label and format; the value comes from the view. */
   create table work.derived_cols as
     select a.*
     from work.derived_cols_all a
-    where a.NAME in (select NAME from work.derived_vars)
-      and a.NAME not in (select NAME from work.cols1);
+    where a.NAME in (select NAME from work.derived_vars);
   quit;
   data work.cols1;
     set work.cols1 work.derived_cols;
diff --git a/sas/sasjs/services/editors/stagedata.sas b/sas/sasjs/services/editors/stagedata.sas
index adec8e8..0b9edb6 100755
--- a/sas/sasjs/services/editors/stagedata.sas
+++ b/sas/sasjs/services/editors/stagedata.sas
@@ -20,7 +20,6 @@
   @li mf_verifymacvars.sas
   @li mf_mkdir.sas
   @li mf_getuniquefileref.sas
-  @li mf_getvarlist.sas
   @li mpe_loader.sas
   @li mpe_filtermaster.sas
   @li mpe_derived.sas
@@ -121,15 +120,11 @@ run;
 %if %mf_nobs(work.derived_vars)>0 %then %do;
   proc contents noprint data=jsdata out=work.jsdata_vars(keep=name);
   run;
-  /* a derived variable may shadow a physical column - that column is still
-    editable, so it must not be dropped */
-  %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
   proc sql noprint;
   select a.name into: derived_droplist separated by ' '
     from work.derived_vars a
     inner join work.jsdata_vars b
-    on a.name=b.name
-    where a.name not in (%unquote(&physlist));
+    on a.name=b.name;
   quit;
 %end;
 
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 5833303..6dfb632 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -18,9 +18,9 @@
   VAR_DEPS must list every input the expression uses, physical columns
   and derived variables alike.  The hook checks the part that does not need
   the target library: a derived variable referenced by an expression must be
-  declared, and the resulting graph must be acyclic - across the incoming
-  rows and the current rows they do not replace.  A self reference is allowed,
-  since it may legitimately read a physical column of the same name.
+  declared, a variable may not depend on itself, and the resulting graph must
+  be acyclic - across the incoming rows and the current rows they do not
+  replace.
 
   The target table is not assigned in this context, so the collision between
   VAR_NM and a physical column, and the resolution of a physical dependency,
@@ -165,6 +165,11 @@ data work.derived_ref;
 run;
 
 proc sql;
+create table work.derived_self as
+  select libref, dsn, name, dep_nm
+  from work.derived_dep
+  where name=dep_nm;
+
 create table work.derived_undeclared as
   select r.libref, r.dsn, r.name, r.tok as dep_nm
   from work.derived_ref r
@@ -180,6 +185,18 @@ create table work.derived_undeclared as
     );
 quit;
 
+%if %mf_nobs(work.derived_self)>0 %then %do;
+  data _null_;
+    set work.derived_self(obs=1);
+    call symputx('errmsg'
+      ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
+  run;
+  %mp_abort(iftrue=1
+    ,mac=mpe_derived_postedit
+    ,msg=%superq(errmsg)
+  )
+%end;
+
 %if %mf_nobs(work.derived_undeclared)>0 %then %do;
   data _null_;
     set work.derived_undeclared(obs=1);
@@ -198,13 +215,10 @@ quit;
   * been placed; a round that places nothing leaves a cycle
   */
 proc sql;
-/* a self reference is not a graph edge - it reads the physical column of the
-  same name, which the macro validates where the target library is assigned */
 create table work.derived_edge as
   select d.libref, d.dsn, d.name as child, d.dep_nm as parent
   from work.derived_dep d
-  where d.dep_nm in (select name from work.derived_rows)
-    and d.dep_nm ne d.name;
+  where d.dep_nm in (select name from work.derived_rows);
 quit;
 
 data work.derived_todo;
diff --git a/sas/sasjs/services/public/getrawdata.sas b/sas/sasjs/services/public/getrawdata.sas
index 131adfd..a9d15d8 100644
--- a/sas/sasjs/services/public/getrawdata.sas
+++ b/sas/sasjs/services/public/getrawdata.sas
@@ -22,7 +22,6 @@
   @li mp_streamfile.sas
   @li mp_validatecol.sas
   @li mf_nobs.sas
-  @li mf_getvarlist.sas
   @li mpe_derived.sas
   @li mpe_validatecol.sas
   @li mpe_filtermaster.sas
@@ -159,13 +158,9 @@ run;
 %let derived_droplist=;
 %if &derived_nobs>0 %then %do;
   %let filterds=work.vw_derived;
-  /* a derived variable may shadow a physical column - that column carries
-    real data, so it stays in the output */
-  %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
   proc sql noprint;
   select name into: derived_droplist separated by ' '
-    from work.derived_vars
-    where name not in (%unquote(&physlist));
+    from work.derived_vars;
   quit;
 %end;
 %else %let filterds=&libds;
diff --git a/sas/sasjs/services/public/viewdata.sas b/sas/sasjs/services/public/viewdata.sas
index 69d52d1..a25504f 100644
--- a/sas/sasjs/services/public/viewdata.sas
+++ b/sas/sasjs/services/public/viewdata.sas
@@ -59,7 +59,6 @@
   @li mp_searchdata.sas
   @li mp_validatecol.sas
   @li mpe_columnlevelsecurity.sas
-  @li mf_getvarlist.sas
   @li mpe_derived.sas
   @li mpe_dsmeta.sas
   @li mpe_getversions.sas
@@ -233,13 +232,9 @@ run;
   %let derived_droplist=;
   %if &derived_nobs>0 %then %do;
     %let filterds=work.vw_derived;
-    /* a derived variable may shadow a physical column - that column carries
-      real data, so it stays in the output */
-    %let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
     proc sql noprint;
     select name into: derived_droplist separated by ' '
-      from work.derived_vars
-      where name not in (%unquote(&physlist));
+      from work.derived_vars;
     quit;
   %end;
   %else %let filterds=&libds;

From 9f70c56b581044bc31584b4abdb6d5e5c5d4aa61 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Tue, 20 Oct 2026 23:00:07 +0000
Subject: [PATCH 27/48] chore(deps): @angular/* 20.3.34

GHSA-jfgx-762w-99vr (XSRF token leakage in the HTTP client, an incomplete fix
for CVE-2025-66035) covers @angular/common 20.0.0 - 20.3.33, and the 'Check
audit' step fails on any vulnerability, so every branch is red until this
lands.

The nine packages are pinned exactly and must move together - Angular pins its
siblings at the same patch level - so package.json and the lock are updated as
a set.  The local tarball for @sheet/crypto is only present in encrypted form,
so npm cannot run here; the client build and unit tests in CI are the proof.
---
 client/package-lock.json | 106 +++++++++++++++++++--------------------
 client/package.json      |  18 +++----
 2 files changed, 62 insertions(+), 62 deletions(-)

diff --git a/client/package-lock.json b/client/package-lock.json
index e677d39..efdccb0 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -7,15 +7,15 @@
       "name": "data_controller-client",
       "hasInstallScript": true,
       "dependencies": {
-        "@angular/animations": "20.3.33",
+        "@angular/animations": "20.3.34",
         "@angular/cdk": "^20.2.14",
-        "@angular/common": "20.3.33",
-        "@angular/compiler": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/forms": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
-        "@angular/platform-browser-dynamic": "20.3.33",
-        "@angular/router": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/compiler": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/forms": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
+        "@angular/platform-browser-dynamic": "20.3.34",
+        "@angular/router": "20.3.34",
         "@cds/core": "^6.15.1",
         "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
         "@clr/icons": "^13.0.2",
@@ -65,7 +65,7 @@
         "@angular-eslint/schematics": "19.8.1",
         "@angular-eslint/template-parser": "19.8.1",
         "@angular/cli": "20.3.37",
-        "@angular/compiler-cli": "20.3.33",
+        "@angular/compiler-cli": "20.3.34",
         "@babel/plugin-proposal-private-methods": "^7.18.6",
         "@compodoc/compodoc": "^2.0.0",
         "@cypress/webpack-preprocessor": "^5.17.1",
@@ -1439,9 +1439,9 @@
       }
     },
     "node_modules/@angular/animations": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.3.33.tgz",
-      "integrity": "sha512-z9gPYm1byXYE0tMGzjjDXE4OprxvjUhV4jOXVfLLlU0Sk5ChM3GwR0m2Vh+hDWDrJ2f7nfKKTeoGlLriP7NPZA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.3.34.tgz",
+      "integrity": "sha512-HdDfbLwpxzx49wAWz4oOUQKSkZcFNRpWEGsDEgufZ3ExwxRk1/2QPJuvZ2CbhKeEcLPrLKUorQ8XlGrNsJmNiQ==",
       "deprecated": "@angular/animations is deprecated. Use `animate.enter` and `animate.leave` instead. For more information see: https://v22.angular.dev/guide/animations.",
       "license": "MIT",
       "dependencies": {
@@ -1451,7 +1451,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/core": "20.3.33"
+        "@angular/core": "20.3.34"
       }
     },
     "node_modules/@angular/cdk": {
@@ -1987,9 +1987,9 @@
       }
     },
     "node_modules/@angular/common": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.33.tgz",
-      "integrity": "sha512-PMfSF2j/C3gGHPp6O6c/BL+32msNzhmkzavrxtmBSxIGAOB5Q4KNktnIuolVrvVvu/0UrZj1/zwT91r/GKRaeA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.34.tgz",
+      "integrity": "sha512-OhB/l0cNldOvQeICqbWqRUzK5fF6zNYy44Q4cIGJZql8HXL/invGiqWLE9f+kU8z4jRisofHIGTBtyJQYYOudg==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -1998,14 +1998,14 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/core": "20.3.33",
+        "@angular/core": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/@angular/compiler": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.33.tgz",
-      "integrity": "sha512-tqKZaapDV05r2SXA6bOzTSpiNhSoqZR9Usuacs5myMpbpt4gwNMAZkgaIoIcPutnhbd3qT+Og54VCVvCOXF8pg==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.34.tgz",
+      "integrity": "sha512-8agPW384c/JER1ersPTmJJfcCOnlYcBbdj0MYRxNdiYe+7gtnspF3ORuJmxqCEpMS17AljSmk2skwUcYnHB6xA==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2015,9 +2015,9 @@
       }
     },
     "node_modules/@angular/compiler-cli": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.3.33.tgz",
-      "integrity": "sha512-NKE4N4ssXysHLJQ7xVqvAv/yf3JfJxaFz5H+Ue+EnYToAmTxXe7uyt2MPRIydef/H63n7O9uphVp5P5GkRxVuA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.3.34.tgz",
+      "integrity": "sha512-PKHzhSOyDUFh64Ull3fUwgFlxnsQNEAuRyz4KqQ8FsTGZ4pOpUaZntO+2Xn1nX/gT2whSyGBJhD3H37zA8ZGEQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -2038,7 +2038,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/compiler": "20.3.33",
+        "@angular/compiler": "20.3.34",
         "typescript": ">=5.8 <6.0"
       },
       "peerDependenciesMeta": {
@@ -2048,9 +2048,9 @@
       }
     },
     "node_modules/@angular/core": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.33.tgz",
-      "integrity": "sha512-Ld/AhODjPf8c14xUOWUnJygaZb2TsRy99ZhwFkEsRJOzqauzEFovb8Hb2MEaB1e4RolADN+HWxPvE4n+u3HFmA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.34.tgz",
+      "integrity": "sha512-xOvSmGoao3SVPz//R7zTrq7+WdrIGi+WNpAdxJEvJJXuSpLGS3kSUqukjY4kOMochg2v0NCEPIdCyH+yPs8JSw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2059,7 +2059,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/compiler": "20.3.33",
+        "@angular/compiler": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0",
         "zone.js": "~0.15.0"
       },
@@ -2073,9 +2073,9 @@
       }
     },
     "node_modules/@angular/forms": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.3.33.tgz",
-      "integrity": "sha512-UmwKS9toyYhgq8LEbvFxNBfD9dUXr+B6Y5OTwBGwIRy8RhUrOiLwrXTRr+2y/yREGgDtGAWl8zQRVMVk2BNybw==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.3.34.tgz",
+      "integrity": "sha512-DndQJEoORsdthFFVRvT9WHn8YcA85jkjMxCLtXyf+TDVk0zbqObbFDfrloRnJwsINqfDNsJrz23ko3YU6sjnTw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2084,16 +2084,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/@angular/platform-browser": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.33.tgz",
-      "integrity": "sha512-od2uw6ee68r5TczjoYDsJuIDWoH4op3/req3sdeFtVHWdQgzf/AhM0B5ZBVRya8THon/b/SDHdhxOrUTgcszTA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.34.tgz",
+      "integrity": "sha512-0bd2Bj05WPoBTFM0A13IwbzxKVQPstx/N+7X0ZWKCifliyydIc7DBHu+r37ybJpl8ZsCVBe7DoR9Gon4lZ+zlw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2102,9 +2102,9 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/animations": "20.3.33",
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33"
+        "@angular/animations": "20.3.34",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34"
       },
       "peerDependenciesMeta": {
         "@angular/animations": {
@@ -2113,9 +2113,9 @@
       }
     },
     "node_modules/@angular/platform-browser-dynamic": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.3.33.tgz",
-      "integrity": "sha512-IwElnTCUW1ZZRr/pw6IRb9dRyNF8KBOh9QoGcmTk6uRAWynenYAe5yICFvgOKozqOHBarFNv8h4jXXJ/W8TcfA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.3.34.tgz",
+      "integrity": "sha512-vqKzExmUDf5jpIlNZRuICsLzaLWY5yTvuXsJhe97KTbC29BD71LhRGhEu7fjmR5uYy+LForpWYm5IsYlJcJAGQ==",
       "deprecated": "@angular/platform-browser-dynamic is deprecated. Use `@angular/platform-browser` instead.",
       "license": "MIT",
       "dependencies": {
@@ -2125,16 +2125,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/compiler": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33"
+        "@angular/common": "20.3.34",
+        "@angular/compiler": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34"
       }
     },
     "node_modules/@angular/router": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.3.33.tgz",
-      "integrity": "sha512-M55ibvrJSWGP7sGf+uZFvC66cxRkFlS4TjtI4Owus4gDCqW69/jhUo8pNGXf0T3QpO3iorbHScZfGwVmLATLLA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.3.34.tgz",
+      "integrity": "sha512-vQo2f/94nUoNMZaxGY0+fK1t9Oe7zJuxHhoMVYhDbCqKxINVmxMX/9uEHlHeu9UIE7JMazepQjRzBLo7bf9xCw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2143,9 +2143,9 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
diff --git a/client/package.json b/client/package.json
index 863d1f4..bd84d26 100644
--- a/client/package.json
+++ b/client/package.json
@@ -41,15 +41,15 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "20.3.33",
+    "@angular/animations": "20.3.34",
     "@angular/cdk": "^20.2.14",
-    "@angular/common": "20.3.33",
-    "@angular/compiler": "20.3.33",
-    "@angular/core": "20.3.33",
-    "@angular/forms": "20.3.33",
-    "@angular/platform-browser": "20.3.33",
-    "@angular/platform-browser-dynamic": "20.3.33",
-    "@angular/router": "20.3.33",
+    "@angular/common": "20.3.34",
+    "@angular/compiler": "20.3.34",
+    "@angular/core": "20.3.34",
+    "@angular/forms": "20.3.34",
+    "@angular/platform-browser": "20.3.34",
+    "@angular/platform-browser-dynamic": "20.3.34",
+    "@angular/router": "20.3.34",
     "@cds/core": "^6.15.1",
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
@@ -99,7 +99,7 @@
     "@angular-eslint/schematics": "19.8.1",
     "@angular-eslint/template-parser": "19.8.1",
     "@angular/cli": "20.3.37",
-    "@angular/compiler-cli": "20.3.33",
+    "@angular/compiler-cli": "20.3.34",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@compodoc/compodoc": "^2.0.0",
     "@cypress/webpack-preprocessor": "^5.17.1",

From 5f5d8681b9d78280a5200309917415e135c3ce5b Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 01:05:10 +0000
Subject: [PATCH 28/48] fix(derived): guard the empty physical column list in
 the dep check

Same fix as the core review: mf_getvarlist returns nothing for a zero-column
table, so the dependency query compiled as 'not in ()'.
---
 sas/sasjs/macros/mpe_derived.sas | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index dfc5d84..7678262 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -251,8 +251,13 @@ proc sql noprint;
 create table work.&ds_baddep. as
   select d.name, d.dep_nm
   from work.&ds_dep. d
-  where d.dep_nm not in (%unquote(&physlist))
-    and d.dep_nm not in (select name from work.&ds_byname.);
+  where d.dep_nm not in (select name from work.&ds_byname.)
+  /* a table with no columns leaves physlist empty - fall back to the derived
+    names alone rather than compiling an empty in-list */
+  %if %length(&physlist)>0 %then %do;
+    and d.dep_nm not in (%unquote(&physlist))
+  %end;
+  ;
 
 /* every declared name must actually appear in the expression */
 create table work.&ds_stale. as

From 1b5c4b3ecd962a28e08fc5671b57dbf7ae0b8d9f Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 01:49:04 +0000
Subject: [PATCH 29/48] fix(derived): truncate the derived label on an even
 quote boundary

The label had 256 chars of storage, so the doubled quotes were truncated on
assignment; a cut between the halves of an escape leaves an unterminated
literal and the view fails to build.  The label now holds 512 chars, quotes
are doubled, then the 256 boundary is applied, retreating one character when
the cut lands on an odd quote count.
---
 sas/sasjs/macros/mpe_derived.sas | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 7678262..d1eeeae 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -452,11 +452,17 @@ run;
 data _null_;
   file &outref;
   set work.&ds_srt.;
-  length stmt $4100 label $256 fmt2 $49;
-  /* labels cannot contain single quotes, and are limited to 256 chars */
+  /* 512 so the doubled quotes fit before the 256 boundary is applied */
+  length stmt $4600 label $512 fmt2 $49;
+  /* labels are limited to 256 chars.  The quotes are doubled first, then the
+    cut is made on an even quote boundary - truncating between the two halves of
+    an escape would leave the generated literal unterminated */
   label=cats(name,'=',translate(logic,' ','0D0A'x));
   label=tranwrd(label,"'","''");
-  label=substr(label,1,256);
+  if length(label)>256 then do;
+    label=substr(label,1,256);
+    if mod(count(label,"'"),2) ne 0 then label=substr(label,1,255);
+  end;
   if not missing(fmt) and index(fmt,'.')=0 then fmt=cats(fmt,'.');
   if type='C' then do;
     if missing(fmt) then fmt2=cats('$',len,'.');

From 7c9ce9a712137e9825a5f2ff52796d765423147b Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 08:14:53 +0000
Subject: [PATCH 30/48] feat(derived): optional VAR_LABEL on MPE_DERIVED

A configured label replaces the default NAME=expression; when it is missing or
blank the default is used, so existing rows are unaffected.

- MPE_DERIVED gains VAR_LABEL varchar(256), in the ddl, mpe_makedatamodel and
  the v7.19 migration
- mpe_derived uses it, through the same escaping and 256-char truncation as
  the default label
- the hook trims it; the mock mirrors the fallback
- test 1 now sets a label on BAND and asserts it
---
 sas/mocks/sasjs/services/admin/makedata.js    |  2 ++
 sas/mocks/sasjs/services/dcMockUtils.js       |  2 +-
 sas/sasjs/db/datactrl/mpe_derived.ddl         |  1 +
 .../db/migrations/20261019_v7.19_release.sas  |  1 +
 sas/sasjs/macros/mpe_derived.sas              | 19 ++++++++++++-------
 sas/sasjs/macros/mpe_derived.test.sas         |  3 ++-
 sas/sasjs/macros/mpe_makedatamodel.sas        |  1 +
 .../services/hooks/mpe_derived_postedit.sas   |  1 +
 8 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index 95c79d9..c5a0521 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -323,6 +323,7 @@ const tables = {
       var_type: 'C',
       var_len: 14,
       var_fmt: '',
+      var_label: 'Region and state',
       var_logic: "cats(REGION,'-',STATE)",
       active: 1
     }
@@ -3317,6 +3318,7 @@ const schema = {
     { name: 'VAR_TYPE', type: 'C', length: 1, format: '', label: '', notnull: false },
     { name: 'VAR_LEN', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'VAR_FMT', type: 'C', length: 32, format: '', label: '', notnull: false },
+    { name: 'VAR_LABEL', type: 'C', length: 256, format: '', label: '', notnull: false },
     { name: 'VAR_LOGIC', type: 'C', length: 4000, format: '', label: '', notnull: false },
     { name: 'ACTIVE', type: 'N', length: 8, format: '', label: '', notnull: false },
     { name: 'TX_TO', type: 'N', length: 8, format: 'datetime19.3', label: '', notnull: true }
diff --git a/sas/mocks/sasjs/services/dcMockUtils.js b/sas/mocks/sasjs/services/dcMockUtils.js
index 9054e36..4a03840 100644
--- a/sas/mocks/sasjs/services/dcMockUtils.js
+++ b/sas/mocks/sasjs/services/dcMockUtils.js
@@ -1361,7 +1361,7 @@ function mpeDerived(opts) {
                 length: Number(colVal(r, 'VAR_LEN')) || 8,
                 format: String(colVal(r, 'VAR_FMT') || ''),
                 logic,
-                label: nm + '=' + logic
+                label: String(colVal(r, 'VAR_LABEL') || '').trim() || nm + '=' + logic
             }
         })
         .filter(d => d.name && (d.type === 'C' || d.type === 'N') && d.logic)
diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
index 8c3ff53..a0c92e8 100644
--- a/sas/sasjs/db/datactrl/mpe_derived.ddl
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -24,6 +24,7 @@ create table &curlib..MPE_DERIVED
     VAR_TYPE varchar(1),
     VAR_LEN int,
     VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
     VAR_LOGIC varchar(4000),
     ACTIVE int,
     TX_TO float not null format=datetime19.,
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index 78517fa..929ba6a 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -27,6 +27,7 @@ create table &dclib..mpe_derived
     var_type varchar(1),
     var_len num,
     var_fmt varchar(32),
+    var_label varchar(256),
     var_logic varchar(4000),
     active num,
     tx_to num not null format=datetime19.
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index d1eeeae..4233518 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -34,8 +34,9 @@
       REGION = <expression>;
 
   Type and length are declared by the administrator, so the result is
-  deterministic.  The label is the variable name, an equals sign and the
-  expression.
+  deterministic.  VAR_LABEL is optional: when it is missing or blank the label
+  is the variable name, an equals sign and the expression, otherwise the
+  configured label is used as given.
 
   VAR_DEPS lists every input the expression uses - physical columns and
   derived variables alike.  The statements are generated in dependency order,
@@ -139,13 +140,14 @@ data work.&ds_cfg.;
     and active=1
     and upcase(libref)="%upcase(%scan(&libds,1,.))"
     and upcase(dsn)="%upcase(%scan(&libds,2,.))";
-  length name $32 logic $4000 fmt $32 deps $4000;
+  length name $32 logic $4000 fmt $32 deps $4000 lbl $256;
   name=upcase(strip(var_nm));
   logic=strip(var_logic);
   type=upcase(strip(var_type));
   len=var_len;
   fmt=strip(upcase(var_fmt));
   deps=upcase(strip(var_deps));
+  lbl=translate(strip(var_label),' ','0D0A'x);
 run;
 
 %if %mf_nobs(work.&ds_cfg.)=0 %then %return;
@@ -454,10 +456,13 @@ data _null_;
   set work.&ds_srt.;
   /* 512 so the doubled quotes fit before the 256 boundary is applied */
   length stmt $4600 label $512 fmt2 $49;
-  /* labels are limited to 256 chars.  The quotes are doubled first, then the
-    cut is made on an even quote boundary - truncating between the two halves of
-    an escape would leave the generated literal unterminated */
-  label=cats(name,'=',translate(logic,' ','0D0A'x));
+  /* the configured label wins when there is one, otherwise the label is the
+    name, an equals sign and the expression.  Labels are limited to 256 chars:
+    the quotes are doubled first, then the cut is made on an even quote
+    boundary - truncating between the two halves of an escape would leave the
+    generated literal unterminated */
+  if not missing(lbl) then label=lbl;
+  else label=cats(name,'=',translate(logic,' ','0D0A'x));
   label=tranwrd(label,"'","''");
   if length(label)>256 then do;
     label=substr(label,1,256);
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index f7d7141..5b67862 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -53,6 +53,7 @@ insert into &dclib..mpe_derived set
     ,var_type='C'
     ,var_len=8
     ,var_fmt=''
+    ,var_label='Age band'
     ,var_logic="ifc(AGE>13,'TEEN','JUNIOR')"
     ,active=1;
 insert into &dclib..mpe_derived set
@@ -140,7 +141,7 @@ proc contents noprint data=work.vw_derived
 run;
 data work.chk1d;
   set work.vw_cols;
-  where name='BAND' and label=:'BAND=' and format='$8.';
+  where name='BAND' and label='Age band' and format='$8.';
 run;
 %mp_assertdsobs(work.chk1d,
   desc=Test1 - the label and default format are applied,
diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas
index a8ba7ea..36dfa2e 100644
--- a/sas/sasjs/macros/mpe_makedatamodel.sas
+++ b/sas/sasjs/macros/mpe_makedatamodel.sas
@@ -704,6 +704,7 @@ create table &lib..MPE_DERIVED
     VAR_TYPE varchar(1),
     VAR_LEN num,
     VAR_FMT varchar(32),
+    VAR_LABEL varchar(256),
     VAR_LOGIC varchar(4000),
     ACTIVE num,
     TX_TO num &notnull format=datetime19.3
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 6dfb632..663210b 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -56,6 +56,7 @@ data work.staging_ds;
     var_nm=upcase(strip(var_nm));
     var_type=upcase(strip(var_type));
     var_fmt=upcase(strip(var_fmt));
+    var_label=strip(var_label);
     var_deps=upcase(strip(var_deps));
     var_logic=strip(var_logic);
   end;

From ace4a6670172632c37be3eeff41c2f5b019b2477 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 09:10:59 +0000
Subject: [PATCH 31/48] test(derived): snapshot the scope after the setup SQL

Same harness fix as the core test: the setup inserts create the SQL automatic
macro variables, so the snapshot has to come after them, and the framework's
_SASJS_WEBOUT_HEADERS is ignored.
---
 sas/sasjs/macros/mpe_derived.test.sas | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 5b67862..c844806 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -22,7 +22,6 @@
     4GL Apps Ltd.
 
 **/
-%mp_assertscope(SNAPSHOT)
 
 
 /* a source table, and a table with no derived variables */
@@ -83,6 +82,10 @@ insert into &dclib..mpe_derived set
 quit;
 
 /* lower case libds on purpose - the config match must be case insensitive */
+/* the setup SQL above creates the SQL automatic macro variables, so the
+  snapshot is taken here - the check is for the macro's own leakage */
+%mp_assertscope(SNAPSHOT)
+
 %mpe_derived(libds=work.class,
   dclib=&dclib,
   outref=derref1,
@@ -239,4 +242,4 @@ proc sql;
 delete from &dclib..mpe_derived where upcase(libref)='WORK';
 quit;
 
-%mp_assertscope(COMPARE, outds=work.test_results)
+%mp_assertscope(COMPARE,ignorelist=_SASJS_WEBOUT_HEADERS,outds=work.test_results)

From ee31782fbff909b540162b1bdfca6f307208331b Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 10:15:33 +0000
Subject: [PATCH 32/48] fix(derived): open the statements file with a long
 record length

A label at the 256 character limit produces an attrib statement of about 300
characters, which the default record length wraps, splitting the statement and
breaking the view.
---
 sas/sasjs/macros/mpe_derived.sas | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 4233518..c90d6d3 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -117,7 +117,9 @@
   ,msg=%str(Target table not found: &libds)
 )
 
-filename &outref temp;
+/* a long label can push a statement past the default record length, which
+  would wrap it and break the generated code */
+filename &outref temp lrecl=32767;
 
 /* ensure the outputs exist, and are empty */
 data _null_;

From 616a2928e78c6bfa010d3f38f31df766bd71ee57 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 11:27:30 +0000
Subject: [PATCH 33/48] refactor(derived): use mp_derivedvars from @sasjs/core

mpe_derived carried its own copy of the derived variable engine. That engine
now ships in @sasjs/core 5.7.0 as mp_derivedvars, so the local copy is gone and
mpe_derived is a wrapper that supplies what is Data Controller's own: the
control table, the bitemporal filter that picks the current active row, and the
list of metadata names that may not be used as a variable.

The macro goes from 511 lines to 102. The contract and the generated statement
format are unchanged, so the callers in the six services need no edit.

A test covers the wrapper's own responsibility: a closed version of a row and
an inactive row are both ignored.
---
 sas/package-lock.json                 |   8 +-
 sas/package.json                      |   2 +-
 sas/sasjs/macros/mpe_derived.sas      | 449 ++------------------------
 sas/sasjs/macros/mpe_derived.test.sas |  50 +++
 4 files changed, 75 insertions(+), 434 deletions(-)

diff --git a/sas/package-lock.json b/sas/package-lock.json
index 7539c9e..8411eef 100644
--- a/sas/package-lock.json
+++ b/sas/package-lock.json
@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "4.20.9",
-        "@sasjs/core": "5.6.0"
+        "@sasjs/core": "5.7.0"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -240,9 +240,9 @@
       "license": "MIT"
     },
     "node_modules/@sasjs/core": {
-      "version": "5.6.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.6.0.tgz",
-      "integrity": "sha512-c8YGglQkQKdTrpKN463a6t5bJsRTBkgX1W53oCYW8zZOJVFXVLPI2t+ORFfLerMYlrcYFn3/OG4JLK2MATK63Q==",
+      "version": "5.7.0",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.7.0.tgz",
+      "integrity": "sha512-vV4RTGieksQfd6fo1hKh8n58XgOCZY9vLmPmD/RdqwTyUbZKzi6GDme6aVpE+fcXxcpHf6Q1nPXrCalZZLPGfA==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
diff --git a/sas/package.json b/sas/package.json
index 6293d62..17caa14 100644
--- a/sas/package.json
+++ b/sas/package.json
@@ -31,7 +31,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "4.20.9",
-    "@sasjs/core": "5.6.0"
+    "@sasjs/core": "5.7.0"
   },
   "overrides": {
     "nanoid": "3.3.18"
diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index c90d6d3..69ca703 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -1,10 +1,15 @@
 /**
   @file
-  @brief Generates the SAS statements that create derived variables
+  @brief Generates the SAS statements that create Data Controller derived variables
   @details Derived (computed) variables are configured in MPE_DERIVED.  Each
   row defines one variable, which is created in a DATA step view ahead of the
   filter, so that it can be filtered on in the EDIT screen.
 
+  The engine lives in @sasjs/core as mp_derivedvars - this macro only supplies
+  what is specific to Data Controller: the control table name, the bitemporal
+  filter that picks the current version of a row, and the list of names that
+  belong to the metadata and may not be used as a variable.
+
   The macro writes the generated statements to a fileref, so that callers can
   compose them into a view:
 
@@ -43,21 +48,13 @@
   so a derived variable may build on another one without a sequence number
   being maintained.  A cycle is rejected.
 
-  The declaration is checked against the expression: a derived variable that
-  is referenced must be declared, a declared name that is not referenced is an
-  error, and a declared name must be a column of the target table or a derived
-  variable defined for it.
-
   Semicolons, macro triggers and comments are rejected, so a row can only ever
   be an expression.  A derived variable may not reuse the name of a physical
   column, and may not reference itself: an intra-row calculation would imply a
   retained value, which depends on the order rows happen to arrive in.
 
   <h4> SAS Macros </h4>
-  @li mf_existds.sas
-  @li mf_getuniquename.sas
-  @li mf_getvarlist.sas
-  @li mf_nobs.sas
+  @li mp_derivedvars.sas
   @li mp_abort.sas
 
   @param [in] libds= The target table (libref.dataset, lib must be assigned)
@@ -81,431 +78,25 @@
   outview=0
 );
 
-%local physlist derived_err resv_list i w left nready nplaced
-  cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
-  ds_cfg ds_byname ds_bad ds_baddep ds_dup ds_dep ds_ref ds_stale
-  ds_undecl ds_self ds_edge ds_todo ds_ready ds_placed ds_srt;
+%local reserved;
 
-/* every working table gets its own name, so a caller with tables of the same
-  name in WORK cannot collide with this macro's intermediates */
-%let ds_cfg=%mf_getuniquename(prefix=dvcfg);
-%let ds_byname=%mf_getuniquename(prefix=dvbyn);
-%let ds_bad=%mf_getuniquename(prefix=dvbad);
-%let ds_baddep=%mf_getuniquename(prefix=dvbdp);
-%let ds_dup=%mf_getuniquename(prefix=dvdup);
-%let ds_dep=%mf_getuniquename(prefix=dvdep);
-%let ds_ref=%mf_getuniquename(prefix=dvref);
-%let ds_stale=%mf_getuniquename(prefix=dvstl);
-%let ds_undecl=%mf_getuniquename(prefix=dvund);
-%let ds_self=%mf_getuniquename(prefix=dvsfl);
-%let ds_edge=%mf_getuniquename(prefix=dvedg);
-%let ds_todo=%mf_getuniquename(prefix=dvtd);
-%let ds_ready=%mf_getuniquename(prefix=dvrdy);
-%let ds_placed=%mf_getuniquename(prefix=dvpld);
-%let ds_srt=%mf_getuniquename(prefix=dvsrt);
+/* the metadata columns of every Data Controller table */
+%let reserved=%str(_____DELETE__THIS__RECORD_____ TX_FROM TX_TO PROCESSED_DTTM
+  VAR_TXFROM VAR_TXTO VAR_PROCESSED);
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
   ,msg=%str(Missing: libds dclib)
 )
-%mp_abort(iftrue= (&outref = 0)
-  ,mac=&sysmacroname
-  ,msg=%str(Please provide an outref for the generated statements)
+
+%mp_derivedvars(
+  libds=&libds
+  ,control=&dclib..mpe_derived
+  ,where=%str(&dc_dttmtfmt lt tx_to and active=1)
+  ,outref=&outref
+  ,outvars=&outvars
+  ,outview=&outview
+  ,reserved=&reserved
 )
-%mp_abort(iftrue= (%mf_existds(libds=&libds) ne 1)
-  ,mac=&sysmacroname
-  ,msg=%str(Target table not found: &libds)
-)
-
-/* a long label can push a statement past the default record length, which
-  would wrap it and break the generated code */
-filename &outref temp lrecl=32767;
-
-/* ensure the outputs exist, and are empty */
-data _null_;
-  file &outref;
-run;
-data &outvars(keep=name);
-  length name $32;
-  name='';
-  if 0;
-run;
-
-%if %mf_existds(libds=&dclib..mpe_derived) ne 1 %then %do;
-  %put &sysmacroname: &dclib..MPE_DERIVED not found - no derived variables;
-  %return;
-%end;
-
-data work.&ds_cfg.;
-  set &dclib..mpe_derived;
-  where &dc_dttmtfmt lt tx_to
-    and active=1
-    and upcase(libref)="%upcase(%scan(&libds,1,.))"
-    and upcase(dsn)="%upcase(%scan(&libds,2,.))";
-  length name $32 logic $4000 fmt $32 deps $4000 lbl $256;
-  name=upcase(strip(var_nm));
-  logic=strip(var_logic);
-  type=upcase(strip(var_type));
-  len=var_len;
-  fmt=strip(upcase(var_fmt));
-  deps=upcase(strip(var_deps));
-  lbl=translate(strip(var_label),' ','0D0A'x);
-run;
-
-%if %mf_nobs(work.&ds_cfg.)=0 %then %return;
-
-proc sort data=work.&ds_cfg. out=work.&ds_byname.;
-  by name;
-run;
-
-%let physlist=%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=SINGLE));
-
-/* names that may never be used - DC's own metadata columns */
-%let resv_list=%str("_____DELETE__THIS__RECORD_____","TX_FROM","TX_TO","PROCESSED_DTTM","VAR_TXFROM","VAR_TXTO","VAR_PROCESSED");
-
-/**
-  * Reject anything that could break the generated data step, or that would
-  * produce a variable we cannot describe to the caller
-  */
-data work.&ds_bad.;
-  set work.&ds_byname.;
-  length reason $200;
-  call missing(reason);
-
-  if name='' then reason='VAR_NM is required';
-  else if verify(strip(name),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')>0
-    then reason='VAR_NM must be a valid SAS name';
-  else if verify(substr(name,1,1),'0123456789')=0
-    then reason='VAR_NM must not start with a digit';
-  %if %length(&physlist)>0 %then %do;
-  else if name in (%unquote(&physlist))
-    then reason='VAR_NM collides with a column in the target table';
-  %end;
-  else if name in (%unquote(&resv_list))
-    then reason='VAR_NM is reserved';
-  else if type not in ('C','N')
-    then reason='VAR_TYPE must be C or N';
-  else if missing(len) or len ne int(len) or len<1
-    then reason='VAR_LEN must be a positive integer';
-  else if type='C' and len>32767
-    then reason='VAR_LEN exceeds 32767 for a character variable';
-  else if type='N' and len>8
-    then reason='VAR_LEN exceeds 8 for a numeric variable';
-  else if index(fmt,';')>0 or indexc(fmt,'&%')>0
-    or index(fmt,'/*')>0 or index(fmt,'*/')>0
-    then reason='VAR_FMT must be a format name, without ; & % or //*';
-  else if verify(strip(fmt),'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$.')>0
-    then reason='VAR_FMT must be a format name';
-  else if missing(logic)
-    then reason='VAR_LOGIC is required';
-  else if index(logic,';')>0
-    then reason='VAR_LOGIC must be a single expression - remove the ;';
-  else if indexc(logic,'&%')>0
-    then reason='VAR_LOGIC must not contain macro triggers';
-  else if index(logic,'/*')>0 or index(logic,'*/')>0
-    then reason='VAR_LOGIC must not contain comments';
-
-  if missing(reason)=0 then output;
-run;
-
-data work.&ds_dup.;
-  set work.&ds_byname.;
-  by name;
-  length reason $200;
-  if first.name and not last.name;
-  reason='VAR_NM is defined more than once';
-run;
-
-proc append base=work.&ds_bad. data=work.&ds_dup.;
-run;
-
-/**
-  * The declared dependencies, one row per word
-  */
-data work.&ds_dep.;
-  set work.&ds_byname.;
-  length dep_nm $32;
-  do i=1 to countw(deps,' ');
-    dep_nm=scan(deps,i,' ');
-    if not missing(dep_nm) then output;
-  end;
-  keep name dep_nm;
-run;
-
-/**
-  * The identifiers the expression actually references - quoted strings are
-  * removed first, so a name inside a literal is not mistaken for a reference
-  */
-data work.&ds_ref.;
-  set work.&ds_byname.;
-  length l $4000 tok $32;
-  l=prxchange("s/'([^']|'')*'//",-1,logic);
-  l=prxchange('s/"([^"]|"")*"//',-1,l);
-  l=prxchange('s/[^A-Za-z0-9_]/ /',-1,upcase(l));
-  do i=1 to countw(l,' ');
-    tok=scan(l,i,' ');
-    if not missing(tok) then output;
-  end;
-  keep name tok;
-run;
-
-/* a declared name must resolve - to a column of the target, or to a variable
-  in this configuration */
-proc sql noprint;
-create table work.&ds_baddep. as
-  select d.name, d.dep_nm
-  from work.&ds_dep. d
-  where d.dep_nm not in (select name from work.&ds_byname.)
-  /* a table with no columns leaves physlist empty - fall back to the derived
-    names alone rather than compiling an empty in-list */
-  %if %length(&physlist)>0 %then %do;
-    and d.dep_nm not in (%unquote(&physlist))
-  %end;
-  ;
-
-/* every declared name must actually appear in the expression */
-create table work.&ds_stale. as
-  select d.name, d.dep_nm
-  from work.&ds_dep. d
-  where not exists (
-    select 1 from work.&ds_ref. r
-    where r.name=d.name and r.tok=d.dep_nm
-  );
-
-/* a derived variable referenced by the expression must be declared, otherwise
-  the statement order would be wrong and the value would be silently missing */
-create table work.&ds_undecl. as
-  select distinct r.name, r.tok as dep_nm
-  from work.&ds_ref. r
-  where r.tok in (select name from work.&ds_byname.)
-    and r.tok ne r.name
-    and not exists (
-      select 1 from work.&ds_dep. d
-      where d.name=r.name and d.dep_nm=r.tok
-    );
-
-/* a variable may not depend on itself */
-create table work.&ds_self. as
-  select name, dep_nm from work.&ds_dep.
-  where name=dep_nm;
-quit;
-
-%if %mf_nobs(work.&ds_baddep.)>0 %then %do;
-  data _null_;
-    set work.&ds_baddep.(obs=1);
-    call symputx('derived_err'
-      ,cats(name,': VAR_DEPS lists ',dep_nm
-        ,', which is not a column of the table or a derived variable'));
-  run;
-  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
-%end;
-
-%if %mf_nobs(work.&ds_self.)>0 %then %do;
-  data _null_;
-    set work.&ds_self.(obs=1);
-    call symputx('derived_err',cats(name,': VAR_DEPS may not list itself'));
-  run;
-  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
-%end;
-
-%if %mf_nobs(work.&ds_undecl.)>0 %then %do;
-  data _null_;
-    set work.&ds_undecl.(obs=1);
-    call symputx('derived_err'
-      ,cats(name,': VAR_LOGIC references ',dep_nm
-        ,', which is not in VAR_DEPS'));
-  run;
-  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
-%end;
-
-%if %mf_nobs(work.&ds_stale.)>0 %then %do;
-  data _null_;
-    set work.&ds_stale.(obs=1);
-    call symputx('derived_err'
-      ,cats(name,': VAR_DEPS lists ',dep_nm
-        ,', which VAR_LOGIC does not reference'));
-  run;
-  %mp_abort(iftrue=1,mac=&sysmacroname,msg=%superq(derived_err))
-%end;
-
-%if %mf_nobs(work.&ds_bad.)>0 %then %do;
-  data _null_;
-    set work.&ds_bad.;
-    putlog 'ERR' 'OR: derived variable config: ' name= reason=;
-  run;
-  data _null_;
-    set work.&ds_bad.(obs=1);
-    call symputx('derived_err',cats(name,': ',reason));
-  run;
-  %mp_abort(iftrue=1
-    ,mac=&sysmacroname
-    ,msg=%str(Invalid derived variable configuration - &derived_err)
-  )
-%end;
-
-/**
-  * Dependency order.  The edges are between derived variables only - a
-  * physical column is available from the start.  Repeatedly take the variables
-  * whose derived inputs have all been placed, alphabetically within a round;
-  * if a round places nothing, what is left is a cycle.
-  */
-proc sql;
-create table work.&ds_edge. as
-  select d.name as child, d.dep_nm as parent
-  from work.&ds_dep. d
-  where d.dep_nm in (select name from work.&ds_byname.);
-quit;
-
-data work.&ds_todo.(keep=name);
-  set work.&ds_byname.;
-run;
-
-data work.&ds_placed.(keep=name seq);
-  length name $32;
-  name='';
-  seq=0;
-  if 0;
-run;
-
-%let left=%mf_nobs(work.&ds_todo.);
-%let nplaced=0;
-
-%do %while (&left>0);
-  proc sql;
-  create table work.&ds_ready. as
-    select t.name
-    from work.&ds_todo. t
-    where not exists (
-      select 1 from work.&ds_edge. e
-      inner join work.&ds_todo. t2 on e.parent=t2.name
-      where e.child=t.name
-    )
-    order by t.name;
-  quit;
-
-  %let nready=%mf_nobs(work.&ds_ready.);
-  %if &nready=0 %then %do;
-    /* walk the edges, so the message names the cycle rather than just the
-      names that could not be placed */
-    data _null_;
-      set work.&ds_todo.(obs=1);
-      call symputx('cyc_start',name);
-    run;
-    %let cyc_path=&cyc_start;
-    %let cyc_cur=&cyc_start;
-    %let cyc_seen=%str(.&cyc_start..);
-    %do cyc_i=1 %to &left;
-      %let cyc_next=;
-      proc sql noprint;
-        select e.parent into: cyc_next
-        from work.&ds_edge. e
-        where e.child="&cyc_cur"
-          and e.parent in (select name from work.&ds_todo.);
-      quit;
-      %if %length(&cyc_next)=0 %then %let cyc_i=&left;
-      %else %do;
-        %let cyc_path=&cyc_path. -> &cyc_next;
-        %if %index(&cyc_seen,%str(.&cyc_next..))>0 %then %let cyc_i=&left;
-        %else %do;
-          %let cyc_seen=&cyc_seen.&cyc_next..;
-          %let cyc_cur=&cyc_next;
-        %end;
-      %end;
-    %end;
-    %mp_abort(iftrue=1
-      ,mac=&sysmacroname
-      ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
-    )
-  %end;
-
-  /* seq carries the dependency order - the placement rounds are not name
-    ordered, so the final merge must not rely on by name alone */
-  data work.&ds_ready.;
-    set work.&ds_ready.;
-    seq=_n_+&nplaced;
-  run;
-  proc append base=work.&ds_placed. data=work.&ds_ready. force;
-  run;
-  %let nplaced=%eval(&nplaced+&nready);
-
-  proc sql;
-  delete from work.&ds_todo.
-    where name in (select name from work.&ds_ready.);
-  quit;
-
-  %let left=%mf_nobs(work.&ds_todo.);
-%end;
-
-proc sort data=work.&ds_placed.;
-  by name;
-run;
-
-data work.&ds_srt.;
-  merge work.&ds_placed.(in=in1) work.&ds_byname.(in=in2);
-  by name;
-  if in1 and in2;
-run;
-
-proc sort data=work.&ds_srt.;
-  by seq;
-run;
-
-/**
-  * Generate the attrib + assignment statements
-  */
-data _null_;
-  file &outref;
-  set work.&ds_srt.;
-  /* 512 so the doubled quotes fit before the 256 boundary is applied */
-  length stmt $4600 label $512 fmt2 $49;
-  /* the configured label wins when there is one, otherwise the label is the
-    name, an equals sign and the expression.  Labels are limited to 256 chars:
-    the quotes are doubled first, then the cut is made on an even quote
-    boundary - truncating between the two halves of an escape would leave the
-    generated literal unterminated */
-  if not missing(lbl) then label=lbl;
-  else label=cats(name,'=',translate(logic,' ','0D0A'x));
-  label=tranwrd(label,"'","''");
-  if length(label)>256 then do;
-    label=substr(label,1,256);
-    if mod(count(label,"'"),2) ne 0 then label=substr(label,1,255);
-  end;
-  if not missing(fmt) and index(fmt,'.')=0 then fmt=cats(fmt,'.');
-  if type='C' then do;
-    if missing(fmt) then fmt2=cats('$',len,'.');
-    else fmt2=fmt;
-    stmt=catx(' ','attrib',name,cats('length=$',len)
-      ,cats("label='",label,"'"),cats('format=',fmt2,';'));
-  end;
-  else do;
-    if missing(fmt) then fmt2=cats(len,'.');
-    else fmt2=fmt;
-    stmt=catx(' ','attrib',name,cats('length=',len)
-      ,cats("label='",label,"'"),cats('format=',fmt2,';'));
-  end;
-  put stmt;
-  stmt=cats(name,' = ',logic,';');
-  put stmt;
-run;
-
-%mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
-  ,msg=%str(Unable to generate derived variables for &libds)
-)
-
-data &outvars(keep=name);
-  set work.&ds_srt.;
-run;
-
-%if %length(&outview)>0 and &outview ne 0 %then %do;
-  data &outview / view=&outview;
-    set &libds;
-    %include &outref;
-  run;
-  %mp_abort(iftrue= (&syscc ne 0)
-    ,mac=&sysmacroname
-    ,msg=%str(Unable to build derived view &outview for &libds)
-  )
-%end;
 
 %mend mpe_derived;
diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index c844806..1c6624e 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -237,6 +237,56 @@ run;
   outds=work.test_results
 )
 
+/**
+  * Test 4 - the wrapper scopes the config: only the current, active row of a
+  * table is used
+  */
+
+proc sql;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='01JAN2020:00:00:00'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='CLOSED_OFF'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 1'
+    ,active=1;
+insert into &dclib..mpe_derived set
+    tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='WORK'
+    ,dsn='CLASS'
+    ,var_nm='NOT_ACTIVE'
+    ,var_deps='AGE'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='AGE + 2'
+    ,active=0;
+quit;
+
+%mpe_derived(libds=WORK.CLASS,
+  dclib=&dclib,
+  outref=derref4,
+  outview=work.vw_derived4,
+  outvars=work.derived_vars4
+)
+
+data work.chk4;
+  set work.derived_vars4;
+  where name in ('CLOSED_OFF','NOT_ACTIVE');
+run;
+
+%mp_assertdsobs(work.chk4,
+  desc=Test4 - a closed version and an inactive row are ignored,
+  test=EQUALS 0,
+  outds=work.test_results
+)
+
 /* tidy up so repeated runs start clean */
 proc sql;
 delete from &dclib..mpe_derived where upcase(libref)='WORK';

From 9a047ad42ec86df7d1eaf5058c0edde56016e88f Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 12:06:35 +0000
Subject: [PATCH 34/48] fix(derived): repair the validation rows for
 MPE_DERIVED

Four stray insert lines and a missing one left the MPE_VALIDATIONS block for
MPE_DERIVED malformed, so makedata aborted and the control table was never
created - found by building the library on a real estate.

The format assertions in the test now read the emitted statement, because proc
contents on a view does not report a format in full ('DATE' for DATE9.).
---
 sas/sasjs/macros/mpe_derived.test.sas | 15 ++++++++-------
 sas/sasjs/macros/mpe_makedata.sas     |  3 ---
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.test.sas b/sas/sasjs/macros/mpe_derived.test.sas
index 1c6624e..341453d 100644
--- a/sas/sasjs/macros/mpe_derived.test.sas
+++ b/sas/sasjs/macros/mpe_derived.test.sas
@@ -138,13 +138,14 @@ run;
   outds=work.test_results
 )
 
-/* the label is the variable name plus the expression, and the format from the length */
+/* proc contents on a view reports a format name without its width, so the
+  format is compared on the name only */
 proc contents noprint data=work.vw_derived
   out=work.vw_cols(keep=name label format);
 run;
 data work.chk1d;
   set work.vw_cols;
-  where name='BAND' and label='Age band' and format='$8.';
+  where name='BAND' and label='Age band' and format=:'$';
 run;
 %mp_assertdsobs(work.chk1d,
   desc=Test1 - the label and default format are applied,
@@ -178,12 +179,12 @@ quit;
   outvars=work.derived_vars2
 )
 
-proc contents noprint data=work.vw_derived2
-  out=work.vw_cols2(keep=name label format);
-run;
+/* asserted on the emitted statement - proc contents on a view does not
+  report the format in full */
 data work.chk2;
-  set work.vw_cols2;
-  where name='FLAG_DT' and format='DATE9.';
+  infile derref2 truncover;
+  input line $4000.;
+  if index(line,'format=DATE9.')>0 then output;
 run;
 %mp_assertdsobs(work.chk2,
   desc=Test2 - the configured format is applied,
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index 17279f0..2d8b824 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -2015,9 +2015,6 @@ insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
-insert into &lib..MPE_VALIDATIONS set
-insert into &lib..MPE_VALIDATIONS set
-insert into &lib..MPE_VALIDATIONS set
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1

From 561d7fc3ed2a460835619962cf4dcb29d3dc87c7 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 12:54:34 +0000
Subject: [PATCH 35/48] fix(derived): eight review findings

- mpe_derived_postedit ran %local at the top level of a file that is included
  at execution time, so the hook stopped before validating anything and every
  MPE_DERIVED submission failed. The macro language now lives in a macro
  definition, errflag and errmsg are declared global, and the file calls it.
- the cycle check aborts after leaving its loop, and the self, undeclared and
  field aborts are driven by a count at the point of call rather than by a
  wrapping %if, so an abort cannot be outlived. The same shapes are fixed
  upstream in sasjs/core#468.
- a case-variant duplicate survived mpe_loader, which compares raw values:
  the hook now rejects it after normalisation and names the variable.
- stagedata dropped the derived columns from the output of the security step
  rather than from its input, so a crafted submission could still satisfy an
  RLS clause through a derived column. work.jsdata itself is rewritten now.
- the VAR_FMT validation rule rejected the bare w.d forms that the hook and
  the engine accept.
- the migration carried the same four dangling insert headers as mpe_makedata,
  so the v7.19 migration failed for existing installs.
- the filter fixtures in getdata.test and validatefilter.test had been caught
  by the variable rename: the filter schema column is VARIABLE_NM.
- the TX_FROM/TX_TO definitions agree across the ddl, the migration and
  mpe_makedatamodel.

@sasjs/core bumped to 5.7.1, which carries the where= fix.
---
 sas/package-lock.json                         |   8 +-
 sas/package.json                              |   2 +-
 sas/sasjs/db/datactrl/mpe_derived.ddl         |   4 +-
 .../db/migrations/20261019_v7.19_release.sas  |   9 +-
 sas/sasjs/macros/mpe_makedata.sas             |   2 +-
 sas/sasjs/services/editors/getdata.test.sas   |  12 +-
 sas/sasjs/services/editors/stagedata.sas      |  13 +-
 .../services/hooks/mpe_derived_postedit.sas   | 118 ++++++++++++------
 .../services/public/validatefilter.test.sas   |   6 +-
 9 files changed, 112 insertions(+), 62 deletions(-)

diff --git a/sas/package-lock.json b/sas/package-lock.json
index 8411eef..8b25d14 100644
--- a/sas/package-lock.json
+++ b/sas/package-lock.json
@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "4.20.9",
-        "@sasjs/core": "5.7.0"
+        "@sasjs/core": "5.7.1"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -240,9 +240,9 @@
       "license": "MIT"
     },
     "node_modules/@sasjs/core": {
-      "version": "5.7.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.7.0.tgz",
-      "integrity": "sha512-vV4RTGieksQfd6fo1hKh8n58XgOCZY9vLmPmD/RdqwTyUbZKzi6GDme6aVpE+fcXxcpHf6Q1nPXrCalZZLPGfA==",
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.7.1.tgz",
+      "integrity": "sha512-IIVQc02u6Hy6k28y0IELKEMHYDC4e+Y4n/Fn38m/zQVwtVccDFQ6Ct3QvgRZUfAB0Fa0ofW0MoOYV3ox/i86iA==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
diff --git a/sas/package.json b/sas/package.json
index 17caa14..c2786bf 100644
--- a/sas/package.json
+++ b/sas/package.json
@@ -31,7 +31,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "4.20.9",
-    "@sasjs/core": "5.7.0"
+    "@sasjs/core": "5.7.1"
   },
   "overrides": {
     "nanoid": "3.3.18"
diff --git a/sas/sasjs/db/datactrl/mpe_derived.ddl b/sas/sasjs/db/datactrl/mpe_derived.ddl
index a0c92e8..3334af0 100644
--- a/sas/sasjs/db/datactrl/mpe_derived.ddl
+++ b/sas/sasjs/db/datactrl/mpe_derived.ddl
@@ -16,7 +16,7 @@
 
 create table &curlib..MPE_DERIVED
    (
-    TX_FROM float not null format=datetime19.,
+    TX_FROM num not null format=datetime19.3,
     LIBREF varchar(8),
     DSN varchar(32),
     VAR_NM varchar(32),
@@ -27,6 +27,6 @@ create table &curlib..MPE_DERIVED
     VAR_LABEL varchar(256),
     VAR_LOGIC varchar(4000),
     ACTIVE int,
-    TX_TO float not null format=datetime19.,
+    TX_TO num not null format=datetime19.3,
     constraint pk_mpe_derived
         primary key(tx_from, libref, dsn, var_nm));
diff --git a/sas/sasjs/db/migrations/20261019_v7.19_release.sas b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
index 929ba6a..30bcfbe 100644
--- a/sas/sasjs/db/migrations/20261019_v7.19_release.sas
+++ b/sas/sasjs/db/migrations/20261019_v7.19_release.sas
@@ -19,7 +19,7 @@ libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
 proc sql;
 create table &dclib..mpe_derived
   (
-    tx_from num not null format=datetime19.,
+    tx_from num not null format=datetime19.3,
     libref varchar(8),
     dsn varchar(32),
     var_nm varchar(32),
@@ -30,7 +30,7 @@ create table &dclib..mpe_derived
     var_label varchar(256),
     var_logic varchar(4000),
     active num,
-    tx_to num not null format=datetime19.
+    tx_to num not null format=datetime19.3
   );
 quit;
 
@@ -125,9 +125,6 @@ insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_DEPS"
     ,rule_type='HARDREGEX',rule_value='^[A-Za-z0-9_ ]*$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
-insert into &dclib..MPE_VALIDATIONS set
-insert into &dclib..MPE_VALIDATIONS set
-insert into &dclib..MPE_VALIDATIONS set
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_TYPE"
     ,rule_type='NOTNULL',rule_value=' ',rule_active=1
@@ -159,7 +156,7 @@ insert into &dclib..MPE_VALIDATIONS set
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
     ,rule_type='HARDREGEX'
-    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?)?$',rule_active=1
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &dclib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&dclib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
diff --git a/sas/sasjs/macros/mpe_makedata.sas b/sas/sasjs/macros/mpe_makedata.sas
index 2d8b824..cac2552 100644
--- a/sas/sasjs/macros/mpe_makedata.sas
+++ b/sas/sasjs/macros/mpe_makedata.sas
@@ -2046,7 +2046,7 @@ insert into &lib..MPE_VALIDATIONS set
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_FMT"
     ,rule_type='HARDREGEX'
-    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?)?$',rule_active=1
+    ,rule_value='^(\$?[A-Za-z][A-Za-z0-9_]*(\.[0-9]+)?\.?|\$?[0-9]+(\.[0-9]+)?\.?)?$',rule_active=1
     ,tx_to='31DEC5999:23:59:59'dt;
 insert into &lib..MPE_VALIDATIONS set
     tx_from=0,base_lib="&lib",base_ds="MPE_DERIVED",base_col="VAR_LOGIC"
diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index b0a0519..e17502e 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -26,11 +26,11 @@
 
 data work.inquery1;
   length GROUP_LOGIC $3 SUBGROUP_LOGIC $3 SUBGROUP_ID 8
-    VAR_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
+    VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;
   GROUP_LOGIC='AND';
   SUBGROUP_LOGIC='AND';
   SUBGROUP_ID=1;
-  VAR_NM='LIBREF';
+  VARIABLE_NM='LIBREF';
   OPERATOR_NM='=';
   RAW_VALUE="'&dclib'";
   output;
@@ -90,7 +90,7 @@ run;
 
 %mp_assertcols(work.query,
   desc=Query dataset carries the filter columns,
-  cols=VAR_NM RAW_VALUE SUBGROUP_ID,
+  cols=VARIABLE_NM RAW_VALUE SUBGROUP_ID,
   test=ALL,
   outds=work.test_results
 )
@@ -108,7 +108,7 @@ run;
 /* first, make filter */
 data work.inquery;
   infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32.
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,LIBREF,CONTAINS,"'DC'"
@@ -188,7 +188,7 @@ run;
 /* now, make filter */
 data work.inquery3;
   infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32.
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
   RAW_VALUE="'&fmtname'";
 datalines4;
@@ -288,7 +288,7 @@ quit;
 
 data work.inquery4;
   infile datalines4 dsd;
-  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32.
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
     OPERATOR_NM:$10. RAW_VALUE:$4000.;
 datalines4;
 AND,AND,1,DERIVED_NOTE,CONTAINS,"'MPE_LOCK'"
diff --git a/sas/sasjs/services/editors/stagedata.sas b/sas/sasjs/services/editors/stagedata.sas
index 0b9edb6..534db47 100755
--- a/sas/sasjs/services/editors/stagedata.sas
+++ b/sas/sasjs/services/editors/stagedata.sas
@@ -128,6 +128,16 @@ run;
   quit;
 %end;
 
+/* drop them from the submitted data itself - a DROP on the output of a later
+  step would leave the columns in that step's input, and the security clause
+  is evaluated against the input */
+%if %length(&derived_droplist)>0 %then %do;
+  data work.jsdata;
+    set work.jsdata;
+    drop &derived_droplist;
+  run;
+%end;
+
 %mpe_filtermaster(ULOAD,&libds,
   dclib=&mpelib,
   outref=filtref,
@@ -152,9 +162,6 @@ run;
 /* apply the query */
 data work.badrecords;
   set work.jsdata;
-  %if %length(&derived_droplist)>0 %then %do;
-    drop &derived_droplist;
-  %end;
   %inc &tempref/source2;;
   putlog (_all_)(=);
 run;
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 663210b..1aa36d9 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -25,8 +25,11 @@
   The target table is not assigned in this context, so the collision between
   VAR_NM and a physical column, and the resolution of a physical dependency,
   are left to %mpe_derived, which runs with the target library assigned.
-  Uniqueness on the business key (LIBREF DSN VAR_NM) is already enforced by
-  mpe_loader, so it is not repeated here.
+  Uniqueness on the business key (LIBREF DSN VAR_NM) is checked by mpe_loader
+  before this hook runs, but on the raw submitted values, so it is case
+  sensitive: 'Region' and 'REGION' for the same table both pass the loader and
+  collide only at approve time.  The post-normalisation check below rejects
+  the second one at submit time, naming the variable.
 
   <h4> SAS Macros </h4>
   @li mf_nobs.sas
@@ -45,7 +48,13 @@
 
 %let errflag=0;
 %let errmsg=;
-%local left nready cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i cyc_n;
+
+/* a hook is included at execution time, so this is open code - the macro
+  language has to be inside a macro definition for %local to be valid */
+%macro _mpe_derived_postedit();
+  %global errflag errmsg;
+  %local left nready cyc_start cyc_cur cyc_next cyc_path cyc_seen cyc_i
+    cyc_n cyc_found;
 
 /* normalise the incoming values */
 data work.staging_ds;
@@ -62,6 +71,31 @@ data work.staging_ds;
   end;
 run;
 
+/**
+  * Duplicate names after normalisation - mpe_loader compares the raw values,
+  * so a case variant reaches us here
+  */
+proc sql;
+create table work.derived_dup as
+  select libref, dsn, var_nm, count(*) as n
+  from work.staging_ds
+  where upcase(_____DELETE__THIS__RECORD_____) ne 'YES'
+  group by libref, dsn, var_nm
+  having count(*)>1;
+quit;
+
+data _null_;
+  set work.derived_dup(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',var_nm,' is declared more than once for '
+      ,libref,'.',dsn));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_dup))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
 /**
   * Field level checks - the same rules %mpe_derived applies at read time
   */
@@ -186,30 +220,30 @@ create table work.derived_undeclared as
     );
 quit;
 
-%if %mf_nobs(work.derived_self)>0 %then %do;
-  data _null_;
-    set work.derived_self(obs=1);
-    call symputx('errmsg'
-      ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
-  run;
-  %mp_abort(iftrue=1
-    ,mac=mpe_derived_postedit
-    ,msg=%superq(errmsg)
-  )
-%end;
+/* the message is always prepared, the abort is driven by the count - an
+  iftrue=1 inside a wrapping %if leaves the step alive on some engines */
+data _null_;
+  set work.derived_self(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS may not list itself'));
+run;
 
-%if %mf_nobs(work.derived_undeclared)>0 %then %do;
-  data _null_;
-    set work.derived_undeclared(obs=1);
-    call symputx('errmsg'
-      ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
-        ,', which is not in VAR_DEPS'));
-  run;
-  %mp_abort(iftrue=1
-    ,mac=mpe_derived_postedit
-    ,msg=%superq(errmsg)
-  )
-%end;
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_self))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
+data _null_;
+  set work.derived_undeclared(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_LOGIC references ',dep_nm
+      ,', which is not in VAR_DEPS'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_undeclared))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
 
 /**
   * Cycle check - repeatedly take the variables whose derived inputs have all
@@ -227,6 +261,7 @@ data work.derived_todo;
 run;
 
 %let left=%mf_nobs(work.derived_todo);
+%let cyc_found=0;
 
 %do %while (&left>0);
   proc sql;
@@ -272,17 +307,28 @@ run;
         %end;
       %end;
     %end;
-    %mp_abort(iftrue=1
-      ,mac=mpe_derived_postedit
-      ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
-    )
+    /* leave the loop before aborting - an abort that only cancels the step
+      would re-enter it with the ready count still zero */
+    %let cyc_found=1;
+    %let left=0;
   %end;
 
-  proc sql;
-  delete from work.derived_todo
-    where catx('.',libref,dsn,name) in
-      (select catx('.',libref,dsn,name) from work.derived_ready);
-  quit;
+  %if &cyc_found=0 %then %do;
+    proc sql;
+    delete from work.derived_todo
+      where catx('.',libref,dsn,name) in
+        (select catx('.',libref,dsn,name) from work.derived_ready);
+    quit;
 
-  %let left=%mf_nobs(work.derived_todo);
+    %let left=%mf_nobs(work.derived_todo);
+  %end;
 %end;
+
+%mp_abort(iftrue= (&cyc_found=1)
+  ,mac=mpe_derived_postedit
+  ,msg=%str(VAR_DEPS contains a cycle: &cyc_path)
+)
+
+%mend _mpe_derived_postedit;
+
+%_mpe_derived_postedit()
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index f2e9558..9ab1bfb 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -26,7 +26,7 @@ data _null_;
   input;
   put _infile_;
 datalines4;
-GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VAR_NM:$32. OPERATOR_NM:$10. RAW_VALUE:$4000.
+GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. OPERATOR_NM:$10. RAW_VALUE:$4000.
 AND,AND,1,LIBREF,CONTAINS,"'DC'"
 AND,OR,2,DSN,=,"'MPE_LOCK_ANYTABLE'"
 ;;;;
@@ -53,7 +53,7 @@ data work.filterquery;
 GROUP_LOGIC='AND';
 SUBGROUP_LOGIC='AND';
 SUBGROUP_ID=1;
-VAR_NM='FMTNAME';
+VARIABLE_NM='FMTNAME';
 OPERATOR_NM='CONTAINS';
 RAW_VALUE="'MORD'";
 ;;;;
@@ -107,7 +107,7 @@ data work.filterquery;
   GROUP_LOGIC='AND';
   SUBGROUP_LOGIC='AND';
   SUBGROUP_ID=1;
-  VAR_NM='DERIVED_NOTE';
+  VARIABLE_NM='DERIVED_NOTE';
   OPERATOR_NM='CONTAINS';
   RAW_VALUE="'MPE_LOCK'";
   output;

From 4c198c6e318005cdbe3bba8d44ca29a21d1ce768 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 13:04:31 +0000
Subject: [PATCH 36/48] chore(deps): @sasjs/core 5.7.2

Carries the abort-shape fix from sasjs/core#468 - the cycle abort no longer
re-enters its placement loop, and the validation aborts are driven by a count
at the point of call.
---
 sas/package-lock.json | 8 ++++----
 sas/package.json      | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sas/package-lock.json b/sas/package-lock.json
index 8b25d14..0f96003 100644
--- a/sas/package-lock.json
+++ b/sas/package-lock.json
@@ -7,7 +7,7 @@
       "name": "dc-sas",
       "dependencies": {
         "@sasjs/cli": "4.20.9",
-        "@sasjs/core": "5.7.1"
+        "@sasjs/core": "5.7.2"
       }
     },
     "node_modules/@asamuzakjp/css-color": {
@@ -240,9 +240,9 @@
       "license": "MIT"
     },
     "node_modules/@sasjs/core": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.7.1.tgz",
-      "integrity": "sha512-IIVQc02u6Hy6k28y0IELKEMHYDC4e+Y4n/Fn38m/zQVwtVccDFQ6Ct3QvgRZUfAB0Fa0ofW0MoOYV3ox/i86iA==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-5.7.2.tgz",
+      "integrity": "sha512-1cjzI5Gg6vLTDX6jqZFQbGjbscXtZRa9pR6Ck+29cVa9VW3Av1FE/tQD4grxXrhEHMv/VJlBHq/odDJeaLtRHg==",
       "license": "MIT"
     },
     "node_modules/@sasjs/lint": {
diff --git a/sas/package.json b/sas/package.json
index c2786bf..9bc4662 100644
--- a/sas/package.json
+++ b/sas/package.json
@@ -31,7 +31,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/cli": "4.20.9",
-    "@sasjs/core": "5.7.1"
+    "@sasjs/core": "5.7.2"
   },
   "overrides": {
     "nanoid": "3.3.18"

From 88a38db877dba9b98ccf50f0cc7242674654586b Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 13:34:02 +0000
Subject: [PATCH 37/48] fix(derived): pair the filterds %else with %do/%end

Both services selected the filter target with
'%if cond %then %let x=y; %else %let x=z;'. In open code the %else is not
paired with the %then and the whole step stops with 'Expected %DO not found',
so neither service ever applied a filter: every assertion downstream failed and
the job produced no results at all.

Found on a real estate - the pattern works inside a macro definition, which is
why the same construct elsewhere in the codebase is fine.
---
 sas/sasjs/services/public/getcolvals.sas     | 10 ++++++++--
 sas/sasjs/services/public/validatefilter.sas | 10 ++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/sas/sasjs/services/public/getcolvals.sas b/sas/sasjs/services/public/getcolvals.sas
index 01d64e8..c251162 100644
--- a/sas/sasjs/services/public/getcolvals.sas
+++ b/sas/sasjs/services/public/getcolvals.sas
@@ -163,8 +163,14 @@ run;
   outvars=work.derived_vars
 )
 %let derived_nobs=%mf_nobs(work.derived_vars);
-%if &derived_nobs>0 %then %let filterds=work.vw_derived;
-%else %let filterds=&libds;
+/* %do/%end blocks, not '%then %let ...; %else %let ...;' - in open code the
+  %else is not paired and SAS stops with 'Expected %DO not found' */
+%if &derived_nobs>0 %then %do;
+  %let filterds=work.vw_derived;
+%end;
+%else %do;
+  %let filterds=&libds;
+%end;
 
 /**
   * Validate the filter query
diff --git a/sas/sasjs/services/public/validatefilter.sas b/sas/sasjs/services/public/validatefilter.sas
index 2085b64..e17dfcd 100644
--- a/sas/sasjs/services/public/validatefilter.sas
+++ b/sas/sasjs/services/public/validatefilter.sas
@@ -79,8 +79,14 @@ run;
   outvars=work.derived_vars
 )
 %let derived_nobs=%mf_nobs(work.derived_vars);
-%if &derived_nobs>0 %then %let filterds=work.vw_derived;
-%else %let filterds=&ds;
+/* %do/%end blocks, not '%then %let ...; %else %let ...;' - in open code the
+  %else is not paired and SAS stops with 'Expected %DO not found' */
+%if &derived_nobs>0 %then %do;
+  %let filterds=work.vw_derived;
+%end;
+%else %do;
+  %let filterds=&ds;
+%end;
 
 %mp_filterstore(
   libds=&ds,

From ccd468b3358c1418ee6d8e198b088a0cf23b8c24 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 14:02:18 +0000
Subject: [PATCH 38/48] fix(derived): address five review findings

- getrawdata selected its filter target with a bare '%else %let' in open code,
  the shape that stops the step with 'Expected %DO not found'. Both branches
  are %do/%end now, so the download service applies a filter at all.
- the conditional drop in the approve.jsdset step and the one against
  MPSEARCH.vwsearch are dead: work.jsdata and work.vwsearch are already
  rewritten without the derived columns, so the drops only warn.
- the post-edit hook checked that a reference is declared, but not that a
  declared dependency is referenced. The engine rejects the stale shape at read
  time, so the same check now rejects it at submit time.
- a row level security rule naming a derived variable is rejected when the rule
  is saved, with a message that says why, instead of failing later at submit.
- getdata.test's cleanup deleted only DERIVED_NOTE, leaving DERIVED_NOTE2 with
  a dependency on a variable that no longer exists, which aborted every call in
  the following run - the cause of the repeated no-result runs.
---
 sas/sasjs/services/editors/getdata.test.sas   |  8 +++--
 sas/sasjs/services/editors/stagedata.sas      |  6 ++--
 .../services/hooks/mpe_derived_postedit.sas   | 25 ++++++++++++++++
 .../hooks/mpe_row_level_security_postedit.sas | 29 +++++++++++++++++++
 sas/sasjs/services/public/getrawdata.sas      |  4 ++-
 sas/sasjs/services/public/viewdata.sas        |  5 ++--
 6 files changed, 66 insertions(+), 11 deletions(-)

diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index e17502e..a979dbf 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -385,10 +385,12 @@ run;
   outds=work.test_results
 )
 
-/* tidy up the config so repeated runs start clean */
+/* tidy up the config so repeated runs start clean.  Delete every row for the
+  table, not just the first variable: leaving DERIVED_NOTE2 behind means the
+  next run starts with a dependency on a variable that is no longer declared,
+  and every call aborts */
 proc sql;
 delete from &dc_libref..mpe_derived
-  where var_nm='DERIVED_NOTE'
-    and upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
+  where upcase(libref)="%scan(&dclib..MPE_TABLES,1,.)"
     and upcase(dsn)="%scan(&dclib..MPE_TABLES,2,.)";
 quit;
diff --git a/sas/sasjs/services/editors/stagedata.sas b/sas/sasjs/services/editors/stagedata.sas
index 534db47..6f38c2b 100755
--- a/sas/sasjs/services/editors/stagedata.sas
+++ b/sas/sasjs/services/editors/stagedata.sas
@@ -253,13 +253,11 @@ data _null_;
 run;
 
 /* derived variables are display-only - they never reach the approval package
-  or the target table (the droplist was built before the security check) */
+  or the target table.  work.jsdata was rewritten without them before the
+  security check, so there is nothing left to drop here */
 data approve.jsdset;
   length _____DELETE__THIS__RECORD_____ $3;
   set jsdata;
-  %if %length(&derived_droplist)>0 %then %do;
-    drop &derived_droplist;
-  %end;
 run;
 
 /**
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 1aa36d9..11c7ff9 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -245,6 +245,31 @@ run;
   ,msg=%superq(errmsg)
 )
 
+/* the inverse of the check above: a declared name the expression never uses
+  is stale, and the read-time engine rejects it too */
+proc sql;
+create table work.derived_stale as
+  select d.libref, d.dsn, d.name, d.dep_nm
+  from work.derived_dep d
+  where not exists (
+    select 1 from work.derived_ref r
+    where r.libref=d.libref and r.dsn=d.dsn and r.name=d.name
+      and r.tok=d.dep_nm
+  );
+quit;
+
+data _null_;
+  set work.derived_stale(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,': VAR_DEPS lists ',dep_nm
+      ,', which VAR_LOGIC does not reference'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_stale))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
 /**
   * Cycle check - repeatedly take the variables whose derived inputs have all
   * been placed; a round that places nothing leaves a cycle
diff --git a/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas b/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas
index 6afa71d..d1c8f41 100644
--- a/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas
@@ -18,6 +18,7 @@
 
   <h4> SAS Macros </h4>
   @li dc_assignlib.sas
+  @li mf_nobs.sas
   @li mp_abort.sas
   @li mp_filtercheck.sas
 
@@ -81,6 +82,34 @@ data _null_;
   end;
 run;
 
+/* derived variables exist only in the view the EDIT screen reads - a rule on
+  one cannot be evaluated against the submitted data, so reject it here rather
+  than let the submit fail later with a variable-not-found error */
+proc sql;
+create table work.rls_derived as
+  select b.rls_libref, b.rls_table, b.variable_nm
+  from work.batches b
+  inner join &DC_LIBREF..mpe_derived d
+    on upcase(d.libref)=b.rls_libref
+    and upcase(d.dsn)=b.rls_table
+    and upcase(d.var_nm)=b.variable_nm
+  where &dc_dttmtfmt lt d.tx_to
+    and d.active=1;
+quit;
+
+data _null_;
+  set work.rls_derived(obs=1);
+  call symputx('rls_msg'
+    ,cats('MPE_ROW_LEVEL_SECURITY: ',variable_nm
+      ,' is a derived variable in ',rls_libref,'.',rls_table
+      ,' - derived variables cannot be used in a row level security rule'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.rls_derived))
+  ,mac=mpe_row_level_security_postedit
+  ,msg=%superq(rls_msg)
+)
+
 %macro quickloop();
 %do i=1 %to &cnt;
   data work.inds&i;
diff --git a/sas/sasjs/services/public/getrawdata.sas b/sas/sasjs/services/public/getrawdata.sas
index a9d15d8..9cb0e21 100644
--- a/sas/sasjs/services/public/getrawdata.sas
+++ b/sas/sasjs/services/public/getrawdata.sas
@@ -163,7 +163,9 @@ run;
     from work.derived_vars;
   quit;
 %end;
-%else %let filterds=&libds;
+%else %do;
+  %let filterds=&libds;
+%end;
 
 options obs=200000;/* stop limit */
 data staged(drop=&txfrom &txto);
diff --git a/sas/sasjs/services/public/viewdata.sas b/sas/sasjs/services/public/viewdata.sas
index a25504f..7cf8247 100644
--- a/sas/sasjs/services/public/viewdata.sas
+++ b/sas/sasjs/services/public/viewdata.sas
@@ -302,11 +302,10 @@ run;
     %end;
     %if %mf_existds(libds=MPSEARCH.vwsearch) %then %do;
       %let dsobs=%mf_nobs(MPSEARCH.vwsearch);
+      /* the derived columns were dropped from work.vwsearch above, so
+        mp_searchdata's output cannot carry them */
       data viewdata;
         set MPSEARCH.vwsearch;
-        %if %length(&derived_droplist)>0 %then %do;
-          drop &derived_droplist;
-        %end;
         if _n_<=&DC_MAXOBS_WEBVIEW;
       run;
     %end;

From c5a8de38b3375ead507e4e6de766d4cdc29a2848 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 17:59:49 +0000
Subject: [PATCH 39/48] test(derived): guard the getdata fixture, fix two
 harness faults, cover the RLS rejection

- getdata.test now checks the child job state before reading its outlib: a
  cancelled child returns no webout, so the previous call's members were being
  asserted against and the failures described the wrong problem.
- validatefilter.test's inputdatasets call needs a _PROGRAM column, which
  mx_execute reads to decide what to run - without it the job stopped at
  'The _PROGRAM column must exist'.
- stagedata.test.3 covers the new RLS check: an MPE_ROW_LEVEL_SECURITY rule
  naming a derived variable must be rejected at save time.
---
 sas/sasjs/services/editors/getdata.test.sas   | 34 ++++++--
 .../services/editors/stagedata.test.3.sas     | 82 +++++++++++++++++++
 .../services/public/validatefilter.test.sas   |  2 +
 3 files changed, 113 insertions(+), 5 deletions(-)

diff --git a/sas/sasjs/services/editors/getdata.test.sas b/sas/sasjs/services/editors/getdata.test.sas
index a979dbf..10ea70e 100644
--- a/sas/sasjs/services/editors/getdata.test.sas
+++ b/sas/sasjs/services/editors/getdata.test.sas
@@ -3,6 +3,7 @@
   @brief testing gethistory service
 
   <h4> SAS Macros </h4>
+  @li mp_assert.sas
   @li mp_assertcols.sas
   @li mp_assertcolvals.sas
   @li mp_assertdsobs.sas
@@ -324,13 +325,36 @@ run;
   mdebug=&sasjs_mdebug
 )
 
-data sasdata4;
-  set web4.sasdata;
-run;
-data cols4;
-  set web4.cols;
+/* a cancelled child returns no webout, so the outlib keeps whatever an
+  earlier call left in it - check the job state before reading its members,
+  otherwise the assertion failures below describe the wrong problem */
+%let abort4=0;
+data _null_;
+  set work.results;
+  if state='canceled' then call symputx('abort4',1);
 run;
 
+%mp_assert(
+  iftrue=(&abort4=0),
+  desc=Test4 - the derived variable call completes,
+  outds=work.test_results
+)
+
+%if &abort4=0 %then %do;
+  data sasdata4;
+    set web4.sasdata;
+  run;
+  data cols4;
+    set web4.cols;
+  run;
+%end;
+%else %do;
+  data cols4;
+    length name $32;
+    stop;
+  run;
+%end;
+
 %mp_assertcols(work.cols4,
   desc=Test4 - derived column is in the COLS payload,
   cols=DERIVED_NOTE,
diff --git a/sas/sasjs/services/editors/stagedata.test.3.sas b/sas/sasjs/services/editors/stagedata.test.3.sas
index f8a2853..6c2ff46 100644
--- a/sas/sasjs/services/editors/stagedata.test.3.sas
+++ b/sas/sasjs/services/editors/stagedata.test.3.sas
@@ -132,6 +132,88 @@ quit;
   outds=work.test_results
 )
 
+/**
+  * Test 3 - an RLS rule naming a DERIVED variable must be rejected at save
+  * time: the rule is evaluated against the submitted data, where a derived
+  * column does not exist
+  */
+%let t3=%sysfunc(datetime());
+
+/* define a derived variable on the target table */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+insert into &dc_libref..mpe_derived
+  set tx_from=0
+    ,tx_to='31DEC5999:23:59:59'dt
+    ,libref='DCTEST'
+    ,dsn='DC_REPLACE'
+    ,var_nm='DERIVED_X'
+    ,var_deps='X'
+    ,var_type='N'
+    ,var_len=8
+    ,var_fmt=''
+    ,var_logic='X + 1'
+    ,active=1;
+quit;
+
+data work.sascontroltable;
+  action='LOAD';
+  message='RLS rule on a derived variable should abort';
+  libds="&dc_libref..MPE_ROW_LEVEL_SECURITY";
+  output;
+  stop;
+run;
+
+data work.jsdata;
+  length RLS_SCOPE $8 RLS_GROUP $128 RLS_LIBREF $8 RLS_TABLE $32
+    RLS_GROUP_LOGIC $3 RLS_SUBGROUP_LOGIC $3 RLS_VARIABLE_NM $32
+    RLS_OPERATOR_NM $12 RLS_RAW_VALUE $4000
+    _____DELETE__THIS__RECORD_____ $3;
+  RLS_SCOPE='EDIT';
+  RLS_GROUP='SASAdministrators';
+  RLS_LIBREF='DCTEST';
+  RLS_TABLE='DC_REPLACE';
+  RLS_GROUP_LOGIC='AND';
+  RLS_SUBGROUP_LOGIC='OR';
+  RLS_SUBGROUP_ID=0;
+  RLS_VARIABLE_NM='DERIVED_X';
+  RLS_OPERATOR_NM='NE';
+  RLS_RAW_VALUE='1';
+  RLS_ACTIVE=1;
+  _____DELETE__THIS__RECORD_____='No';
+  output;
+run;
+
+%mx_execute(&_program,
+  viyacontext=&defaultcontext,
+  inputdatasets=work.sascontroltable work.jsdata,
+  outlib=web3,
+  outref=wb3,
+  mdebug=&sasjs_mdebug
+)
+
+%let abort3=0;
+proc sql noprint;
+select count(*) into: abort3
+  from &dc_libref..mpe_loads
+  where processed_dttm>&t3
+    and reason_txt="RLS rule on a derived variable should abort"
+    and upcase(status) ne 'SUCCESS';
+quit;
+
+%mp_assert(
+  iftrue=(&abort3>0),
+  desc=Checking an RLS rule on a derived variable is rejected,
+  outds=work.test_results
+)
+
+/* tidy up the derived configuration */
+proc sql noprint;
+delete from &dc_libref..mpe_derived
+  where upcase(libref)='DCTEST' and upcase(dsn)='DC_REPLACE';
+quit;
+
 /**
   * Test 2 - applying REPLACE loadtype to a table with RLS EDIT security
   * should abort.  The active EDIT rule was inserted in the prep section,
diff --git a/sas/sasjs/services/public/validatefilter.test.sas b/sas/sasjs/services/public/validatefilter.test.sas
index 9ab1bfb..0f9d397 100644
--- a/sas/sasjs/services/public/validatefilter.test.sas
+++ b/sas/sasjs/services/public/validatefilter.test.sas
@@ -48,6 +48,8 @@ run;
 
 data work.iwant;
   filter_table='DCTEST.DCFMTS-FC';
+  /* mx_execute reads the program to run from the input dataset */
+  _program="&_program";
 run;
 data work.filterquery;
 GROUP_LOGIC='AND';

From af31c36b670ec63a2a4ba056bf6c4b28f6096e08 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 22:19:52 +0000
Subject: [PATCH 40/48] fix(derived): only the delete marker is a reserved name

A table's metadata columns are named per table - VAR_TXFROM, VAR_TXTO and
VAR_PROCESSED in MPE_TABLES hold those names, and they can be anything (the
bitemporal demo table uses VER_FROM_DTTM and VER_TO_DTTM).  When they exist
they are physical columns of the target, so the engine's collision check
already refuses a VAR_NM that matches one, whatever it is called.

The hard-coded list treated three example names as special: it refused them on
tables that use other names, and told the reader something untrue about what
is reserved.
---
 sas/sasjs/macros/mpe_derived.sas                  | 7 ++++---
 sas/sasjs/services/hooks/mpe_derived_postedit.sas | 3 +--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sas/sasjs/macros/mpe_derived.sas b/sas/sasjs/macros/mpe_derived.sas
index 69ca703..c77c529 100644
--- a/sas/sasjs/macros/mpe_derived.sas
+++ b/sas/sasjs/macros/mpe_derived.sas
@@ -80,9 +80,10 @@
 
 %local reserved;
 
-/* the metadata columns of every Data Controller table */
-%let reserved=%str(_____DELETE__THIS__RECORD_____ TX_FROM TX_TO PROCESSED_DTTM
-  VAR_TXFROM VAR_TXTO VAR_PROCESSED);
+/* the only name that has to be refused outright - the metadata columns of the
+  target table are physical columns, so the engine's collision check covers
+  them whatever MPE_TABLES calls them */
+%let reserved=%str(_____DELETE__THIS__RECORD_____);
 
 %mp_abort(iftrue= (%length(&libds)=0 or %length(&dclib)=0)
   ,mac=&sysmacroname
diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 11c7ff9..3e1e9fd 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -111,8 +111,7 @@ data work.derived_check;
     then reason='VAR_NM must be a valid SAS name';
   else if verify(substr(var_nm,1,1),'0123456789')=0
     then reason='VAR_NM must not start with a digit';
-  else if var_nm in ('_____DELETE__THIS__RECORD_____','TX_FROM','TX_TO'
-    ,'PROCESSED_DTTM','VAR_TXFROM','VAR_TXTO','VAR_PROCESSED')
+  else if var_nm='_____DELETE__THIS__RECORD_____'
     then reason='VAR_NM is reserved';
   else if var_type not in ('C','N')
     then reason='VAR_TYPE must be C or N';

From 422f766cd43db4ce740e1890fd5df30790eaa7e3 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Wed, 21 Oct 2026 22:31:56 +0000
Subject: [PATCH 41/48] fix(derived): close the two gaps in the derived/RLS
 invariant

- the join against MPE_DERIVED compared an upcased var_nm with the rule's
  column as stored, so a case variant slipped past the explanatory rejection
  and fell through to the generic filter check.  Both sides are upcased.
- the invariant was enforced only from the RLS side: configuring a derived
  variable for a table whose active rule already names it passed the derived
  hook.  The same check now runs there, against the incoming libref/dsn/var_nm.
---
 .../services/hooks/mpe_derived_postedit.sas   | 27 +++++++++++++++++++
 .../hooks/mpe_row_level_security_postedit.sas |  2 +-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/sas/sasjs/services/hooks/mpe_derived_postedit.sas b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
index 3e1e9fd..3859105 100644
--- a/sas/sasjs/services/hooks/mpe_derived_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_derived_postedit.sas
@@ -269,6 +269,33 @@ run;
   ,msg=%superq(errmsg)
 )
 
+/* the same invariant the RLS hook enforces from its side: a derived variable
+  cannot be named by an active row level security rule, because the columns
+  are dropped before the security clause is evaluated */
+proc sql;
+create table work.derived_rls as
+  select d.libref, d.dsn, d.name
+  from work.derived_rows d
+  inner join &DC_LIBREF..mpe_row_level_security r
+    on upcase(r.rls_libref)=upcase(d.libref)
+    and upcase(r.rls_table)=upcase(d.dsn)
+    and upcase(r.rls_variable_nm)=upcase(d.name)
+  where r.rls_active=1
+    and &dc_dttmtfmt lt r.tx_to;
+quit;
+
+data _null_;
+  set work.derived_rls(obs=1);
+  call symputx('errmsg'
+    ,cats('MPE_DERIVED: ',name,' is named by a row level security rule on '
+      ,libref,'.',dsn,' - derived variables cannot be used there'));
+run;
+
+%mp_abort(iftrue= (0 < %mf_nobs(work.derived_rls))
+  ,mac=mpe_derived_postedit
+  ,msg=%superq(errmsg)
+)
+
 /**
   * Cycle check - repeatedly take the variables whose derived inputs have all
   * been placed; a round that places nothing leaves a cycle
diff --git a/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas b/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas
index d1c8f41..7d836ce 100644
--- a/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas
+++ b/sas/sasjs/services/hooks/mpe_row_level_security_postedit.sas
@@ -92,7 +92,7 @@ create table work.rls_derived as
   inner join &DC_LIBREF..mpe_derived d
     on upcase(d.libref)=b.rls_libref
     and upcase(d.dsn)=b.rls_table
-    and upcase(d.var_nm)=b.variable_nm
+    and upcase(d.var_nm)=upcase(b.variable_nm)
   where &dc_dttmtfmt lt d.tx_to
     and d.active=1;
 quit;

From 34691dceecb63f952b23ad8acc5bc93753c6cfee Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 16:38:37 +0000
Subject: [PATCH 42/48] feat(editor): serve a truncated selection with a
 warning instead of aborting

A selection over DC_MAXOBS_WEBEDIT or DC_MAXCELLS_WEBEDIT was refused with
"Table is too big", leaving the user with no grid and no way to see what the
filter had matched.  getdata now serves the first rows that fit and flags it:

- the row cap is min(DC_MAXOBS_WEBEDIT, DC_MAXCELLS_WEBEDIT / columns), so a
  wide table is still bounded by the cell budget
- sasparams carries TRUNCATED (0/1) and TRUNC_MSG, naming the limit reached
- the EDIT screen shows a warning triangle next to the table name, with the
  message in a tooltip and the advice to filter

The mock mirrors the new behaviour, the Cypress spec asserts the truncated
grid and the tooltip (real hover, opacity checked so a hidden tooltip cannot
pass), and the SAS limit test now asserts the served row counts and message
rather than a cancelled child job.
---
 client/cypress/e2e/row-cell-limits.cy.ts      | 133 +++++++++++-------
 client/src/app/editor/editor.component.html   |  19 +++
 client/src/app/editor/editor.component.ts     |   8 ++
 client/src/app/models/TableData.ts            |   4 +
 sas/mocks/sasjs/services/editors/getdata.js   |  96 +++++++------
 sas/sasjs/services/editors/getdata.sas        |  58 +++++---
 sas/sasjs/services/editors/getdata.test.1.sas | 128 ++++++++++-------
 sas/sasjs/services/editors/getdata.test.2.sas |   8 +-
 8 files changed, 286 insertions(+), 168 deletions(-)

diff --git a/client/cypress/e2e/row-cell-limits.cy.ts b/client/cypress/e2e/row-cell-limits.cy.ts
index f154801..571cc6b 100644
--- a/client/cypress/e2e/row-cell-limits.cy.ts
+++ b/client/cypress/e2e/row-cell-limits.cy.ts
@@ -1,13 +1,15 @@
-// The EDIT screen refuses a selection that is too large - DC_MAXOBS_WEBEDIT
-// rows, or DC_MAXCELLS_WEBEDIT cells (rows x columns), whichever is reached
-// first - and the VIEW screen caps at DC_MAXOBS_WEBVIEW rows.
+// The EDIT screen is bounded by DC_MAXOBS_WEBEDIT rows and DC_MAXCELLS_WEBEDIT
+// cells (rows x columns), whichever is reached first, and the VIEW screen caps
+// at DC_MAXOBS_WEBVIEW rows.
 //
+// A selection over an EDIT limit is served TRUNCATED - the grid carries the
+// first rows that fit, and a warning triangle against the table name says why.
 // This spec pins those limits against the mock, which mirrors the shipped
 // defaults (250 rows / 200000 cells for EDIT, 2000 rows for VIEW).  It uses
 // two existing fixtures rather than new ones:
 //   MPE_X_SEARCH - 1000 rows x 9 cols, so it trips the ROW limit
-//   MPE_X_WIDE   - 200 rows x 1001 cols (200,400 cells), under the row limit
-//                  but over the CELL limit
+//   MPE_X_WIDE   - 200 rows x 1001 cols (200,200 cells), under the row limit
+//                  but over the CELL limit, so the row cap is 199
 export {}
 
 const hostUrl = Cypress.env('hosturl')
@@ -60,27 +62,51 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
     })
   })
 
-  it('2 | EDIT refuses a table over the row limit', () => {
+  it('2 | EDIT serves a table over the row limit, truncated, with a warning', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
     openEdit()
 
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 1000 rows x 9 columns is over the 250 row limit, so the grid carries the
+    // first 250 rows and the header says how many it is showing.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
+      '250 rows'
     )
+    cy.get('#hotTable .ht_master .htCore tbody tr', {
+      timeout: longerCommandTimeout
+    }).should('have.length.at.least', 1)
+
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXOBS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
   })
 
-  it('3 | EDIT refuses a table over the cell limit', () => {
+  it('3 | EDIT serves a table over the cell limit, truncated, with a warning', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_wide')
     openEdit()
 
-    // 200 rows x 1002 columns (the 1001 stored columns plus the housekeeping
-    // delete column) = 200,400 cells, over the 200,000 cell limit, while the
-    // row count is under the 250 row limit.
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+    // 200 rows x 1001 stored columns = 200,200 cells, over the 200,000 cell
+    // limit, while the row count is under the 250 row limit - so the cell
+    // limit sets the cap: floor(200000 / 1001) = 199 rows.
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      'Selection is too wide (200 rows x 1002 cols) - please filter and try again!'
+      '199 rows'
     )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
   })
 
   it('4 | the EDIT limits follow MPE_CONFIG, not the shipped defaults', () => {
@@ -90,20 +116,29 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
     // all.  And MPE_CONFIG lives in the control library, so reading it from the
     // table's own library would fail for this table in particular.
     //
-    // The cell limit is the one to move: 7 rows x 18 columns is 126 cells, so
-    // at 110 the TESTDATA.MPE_X_NEW edit must be refused, where a hard-coded
-    // 200000 - or a config read that fails and falls back to it - lets it
-    // through.  110 is still above MPE_CONFIG's own 17 x 6 = 102, so the option
-    // can be put back afterwards.
+    // The cell limit is the one to move: 7 rows x 17 columns is 119 cells, so
+    // at 110 the TESTDATA.MPE_X_NEW edit must be truncated to
+    // floor(110 / 17) = 6 rows, where a hard-coded 200000 - or a config read
+    // that fails and falls back to it - lets it through.  110 is still above
+    // MPE_CONFIG's own 6 x 17 = 102, so the option can be put back afterwards.
     setConfigValue('DC_MAXCELLS_WEBEDIT', '110', () => {
       visitPage('view/data/TESTDATA.MPE_X_NEW')
       closeAbortModalIfPresent()
       openEdit()
 
-      cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
         'contain.text',
-        'Selection is too wide (7 rows x 18 cols) - please filter and try again!'
+        '6 rows'
       )
+      cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+        .should('be.visible')
+        .realHover()
+      cy.get('.editor-title clr-tooltip-content', {
+        timeout: longerCommandTimeout
+      })
+        .should('be.visible')
+        .should('contain.text', 'DC_MAXCELLS_WEBEDIT')
+        .should(($el) => expect($el.css('opacity')).to.eq('1'))
     })
 
     // Put the option back, so the rest of the estate sees the shipped default.
@@ -112,28 +147,31 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
       closeAbortModalIfPresent()
       openEdit()
 
-      // 126 cells under a 200000 cell limit: the editor loads, no abort.
-      cy.get('#hotTable .ht_master .htCore tbody tr', {
-        timeout: longerCommandTimeout
-      }).should('exist')
+      // 119 cells under a 200000 cell limit: all 7 rows load, no warning.
+      cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+        'contain.text',
+        '7 rows'
+      )
+      cy.get('.truncation-warning').should('not.exist')
     })
   })
 
-  it('5 | the filter picker still lists the variables after the abort', () => {
+  it('5 | the warning says to filter, and filtering narrows the grid', () => {
     openTableFromTree(libraryToOpenIncludes, 'mpe_x_search')
     openEdit()
 
-    cy.get('.abortMsg', { timeout: longerCommandTimeout }).should(
-      'contain.text',
-      'Table is too big (1000 rows) - please filter and try again!'
-    )
+    cy.get('.truncation-warning', { timeout: longerCommandTimeout })
+      .should('be.visible')
+      .realHover()
+    cy.get('.editor-title clr-tooltip-content', {
+      timeout: longerCommandTimeout
+    })
+      .should('be.visible')
+      .should('contain.text', 'Filter the table to work with the rest')
+      .should(($el) => expect($el.css('opacity')).to.eq('1'))
 
-    closeAbortModal()
-
-    // The abort returns no table, and the picker's variable list normally comes
-    // from the table's own columns - so without the public/getcols fallback the
-    // "please filter and try again" advice has nothing to pick from.  Every
-    // column of MPE_X_SEARCH must be on offer.
+    // The advice has to be actionable: the picker must offer the table's
+    // columns and the filter must reload the grid.
     openFilterPopup()
 
     filterVariables().should(($options) => {
@@ -155,20 +193,20 @@ context('EDIT row and cell limits, VIEW row limit: ', function () {
       ])
     })
 
-    // and the suggestion list is usable, not just populated
     setFilterVariable('SITE_NAME')
-
-    // Submitting is what exercises TYPE.  The picker keys its operator set and
-    // its value quoting on it, and the fallback list never passes through
-    // mergeColsRules, so without TYPE on this payload a character value would be
-    // submitted unquoted and the backend's %mp_filtercheck would reject it.
-    setFilterValue('Bristol')
+    // 100 of the 1000 rows: under the limit, so nothing is truncated.
+    setFilterValue('Nereid Trench')
     submitFilter()
 
     cy.get('.infoBar-text', { timeout: longerCommandTimeout }).should(
       'contain.text',
-      "SITE_NAME = 'Bristol'"
+      "SITE_NAME = 'Nereid Trench'"
     )
+    cy.get('.editor-title', { timeout: longerCommandTimeout }).should(
+      'contain.text',
+      '100 rows'
+    )
+    cy.get('.truncation-warning').should('not.exist')
   })
 })
 
@@ -185,13 +223,6 @@ const openEdit = () => {
   }).click()
 }
 
-/** The abort modal offers Close, and the filter button sits behind it. */
-const closeAbortModal = () => {
-  cy.get('app-info-modal', { timeout: longerCommandTimeout })
-    .contains('button', 'Close')
-    .click()
-}
-
 /**
  * The EDIT screen's Filter button opens the same .filter-modal the viewer uses.
  * It only renders once the editor is in its read-only (non-editing) state,
diff --git a/client/src/app/editor/editor.component.html b/client/src/app/editor/editor.component.html
index 503618b..57bd427 100644
--- a/client/src/app/editor/editor.component.html
+++ b/client/src/app/editor/editor.component.html
@@ -245,6 +245,25 @@
               }
             </clr-tooltip>
 
+            @if (truncationNote) {
+              <clr-tooltip>
+                <clr-icon
+                  clrTooltipTrigger
+                  shape="warning"
+                  aria-label="The selection was truncated"
+                  class="color-orange truncation-warning"
+                  size="20"
+                ></clr-icon>
+                <clr-tooltip-content
+                  clrPosition="right"
+                  clrSize="lg"
+                  *clrIfOpen
+                >
+                  {{ truncationNote }}
+                </clr-tooltip-content>
+              </clr-tooltip>
+            }
+
             @if (dataSource) {
               @if (!zeroFilterRows) {
                 ({{ dataSource.length | thousandSeparator: ',' }}
diff --git a/client/src/app/editor/editor.component.ts b/client/src/app/editor/editor.component.ts
index 74d2ae8..02b9971 100644
--- a/client/src/app/editor/editor.component.ts
+++ b/client/src/app/editor/editor.component.ts
@@ -160,6 +160,9 @@ export class EditorComponent
   dsmeta: DSMeta[] = []
   versions: Version[] = []
   dsNote = ''
+  /** Set when getdata served a truncated selection - shown as a warning
+   * against the table name, with the reason in a tooltip. */
+  truncationNote = ''
 
   viewboxes = false
 
@@ -3930,6 +3933,11 @@ export class EditorComponent
       this.dsNote = ''
     }
 
+    // getdata serves a selection over DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT
+    // truncated to the first rows within the limit, and carries the reason
+    // here.  Empty means the whole selection is in the grid.
+    this.truncationNote = response.data.sasparams[0].TRUNC_MSG || ''
+
     const hot = this.hotInstance
 
     const approvers: Approver[] = response.data.approvers
diff --git a/client/src/app/models/TableData.ts b/client/src/app/models/TableData.ts
index 8cacdc1..61ff40e 100644
--- a/client/src/app/models/TableData.ts
+++ b/client/src/app/models/TableData.ts
@@ -47,6 +47,10 @@ export interface SASParam {
   PKCNT: number
   RK_FLAG: number
   TMVARS: string
+  /** 1 when getdata cut the selection down to the row / cell limit */
+  TRUNCATED?: number
+  /** Reason shown in the tooltip when TRUNCATED is 1, else empty */
+  TRUNC_MSG?: string
 }
 
 export interface ExcelRule {
diff --git a/sas/mocks/sasjs/services/editors/getdata.js b/sas/mocks/sasjs/services/editors/getdata.js
index c2b6087..9badcde 100644
--- a/sas/mocks/sasjs/services/editors/getdata.js
+++ b/sas/mocks/sasjs/services/editors/getdata.js
@@ -191,9 +191,8 @@ if (tableData && tableData.rows && tableData.columns) {
 // ─── Build response ──────────────────────────────────────────────────────────
 
 let response
-// Set when the row or cell limit is exceeded - mirrors the %mp_abort in
-// getdata.sas, and is emitted as sasjsAbort below.
-let abortMsg = ''
+// Set when the row or cell limit cut the selection down - mirrors the
+// TRUNCATED / TRUNC_MSG in sasparams in getdata.sas.
 
 if (tableData && tableData.columns && tableData.rows) {
     // Exclude temporal columns (TX_FROM/TX_TO etc.) from the editor view
@@ -236,15 +235,15 @@ if (tableData && tableData.columns && tableData.rows) {
     }
 
     // ─── Row and cell limits ─────────────────────────────────────────────────
-    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows, or DC_MAXCELLS_WEBEDIT
-    // cells (rows x columns), whichever is reached first.  Both are read from
-    // MPE_CONFIG, the way the real service picks them up from the settings
-    // program it generates - so editing the option in DC changes what the mock
-    // allows, and an option that is absent (or switched off, var_active=0)
-    // falls back to the shipped default.  The check runs after the
-    // PRE_EDIT_HOOK, as the single check in getdata.sas does - a hook may
-    // replace the data with a larger table.  The cell count includes the
-    // housekeeping delete column, which is what getdata.sas counts.
+    // Mirrors getdata.sas: the selection is served TRUNCATED, not refused, at
+    // DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT cells (rows x columns),
+    // whichever is reached first - the row cap is min(rows, cells/columns).
+    // Both options are read from MPE_CONFIG, the way the real service picks
+    // them up from the settings program it generates - so editing the option
+    // in DC changes what the mock serves, and an option that is absent (or
+    // switched off, var_active=0) falls back to the shipped default.  The
+    // check runs after the PRE_EDIT_HOOK, as the single check in getdata.sas
+    // does - a hook may replace the data with a larger table.
     // MPE_CONFIG lives in the control library, not in the table's own library,
     // so this reads mpeDataDir - the same directory the MPE_TABLES lookup above
     // uses.  Passing the requested table's dataDir made the read fail for every
@@ -252,20 +251,32 @@ if (tableData && tableData.columns && tableData.rows) {
     const dcCfg = loadDcConfig(driveRoot, appLoc, mpeDataDir)
     const maxObsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXOBS_WEBEDIT', 250)
     const maxCellsWebEdit = dcConfigNumber(dcCfg, 'DC_MAXCELLS_WEBEDIT', 200000)
-    const editColCount = visibleColumns.length + 1
-    const cellCount = visibleRows.length * editColCount
-    if (visibleRows.length > maxObsWebEdit) {
-        abortMsg =
-            'Table is too big (' +
-            visibleRows.length +
-            ' rows) - please filter and try again!'
-    } else if (cellCount > maxCellsWebEdit) {
-        abortMsg =
-            'Selection is too wide (' +
-            visibleRows.length +
-            ' rows x ' +
-            editColCount +
-            ' cols) - please filter and try again!'
+    // getdata.sas counts the columns of work.out, i.e. the table's own
+    // columns - the delete column is client-owned and not part of it.
+    const editColCount = visibleColumns.length
+    let maxRows = maxObsWebEdit
+    let limitVar = 'DC_MAXOBS_WEBEDIT'
+    if (editColCount > 0) {
+        const cellRows = Math.floor(maxCellsWebEdit / editColCount)
+        if (cellRows < maxRows) {
+            maxRows = cellRows
+            limitVar = 'DC_MAXCELLS_WEBEDIT'
+        }
+    }
+    if (maxRows < 1) maxRows = 1
+    let truncated = 0
+    let truncMsg = ''
+    if (visibleRows.length > maxRows) {
+        truncated = 1
+        // Same wording as the macro getdata.sas sets, so the spec can assert
+        // on the text the real service would send.
+        truncMsg =
+            'Only the first ' +
+            maxRows +
+            ' rows are loaded - the ' +
+            limitVar +
+            ' limit has been reached.  Filter the table to work with the rest.'
+        visibleRows = visibleRows.slice(0, maxRows)
     }
 
     // Augment rows: add delete flag, normalise keys, convert temporal values to ISO
@@ -371,7 +382,9 @@ if (tableData && tableData.columns && tableData.rows) {
             LOADTYPE: tableReg ? tableReg.loadtype || 'UPDATE' : 'UPDATE',
             RK_FLAG: tableReg && tableReg.rk_underlying ? 1 : 0,
             CLS_FLAG: 0,
-            ISMAP: 0
+            ISMAP: 0,
+            TRUNCATED: truncated,
+            TRUNC_MSG: truncMsg
         }],
         versions: [],
         xl_rules: []
@@ -386,26 +399,21 @@ if (tableData && tableData.columns && tableData.rows) {
         ],
         maxvarlengths: [], query: [], sasdata: [],
         $sasdata: { vars: {} },
-        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0 }],
+        sasparams: [{ COLHEADERS: '', FILTER_TEXT: ' ', PKCNT: 0, PK: '', DTVARS: '', DTTMVARS: '', TMVARS: '', LOADTYPE: 'UPDATE', RK_FLAG: 0, CLS_FLAG: 0, ISMAP: 0, TRUNCATED: 0, TRUNC_MSG: '' }],
         versions: [], xl_rules: []
     }
 }
 
 webOutOpen()
-if (abortMsg) {
-    webOutObj([{ MSG: abortMsg, MAC: 'getdata' }], 'sasjsAbort')
-    webOutObj([{ STATUS: 'ERROR' }], 'sasparams')
-} else {
-    webOutObj(response.approvers, 'approvers')
-    webOutObj(response.cols, 'cols')
-    webOutObj(response.dqdata, 'dqdata')
-    webOutObj(response.dqrules, 'dqrules')
-    webOutObj(response.dsmeta, 'dsmeta')
-    webOutObj(response.maxvarlengths, 'maxvarlengths')
-    webOutObj(response.query, 'query')
-    webOutObj(response.sasdata, 'sasdata', response.$sasdata)
-    webOutObj(response.sasparams, 'sasparams')
-    webOutObj(response.versions, 'versions')
-    webOutObj(response.xl_rules, 'xl_rules')
-}
+webOutObj(response.approvers, 'approvers')
+webOutObj(response.cols, 'cols')
+webOutObj(response.dqdata, 'dqdata')
+webOutObj(response.dqrules, 'dqrules')
+webOutObj(response.dsmeta, 'dsmeta')
+webOutObj(response.maxvarlengths, 'maxvarlengths')
+webOutObj(response.query, 'query')
+webOutObj(response.sasdata, 'sasdata', response.$sasdata)
+webOutObj(response.sasparams, 'sasparams')
+webOutObj(response.versions, 'versions')
+webOutObj(response.xl_rules, 'xl_rules')
 webOutClose()
diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index f9f5c90..c0de9d7 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -17,6 +17,10 @@
   @li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
     else set to 1 (CLS rules exist)
   @li ISMAP - set to 1 if the target DS is an excel map target, else 0
+  @li TRUNCATED - set to 1 if the selection was cut down to the
+    DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
+  @li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
+    else empty
 
   <h5> approvers </h5>
   <h5> dqrules </h5>
@@ -262,30 +266,38 @@ select upcase(loadtype)
   )
 
   /* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
-    then run the hook, then check the row and cell limits.  Checking once,
-    after both, means the counts cover everything that could reach the
-    browser: the rows the selection matched, and anything the hook has added
-    to work.out.  A wide table exhausts the cell budget long before it
-    reaches the row limit - 250 rows of a 1000 column table is a quarter of
-    a million cells - so capping rows alone does not bound the work the
-    browser is asked to do.  The payload itself is capped further down, when
-    work.outdata is built. */
+    then run the hook, then apply the row and cell limits (below). */
   %let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
   proc sort data=work.out; by &sortPK; run;
 
   %mpe_runhook(PRE_EDIT_HOOK)
 
+  /* A selection over a limit is served truncated, not refused: the browser
+    gets the first rows within the limit and sasparams carries a warning the
+    EDIT screen shows against the table name.  Truncating bounds the browser's
+    work the same way an abort did, without leaving the user with no grid and
+    no explanation of what happened.  A wide table exhausts the cell budget
+    long before it reaches the row limit - 250 rows of a 1000 column table is
+    a quarter of a million cells - so the row cap is min(rows, cells/columns).
+    Counting after the PRE_EDIT_HOOK means the cap covers anything the hook
+    added to work.out. */
   %let obscnt=%mf_getattrn(work.out,NLOBS);
   %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
-  %let cellcnt=%eval(&obscnt * &colcnt);
-  %mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Table is too big (&obscnt rows) - please filter and try again!
-  )
-  %mp_abort(iftrue=(&cellcnt>&DC_MAXCELLS_WEBEDIT)
-    ,mac=&_program
-    ,msg=Selection is too wide (&obscnt rows x &colcnt cols) - please filter and try again!
-  )
+  %let maxrows=&DC_MAXOBS_WEBEDIT;
+  %let limitvar=DC_MAXOBS_WEBEDIT;
+  %if &colcnt>0 %then %do;
+    %let cellrows=%sysfunc(floor(&DC_MAXCELLS_WEBEDIT/&colcnt));
+    %if %sysevalf(&cellrows lt &maxrows) %then %do;
+      %let maxrows=&cellrows;
+      %let limitvar=DC_MAXCELLS_WEBEDIT;
+    %end;
+  %end;
+  %if %sysevalf(&maxrows lt 1) %then %let maxrows=1;
+  %global truncated trunc_msg;
+  %let truncated=0;
+  %if %sysevalf(&obscnt gt &maxrows) %then %let truncated=1;
+  %let trunc_msg=;
+  %if &truncated=1 %then %let trunc_msg=Only the first &maxrows rows are loaded - the &limitvar limit has been reached.  Filter the table to work with the rest.;
 
 
   /* order pk fields at start of table.  The delete flag is NOT added here:
@@ -297,9 +309,9 @@ select upcase(loadtype)
                             ,Str2=&pk
                           );
   %put sourcevars=&sourcevars;
-  /* the payload is capped here, at the row limit - the limits above are
-    checked against the whole selection, this is what goes to the browser */
-  options obs=&DC_MAXOBS_WEBEDIT;
+  /* the payload is capped here, at &maxrows - whichever of the row and cell
+    limits that works out to - this is what goes to the browser */
+  options obs=&maxrows;
   data outdata;
     /* pk fields come first */
     %if %length(&pk)>0 %then %do;
@@ -572,7 +584,7 @@ select count(*) into: ismap from &mpelib..mpe_xlmap_info
   where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
 
 data sasparams;
-  length colHeaders $20000 filter_text $32767;
+  length colHeaders $20000 filter_text $32767 trunc_msg $200;
   colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
   pkCnt=countw("&pk");
   pk="&pk";
@@ -583,6 +595,10 @@ data sasparams;
   if trim(symget('rk_underlying')) ne '' then rk_flag=1;
   else rk_flag=0;
   filter_text=symget('filter_text');
+  /* set when the selection was cut down to the row / cell limit, so the EDIT
+    screen can say so against the table name */
+  truncated=&truncated;
+  trunc_msg="&trunc_msg";
   if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
   else cls_flag=1;
   put (_all_)(=);
diff --git a/sas/sasjs/services/editors/getdata.test.1.sas b/sas/sasjs/services/editors/getdata.test.1.sas
index 67ccbe8..b153a88 100644
--- a/sas/sasjs/services/editors/getdata.test.1.sas
+++ b/sas/sasjs/services/editors/getdata.test.1.sas
@@ -2,17 +2,19 @@
   @file
   @brief testing getdata row and cell limits
 
-  @details The EDIT screen refuses a selection that is too big on either
-  count - rows (DC_MAXOBS_WEBEDIT) or cells (DC_MAXCELLS_WEBEDIT).  The row
-  check runs first, so a selection over the row limit reports "Table is too
-  big" and one under the row limit but over the cell limit reports
-  "Selection is too wide".  Both aborts are proven by the child job state
-  (an aborted service registers no webout).
+  @details The EDIT screen is bounded by two limits - rows
+  (DC_MAXOBS_WEBEDIT) and cells, rows x columns (DC_MAXCELLS_WEBEDIT) -
+  whichever is reached first.  A selection over either is SERVED TRUNCATED,
+  not refused: the grid gets the first rows that fit, and sasparams carries
+  TRUNCATED=1 plus a TRUNC_MSG naming the limit, which the EDIT screen shows
+  as a warning against the table name.
 
-  Fixture: dctest.wideboy (1000 rows x 1004 columns), created and registered
-  by testsetup.  Filtered to 300 rows it is over the 250 row limit; filtered
-  to 200 rows it is under the row limit but 200 x 1004 = 200,800 cells, over
-  the 200,000 cell limit.
+  Test 1 uses MPE_X_TEST (several hundred rows, few columns) so the row
+  limit alone decides the cap.  Test 2 uses the fixture dctest.wideboy (1000
+  rows x 1001 columns, created and registered by testsetup) filtered to 200
+  rows: that is under the row limit but 200 x 1001 = 200,200 cells, over the
+  200,000 cell limit, so the cell limit decides the cap -
+  floor(200000 / 1001) = 199 rows.
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
@@ -25,7 +27,7 @@
 %let _program=&appLoc/services/editors/getdata;
 
 /**
-  * build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
+  * Build a ROW_ID <= &maxrow filter on dctest.wideboy, leaving the filter_rk
   * in &filter_rk
   */
 %macro wideboy_filter(maxrow);
@@ -57,61 +59,91 @@
 %mend wideboy_filter;
 
 /**
-  * Test 1 - row limit (300 rows > 250)
+  * Call getdata as a child job and expose the webout as the &outlib JSON
+  * libname, so the test can read what the browser would have received.
   */
-%wideboy_filter(300)
-%let f1=%mf_getuniquefileref();
-data _null_;
-  file &f1 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%macro call_getdata(libds,filter_rk,outlib);
+  %let fref=%mf_getuniquefileref();
+  data _null_;
+    file &fref termstr=crlf;
+    put 'LIBDS:$43. FILTER_RK:8.';
+    put "&libds,&filter_rk";
+  run;
+  %mx_execute(&_program,
+    viyacontext=&defaultcontext,
+    inputfiles=&fref:sascontroltable,
+    outlib=&outlib
+  )
+  %let syscc=0;
+%mend call_getdata;
 
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f1:sascontroltable,
-  outref=web1
-)
+/**
+  * Test 1 - row limit.  MPE_X_TEST is well under the cell limit, so the
+  * 250 row default is what caps the payload.
+  */
+%call_getdata(&dclib..MPE_X_TEST,-1,web1)
 
-%let abort1=0;
+%let t1_nobs=-1;
+%let t1_trunc=-1;
+%let t1_msg=;
+proc sql noprint;
+select count(*) into: t1_nobs from web1.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort1',1);
+  set web1.sasparams;
+  call symputx('t1_trunc',truncated);
+  call symputx('t1_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
-%let syscc=0;
 
 %mp_assert(
-  iftrue=(&abort1=1),
-  desc=Test 1 - a selection over the row limit is refused,
+  iftrue=(&t1_trunc=1),
+  desc=Test 1 - a selection over the row limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t1_nobs=250),
+  desc=Test 1 - the payload is capped at DC_MAXOBS_WEBEDIT (250) rows,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t1_msg,DC_MAXOBS_WEBEDIT)>0),
+  desc=Test 1 - the truncation message names the DC_MAXOBS_WEBEDIT limit,
   outds=work.test_results
 )
 
 /**
-  * Test 2 - cell limit (200 rows x 1004 cols = 200,800 > 200,000)
+  * Test 2 - cell limit (200 rows x 1001 cols = 200,200 > 200,000)
   */
 %wideboy_filter(200)
-%let f2=%mf_getuniquefileref();
-data _null_;
-  file &f2 termstr=crlf;
-  put 'LIBDS:$43. FILTER_RK:8.';
-  put "dctest.wideboy,&filter_rk";
-run;
+%call_getdata(dctest.wideboy,&filter_rk,web2)
 
-%mx_execute(&_program,
-  viyacontext=&defaultcontext,
-  inputfiles=&f2:sascontroltable,
-  outref=web2
-)
-
-%let abort2=0;
+%let t2_nobs=-1;
+%let t2_trunc=-1;
+%let t2_msg=;
+proc sql noprint;
+select count(*) into: t2_nobs from web2.sasdata;
+quit;
 data _null_;
-  set work.results;
-  if state='canceled' then call symputx('abort2',1);
+  set web2.sasparams;
+  call symputx('t2_trunc',truncated);
+  call symputx('t2_msg',trunc_msg);
+  putlog (_all_)(=);
 run;
 
 %mp_assert(
-  iftrue=(&abort2=1),
-  desc=Test 2 - a selection over the cell limit is refused,
+  iftrue=(&t2_trunc=1),
+  desc=Test 2 - a selection over the cell limit is served, flagged as truncated,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&t2_nobs=199),
+  desc=Test 2 - the payload is capped at floor(DC_MAXCELLS_WEBEDIT / cols),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%index(&t2_msg,DC_MAXCELLS_WEBEDIT)>0),
+  desc=Test 2 - the truncation message names the DC_MAXCELLS_WEBEDIT limit,
   outds=work.test_results
 )
 
diff --git a/sas/sasjs/services/editors/getdata.test.2.sas b/sas/sasjs/services/editors/getdata.test.2.sas
index d673b9c..2ff03f6 100644
--- a/sas/sasjs/services/editors/getdata.test.2.sas
+++ b/sas/sasjs/services/editors/getdata.test.2.sas
@@ -21,10 +21,10 @@
   editing MPE_GROUPS alone - Row Level Security is skipped for them - so the
   test reports a skip row instead of running the assertions.
 
-  The call passes a filter (PRIMARY_KEY_FIELD < 100) because getdata.sas also
-  enforces the row / cell limits, and the unfiltered MPE_X_TEST is above the
-  row limit (642 rows against DC_MAXOBS_WEBEDIT:250) - the limits test
-  (getdata.test.1.sas) covers that abort separately.
+  The call passes a filter (PRIMARY_KEY_FIELD < 100) so the whole selection is
+  served - the unfiltered MPE_X_TEST is above the row limit (642 rows against
+  DC_MAXOBS_WEBEDIT:250) and would come back truncated.  The limits test
+  (getdata.test.1.sas) covers that truncation separately.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas

From 5585ae481e997d52092059ce5e59a2f0d10966ab Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 16:49:21 +0000
Subject: [PATCH 43/48] docs(comments): describe the limit handling, not the
 behaviour it replaced

The getdata comment explained the truncation by contrast with the abort it
replaces.  State what the code does and why instead.  Same for the two
not-refused phrasings in the mock and the test header.
---
 sas/mocks/sasjs/services/editors/getdata.js   |  6 +++---
 sas/sasjs/services/editors/getdata.sas        | 20 +++++++++----------
 sas/sasjs/services/editors/getdata.test.1.sas |  8 ++++----
 3 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/sas/mocks/sasjs/services/editors/getdata.js b/sas/mocks/sasjs/services/editors/getdata.js
index 9badcde..019d515 100644
--- a/sas/mocks/sasjs/services/editors/getdata.js
+++ b/sas/mocks/sasjs/services/editors/getdata.js
@@ -235,9 +235,9 @@ if (tableData && tableData.columns && tableData.rows) {
     }
 
     // ─── Row and cell limits ─────────────────────────────────────────────────
-    // Mirrors getdata.sas: the selection is served TRUNCATED, not refused, at
-    // DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT cells (rows x columns),
-    // whichever is reached first - the row cap is min(rows, cells/columns).
+    // Mirrors getdata.sas: DC_MAXOBS_WEBEDIT rows or DC_MAXCELLS_WEBEDIT
+    // cells (rows x columns), whichever is reached first - the selection is
+    // cut to min(rows, cells/columns) and flagged in sasparams.
     // Both options are read from MPE_CONFIG, the way the real service picks
     // them up from the settings program it generates - so editing the option
     // in DC changes what the mock serves, and an option that is absent (or
diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index c0de9d7..27957fa 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -272,15 +272,13 @@ select upcase(loadtype)
 
   %mpe_runhook(PRE_EDIT_HOOK)
 
-  /* A selection over a limit is served truncated, not refused: the browser
-    gets the first rows within the limit and sasparams carries a warning the
-    EDIT screen shows against the table name.  Truncating bounds the browser's
-    work the same way an abort did, without leaving the user with no grid and
-    no explanation of what happened.  A wide table exhausts the cell budget
-    long before it reaches the row limit - 250 rows of a 1000 column table is
-    a quarter of a million cells - so the row cap is min(rows, cells/columns).
-    Counting after the PRE_EDIT_HOOK means the cap covers anything the hook
-    added to work.out. */
+  /* Serve the selection truncated, at whichever limit binds first, and carry
+    the reason in sasparams for the EDIT screen to show against the table
+    name.  Bounding the browser's work is the point: a wide table exhausts the
+    cell budget long before it reaches the row limit - 250 rows of a 1000
+    column table is a quarter of a million cells - so the row cap is
+    min(DC_MAXOBS_WEBEDIT, DC_MAXCELLS_WEBEDIT / columns).  Counting after the
+    PRE_EDIT_HOOK means the cap covers what the hook added to work.out. */
   %let obscnt=%mf_getattrn(work.out,NLOBS);
   %let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
   %let maxrows=&DC_MAXOBS_WEBEDIT;
@@ -309,8 +307,8 @@ select upcase(loadtype)
                             ,Str2=&pk
                           );
   %put sourcevars=&sourcevars;
-  /* the payload is capped here, at &maxrows - whichever of the row and cell
-    limits that works out to - this is what goes to the browser */
+  /* the payload is capped at &maxrows - whichever limit that works out to be
+    - this is what goes to the browser */
   options obs=&maxrows;
   data outdata;
     /* pk fields come first */
diff --git a/sas/sasjs/services/editors/getdata.test.1.sas b/sas/sasjs/services/editors/getdata.test.1.sas
index b153a88..dee811e 100644
--- a/sas/sasjs/services/editors/getdata.test.1.sas
+++ b/sas/sasjs/services/editors/getdata.test.1.sas
@@ -4,10 +4,10 @@
 
   @details The EDIT screen is bounded by two limits - rows
   (DC_MAXOBS_WEBEDIT) and cells, rows x columns (DC_MAXCELLS_WEBEDIT) -
-  whichever is reached first.  A selection over either is SERVED TRUNCATED,
-  not refused: the grid gets the first rows that fit, and sasparams carries
-  TRUNCATED=1 plus a TRUNC_MSG naming the limit, which the EDIT screen shows
-  as a warning against the table name.
+  whichever is reached first.  A selection over either is SERVED TRUNCATED:
+  the grid gets the first rows that fit, and sasparams carries TRUNCATED=1
+  plus a TRUNC_MSG naming the limit, which the EDIT screen shows as a warning
+  against the table name.
 
   Test 1 uses MPE_X_TEST (several hundred rows, few columns) so the row
   limit alone decides the cap.  Test 2 uses the fixture dctest.wideboy (1000

From 66c233602f6ae89c1c46e31b48bff86a0f946cad Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 16:56:27 +0000
Subject: [PATCH 44/48] style(sas): keep the truncation message within the lint
 line limit

The message is built in three short assignments rather than one 161
character %let, which .sasjslint rejects (maxLineLength 107).  The
resulting text is unchanged.
---
 sas/sasjs/services/editors/getdata.sas | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index 27957fa..5c0ab84 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -295,7 +295,11 @@ select upcase(loadtype)
   %let truncated=0;
   %if %sysevalf(&obscnt gt &maxrows) %then %let truncated=1;
   %let trunc_msg=;
-  %if &truncated=1 %then %let trunc_msg=Only the first &maxrows rows are loaded - the &limitvar limit has been reached.  Filter the table to work with the rest.;
+  %if &truncated=1 %then %do;
+    %let trunc_msg=Only the first &maxrows rows are loaded;
+    %let trunc_msg=&trunc_msg - the &limitvar limit has been reached.;
+    %let trunc_msg=&trunc_msg  Filter the table to work with the rest.;
+  %end;
 
 
   /* order pk fields at start of table.  The delete flag is NOT added here:

From 7df4cb29cce625abcacdf2494728d8d86f8f0e1b Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 17:44:56 +0000
Subject: [PATCH 45/48] chore(getdata): declare mpeinit and mpeterm, correct
 the fixture counts

The doxygen list omitted mpeinit and mpeterm, which the service calls - both
are now declared the way the other services that use mpeinit declare it.  The
compiler inlines them, as it does for those services.  webout stays out of the
list: it is supplied by the framework rather than a macro in the repo, so that
one warning needs an exception in the linter.

dctest.wideboy has 1004 columns, not 1001 - row_id, char1-char500,
num1-num500, plus the loop variables field and position and the scratch
variable alphabet - so the test header's cell arithmetic is 200 x 1004 =
200,800.  The 199 row cap is unchanged, floor(200000/1004) is still 199.

The MPE_X_WIDE mock comment said 200,400 cells for 200 x 1001 columns, which
was the old counting that added the client-owned delete column.  The mock now
counts the table's own columns, so it is 200,200.
---
 sas/mocks/sasjs/services/admin/makedata.js    | 2 +-
 sas/sasjs/services/editors/getdata.sas        | 2 ++
 sas/sasjs/services/editors/getdata.test.1.sas | 6 +++---
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/sas/mocks/sasjs/services/admin/makedata.js b/sas/mocks/sasjs/services/admin/makedata.js
index b1d8422..5861385 100644
--- a/sas/mocks/sasjs/services/admin/makedata.js
+++ b/sas/mocks/sasjs/services/admin/makedata.js
@@ -2226,7 +2226,7 @@ tables.MPE_TABLES.push({
 
 // ─── MPE_X_WIDE: wide table for the EDIT cell limit ─────────────────────────
 // 200 rows x 1001 columns (ROW_ID + 500 single-character + 500 numeric) is
-// 200,400 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
+// 200,200 cells - over DC_MAXCELLS_WEBEDIT (200000) while staying under
 // DC_MAXOBS_WEBEDIT (250), so it exercises the CELL limit on its own and the
 // row limit spec can use MPE_X_SEARCH (1000 rows) instead.  Values are one
 // character / one digit on purpose: the point is width, and small values keep
diff --git a/sas/sasjs/services/editors/getdata.sas b/sas/sasjs/services/editors/getdata.sas
index 5c0ab84..59cfc4e 100755
--- a/sas/sasjs/services/editors/getdata.sas
+++ b/sas/sasjs/services/editors/getdata.sas
@@ -77,6 +77,8 @@
   @li mpe_getversions.sas
   @li mpe_filtermaster.sas
   @li mpe_runhook.sas
+  @li mpeinit.sas
+  @li mpeterm.sas
 
   @version 9.2
   @author 4GL Apps Ltd
diff --git a/sas/sasjs/services/editors/getdata.test.1.sas b/sas/sasjs/services/editors/getdata.test.1.sas
index dee811e..8f476d2 100644
--- a/sas/sasjs/services/editors/getdata.test.1.sas
+++ b/sas/sasjs/services/editors/getdata.test.1.sas
@@ -11,10 +11,10 @@
 
   Test 1 uses MPE_X_TEST (several hundred rows, few columns) so the row
   limit alone decides the cap.  Test 2 uses the fixture dctest.wideboy (1000
-  rows x 1001 columns, created and registered by testsetup) filtered to 200
-  rows: that is under the row limit but 200 x 1001 = 200,200 cells, over the
+  rows x 1004 columns, created and registered by testsetup) filtered to 200
+  rows: that is under the row limit but 200 x 1004 = 200,800 cells, over the
   200,000 cell limit, so the cell limit decides the cap -
-  floor(200000 / 1001) = 199 rows.
+  floor(200000 / 1004) = 199 rows.
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas

From 9de56f8a0b1eda8d4c354e5d86f81ad80f600441 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 17:48:47 +0000
Subject: [PATCH 46/48] chore(deps): bump Angular to 20.3.34 and
 follow-redirects to 1.16.1

The audit gate fails on advisories published against the pinned versions:
@angular/common 20.3.33 carries the XSRF token leakage reported in
GHSA-jfgx-762w-99vr, which reaches forms, platform-browser,
platform-browser-dynamic and router through their peer dependency on
common, and follow-redirects 1.16.0 carries GHSA-q3jh-wrc6-jgq8 and
GHSA-8r9p-f939-6h3c.

Both are patch bumps.  The lockfile is edited in place - the nine
@angular pins and the follow-redirects entry - rather than re-resolved,
because a full install prunes the stale entries the lockfile carries and
turns a pin change into hundreds of unrelated lines.  axios allows
^1.16.0, so the transitive bump needs no parent change.

npm audit --omit=dev reports 0 vulnerabilities at the repo root, in sas/
and in client/.  ng build and the client unit suite (589) pass.
---
 client/package-lock.json | 112 +++++++++++++++++++--------------------
 client/package.json      |  18 +++----
 2 files changed, 65 insertions(+), 65 deletions(-)

diff --git a/client/package-lock.json b/client/package-lock.json
index a9fb988..efdccb0 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -7,15 +7,15 @@
       "name": "data_controller-client",
       "hasInstallScript": true,
       "dependencies": {
-        "@angular/animations": "20.3.33",
+        "@angular/animations": "20.3.34",
         "@angular/cdk": "^20.2.14",
-        "@angular/common": "20.3.33",
-        "@angular/compiler": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/forms": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
-        "@angular/platform-browser-dynamic": "20.3.33",
-        "@angular/router": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/compiler": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/forms": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
+        "@angular/platform-browser-dynamic": "20.3.34",
+        "@angular/router": "20.3.34",
         "@cds/core": "^6.15.1",
         "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
         "@clr/icons": "^13.0.2",
@@ -65,7 +65,7 @@
         "@angular-eslint/schematics": "19.8.1",
         "@angular-eslint/template-parser": "19.8.1",
         "@angular/cli": "20.3.37",
-        "@angular/compiler-cli": "20.3.33",
+        "@angular/compiler-cli": "20.3.34",
         "@babel/plugin-proposal-private-methods": "^7.18.6",
         "@compodoc/compodoc": "^2.0.0",
         "@cypress/webpack-preprocessor": "^5.17.1",
@@ -1439,9 +1439,9 @@
       }
     },
     "node_modules/@angular/animations": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.3.33.tgz",
-      "integrity": "sha512-z9gPYm1byXYE0tMGzjjDXE4OprxvjUhV4jOXVfLLlU0Sk5ChM3GwR0m2Vh+hDWDrJ2f7nfKKTeoGlLriP7NPZA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/animations/-/animations-20.3.34.tgz",
+      "integrity": "sha512-HdDfbLwpxzx49wAWz4oOUQKSkZcFNRpWEGsDEgufZ3ExwxRk1/2QPJuvZ2CbhKeEcLPrLKUorQ8XlGrNsJmNiQ==",
       "deprecated": "@angular/animations is deprecated. Use `animate.enter` and `animate.leave` instead. For more information see: https://v22.angular.dev/guide/animations.",
       "license": "MIT",
       "dependencies": {
@@ -1451,7 +1451,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/core": "20.3.33"
+        "@angular/core": "20.3.34"
       }
     },
     "node_modules/@angular/cdk": {
@@ -1987,9 +1987,9 @@
       }
     },
     "node_modules/@angular/common": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.33.tgz",
-      "integrity": "sha512-PMfSF2j/C3gGHPp6O6c/BL+32msNzhmkzavrxtmBSxIGAOB5Q4KNktnIuolVrvVvu/0UrZj1/zwT91r/GKRaeA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/common/-/common-20.3.34.tgz",
+      "integrity": "sha512-OhB/l0cNldOvQeICqbWqRUzK5fF6zNYy44Q4cIGJZql8HXL/invGiqWLE9f+kU8z4jRisofHIGTBtyJQYYOudg==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -1998,14 +1998,14 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/core": "20.3.33",
+        "@angular/core": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/@angular/compiler": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.33.tgz",
-      "integrity": "sha512-tqKZaapDV05r2SXA6bOzTSpiNhSoqZR9Usuacs5myMpbpt4gwNMAZkgaIoIcPutnhbd3qT+Og54VCVvCOXF8pg==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/compiler/-/compiler-20.3.34.tgz",
+      "integrity": "sha512-8agPW384c/JER1ersPTmJJfcCOnlYcBbdj0MYRxNdiYe+7gtnspF3ORuJmxqCEpMS17AljSmk2skwUcYnHB6xA==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2015,9 +2015,9 @@
       }
     },
     "node_modules/@angular/compiler-cli": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.3.33.tgz",
-      "integrity": "sha512-NKE4N4ssXysHLJQ7xVqvAv/yf3JfJxaFz5H+Ue+EnYToAmTxXe7uyt2MPRIydef/H63n7O9uphVp5P5GkRxVuA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/compiler-cli/-/compiler-cli-20.3.34.tgz",
+      "integrity": "sha512-PKHzhSOyDUFh64Ull3fUwgFlxnsQNEAuRyz4KqQ8FsTGZ4pOpUaZntO+2Xn1nX/gT2whSyGBJhD3H37zA8ZGEQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -2038,7 +2038,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/compiler": "20.3.33",
+        "@angular/compiler": "20.3.34",
         "typescript": ">=5.8 <6.0"
       },
       "peerDependenciesMeta": {
@@ -2048,9 +2048,9 @@
       }
     },
     "node_modules/@angular/core": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.33.tgz",
-      "integrity": "sha512-Ld/AhODjPf8c14xUOWUnJygaZb2TsRy99ZhwFkEsRJOzqauzEFovb8Hb2MEaB1e4RolADN+HWxPvE4n+u3HFmA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/core/-/core-20.3.34.tgz",
+      "integrity": "sha512-xOvSmGoao3SVPz//R7zTrq7+WdrIGi+WNpAdxJEvJJXuSpLGS3kSUqukjY4kOMochg2v0NCEPIdCyH+yPs8JSw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2059,7 +2059,7 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/compiler": "20.3.33",
+        "@angular/compiler": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0",
         "zone.js": "~0.15.0"
       },
@@ -2073,9 +2073,9 @@
       }
     },
     "node_modules/@angular/forms": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.3.33.tgz",
-      "integrity": "sha512-UmwKS9toyYhgq8LEbvFxNBfD9dUXr+B6Y5OTwBGwIRy8RhUrOiLwrXTRr+2y/yREGgDtGAWl8zQRVMVk2BNybw==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/forms/-/forms-20.3.34.tgz",
+      "integrity": "sha512-DndQJEoORsdthFFVRvT9WHn8YcA85jkjMxCLtXyf+TDVk0zbqObbFDfrloRnJwsINqfDNsJrz23ko3YU6sjnTw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2084,16 +2084,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
     "node_modules/@angular/platform-browser": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.33.tgz",
-      "integrity": "sha512-od2uw6ee68r5TczjoYDsJuIDWoH4op3/req3sdeFtVHWdQgzf/AhM0B5ZBVRya8THon/b/SDHdhxOrUTgcszTA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/platform-browser/-/platform-browser-20.3.34.tgz",
+      "integrity": "sha512-0bd2Bj05WPoBTFM0A13IwbzxKVQPstx/N+7X0ZWKCifliyydIc7DBHu+r37ybJpl8ZsCVBe7DoR9Gon4lZ+zlw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2102,9 +2102,9 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/animations": "20.3.33",
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33"
+        "@angular/animations": "20.3.34",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34"
       },
       "peerDependenciesMeta": {
         "@angular/animations": {
@@ -2113,9 +2113,9 @@
       }
     },
     "node_modules/@angular/platform-browser-dynamic": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.3.33.tgz",
-      "integrity": "sha512-IwElnTCUW1ZZRr/pw6IRb9dRyNF8KBOh9QoGcmTk6uRAWynenYAe5yICFvgOKozqOHBarFNv8h4jXXJ/W8TcfA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/platform-browser-dynamic/-/platform-browser-dynamic-20.3.34.tgz",
+      "integrity": "sha512-vqKzExmUDf5jpIlNZRuICsLzaLWY5yTvuXsJhe97KTbC29BD71LhRGhEu7fjmR5uYy+LForpWYm5IsYlJcJAGQ==",
       "deprecated": "@angular/platform-browser-dynamic is deprecated. Use `@angular/platform-browser` instead.",
       "license": "MIT",
       "dependencies": {
@@ -2125,16 +2125,16 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/compiler": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33"
+        "@angular/common": "20.3.34",
+        "@angular/compiler": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34"
       }
     },
     "node_modules/@angular/router": {
-      "version": "20.3.33",
-      "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.3.33.tgz",
-      "integrity": "sha512-M55ibvrJSWGP7sGf+uZFvC66cxRkFlS4TjtI4Owus4gDCqW69/jhUo8pNGXf0T3QpO3iorbHScZfGwVmLATLLA==",
+      "version": "20.3.34",
+      "resolved": "https://registry.npmjs.org/@angular/router/-/router-20.3.34.tgz",
+      "integrity": "sha512-vQo2f/94nUoNMZaxGY0+fK1t9Oe7zJuxHhoMVYhDbCqKxINVmxMX/9uEHlHeu9UIE7JMazepQjRzBLo7bf9xCw==",
       "license": "MIT",
       "dependencies": {
         "tslib": "^2.3.0"
@@ -2143,9 +2143,9 @@
         "node": "^20.19.0 || ^22.12.0 || >=24.0.0"
       },
       "peerDependencies": {
-        "@angular/common": "20.3.33",
-        "@angular/core": "20.3.33",
-        "@angular/platform-browser": "20.3.33",
+        "@angular/common": "20.3.34",
+        "@angular/core": "20.3.34",
+        "@angular/platform-browser": "20.3.34",
         "rxjs": "^6.5.3 || ^7.4.0"
       }
     },
@@ -15420,9 +15420,9 @@
       "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
-      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
+      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
       "funding": [
         {
           "type": "individual",
diff --git a/client/package.json b/client/package.json
index 863d1f4..bd84d26 100644
--- a/client/package.json
+++ b/client/package.json
@@ -41,15 +41,15 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "20.3.33",
+    "@angular/animations": "20.3.34",
     "@angular/cdk": "^20.2.14",
-    "@angular/common": "20.3.33",
-    "@angular/compiler": "20.3.33",
-    "@angular/core": "20.3.33",
-    "@angular/forms": "20.3.33",
-    "@angular/platform-browser": "20.3.33",
-    "@angular/platform-browser-dynamic": "20.3.33",
-    "@angular/router": "20.3.33",
+    "@angular/common": "20.3.34",
+    "@angular/compiler": "20.3.34",
+    "@angular/core": "20.3.34",
+    "@angular/forms": "20.3.34",
+    "@angular/platform-browser": "20.3.34",
+    "@angular/platform-browser-dynamic": "20.3.34",
+    "@angular/router": "20.3.34",
     "@cds/core": "^6.15.1",
     "@clr/angular": "file:libraries/clr-angular-17.9.0.tgz",
     "@clr/icons": "^13.0.2",
@@ -99,7 +99,7 @@
     "@angular-eslint/schematics": "19.8.1",
     "@angular-eslint/template-parser": "19.8.1",
     "@angular/cli": "20.3.37",
-    "@angular/compiler-cli": "20.3.33",
+    "@angular/compiler-cli": "20.3.34",
     "@babel/plugin-proposal-private-methods": "^7.18.6",
     "@compodoc/compodoc": "^2.0.0",
     "@cypress/webpack-preprocessor": "^5.17.1",

From f38d0f80cafd2193af3105956f838e0d68558a1a Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 18:13:42 +0000
Subject: [PATCH 47/48] chore(getdata): correct the Test 2 banner column count

dctest.wideboy is 1004 columns (row_id, 500 char, 500 numeric, plus the
field and position loop variables), so the Test 2 banner says 200 x 1004 =
200,800 cells, matching the @details block.  The row cap is unchanged.
---
 sas/sasjs/services/editors/getdata.test.1.sas | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sas/sasjs/services/editors/getdata.test.1.sas b/sas/sasjs/services/editors/getdata.test.1.sas
index 8f476d2..016ad16 100644
--- a/sas/sasjs/services/editors/getdata.test.1.sas
+++ b/sas/sasjs/services/editors/getdata.test.1.sas
@@ -113,7 +113,7 @@ run;
 )
 
 /**
-  * Test 2 - cell limit (200 rows x 1001 cols = 200,200 > 200,000)
+  * Test 2 - cell limit (200 rows x 1004 cols = 200,800 > 200,000)
   */
 %wideboy_filter(200)
 %call_getdata(dctest.wideboy,&filter_rk,web2)

From 04ce1861e7821063b1be42c22f92f62aed509602 Mon Sep 17 00:00:00 2001
From: dc <dc@4gl.io>
Date: Thu, 22 Oct 2026 19:56:50 +0000
Subject: [PATCH 48/48] fix(editor): keep decimal values when an uploaded file
 is edited manually

Confirming the "source file will be discarded" warning coerced every
numeric column of the preview with parseInt, which truncated the
fraction: 0.901045951946194 became 0, and 1234.56 became 1234.
parseFloat converts the same values without dropping precision, and a
cell the parser could not read as a number is left as text so the
editor's numeric validation flags it instead of showing NaN.

Adds an Excel spec case that reads the first data row before and after
the switch to edit mode and asserts the decimal value survives.
---
 client/cypress/e2e/excel.cy.ts            | 33 +++++++++++++++++++++++
 client/src/app/editor/editor.component.ts | 10 +++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/client/cypress/e2e/excel.cy.ts b/client/cypress/e2e/excel.cy.ts
index 0f08a7e..77f4b72 100644
--- a/client/cypress/e2e/excel.cy.ts
+++ b/client/cypress/e2e/excel.cy.ts
@@ -399,6 +399,26 @@ context('excel tests: ', function () {
     })
   })
 
+  it('25 | Keeps decimals when the upload preview becomes a manual edit', () => {
+    openTableFromTree(libraryToOpenIncludes, 'mpe_x_test')
+
+    // Entering manual edit mode discards the source file but keeps the values
+    // parsed from it. SOME_NUM holds 0.901045951946194 in the first data row
+    // of the fixture, so a truncated or zeroed cell fails the assertion.
+    attachExcelFile('regular_excel.xlsx', () => {
+      getFirstRowCells().then((previewCells) => {
+        expect(previewCells).to.include(decimalSomeNum)
+
+        cy.get('.btn-upload-preview.btn-primary').click()
+        cy.get('.modal-footer .btn-success-outline').click()
+
+        getFirstRowCells().then((editCells) => {
+          expect(editCells).to.include(decimalSomeNum)
+        })
+      })
+    })
+  })
+
   // Large files break Cypress
 
   // it ('? | Uploads Excel with size of 5MB', (done) => {
@@ -476,6 +496,19 @@ const attachExcelFile = (excelFilename: string, callback?: any) => {
     })
 }
 
+const decimalSomeNum = '0.901045951946194'
+
+// The row's cell texts, without depending on column positions: the preview
+// grid renders row-status and delete cells in an order that varies.
+const getFirstRowCells = () =>
+  cy
+    .get('#hotTable', { timeout: longerCommandTimeout })
+    .find('div.ht_master.handsontable tbody tr')
+    .eq(0)
+    .then(($row) =>
+      Array.from($row[0].children).map((cell: any) => cell.innerText.trim())
+    )
+
 const submitExcel = (callback?: any) => {
   cy.get('.buttonBar button.preview-submit', { timeout: longerCommandTimeout })
     .click()
diff --git a/client/src/app/editor/editor.component.ts b/client/src/app/editor/editor.component.ts
index 02b9971..4f68a63 100644
--- a/client/src/app/editor/editor.component.ts
+++ b/client/src/app/editor/editor.component.ts
@@ -1363,8 +1363,14 @@ export class EditorComponent
             colSpecs.TYPE === 'num' &&
             !colSpecs.DDTYPE.includes('TIME') &&
             !colSpecs.DDTYPE.includes('DATE')
-          )
-            row[colKey] = parseInt(row[colKey])
+          ) {
+            const numericValue = parseFloat(row[colKey])
+
+            // A cell the file parser could not read as a number stays as it
+            // was: the editor's numeric validation then flags it as text,
+            // where a NaN would render as an empty cell.
+            if (!Number.isNaN(numericValue)) row[colKey] = numericValue
+          }
         }
       }
     }