fix: correctly applying deletes on viya, also

adding more info to the staged directory in relation to deletes. Also refactoring the demo data.
2026-02-09 23:23:08 +00:00
parent d41f88f8bf
commit 46cdeb0bab
8 changed files with 685 additions and 489 deletions
--- a/client/package-lock.json
+++ b/client/package-lock.json
--- a/client/package.json
+++ b/client/package.json
@@ -51,7 +51,7 @@
    "@clr/icons": "^13.0.2",
    "@clr/ui": "file:libraries/clr-ui-17.9.0.tgz",
    "@handsontable/angular-wrapper": "16.0.1",
-    "@sasjs/adapter": "^4.16.2",
+    "@sasjs/adapter": "^4.16.3",
    "@sasjs/utils": "^3.5.3",
    "@sheet/crypto": "file:libraries/sheet-crypto.tgz",
    "@types/d3-graphviz": "^2.6.7",
--- a/sas/deploy/makeDataServer.json
+++ b/sas/deploy/makeDataServer.json
@@ -2,7 +2,7 @@
  "fromjs": [
    {
      "ADMIN": "DCDEFAULT",
-      "DCPATH": "/tmp/mihajlo/dcserverfrs"
+      "DCPATH": "/tmp/dcdata"
    }
  ]
 }
--- a/sas/package-lock.json
+++ b/sas/package-lock.json
@@ -6,8 +6,8 @@
    "": {
      "name": "dc-sas",
      "dependencies": {
-        "@sasjs/cli": "^4.12.15",
-        "@sasjs/core": "^4.59.9"
+        "@sasjs/cli": "^4.13.1",
+        "@sasjs/core": "^4.60.0"
      }
    },
    "node_modules/@coolaj86/urequest": {
@@ -31,29 +31,62 @@
      }
    },
    "node_modules/@sasjs/adapter": {
-      "version": "4.16.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.16.0.tgz",
-      "integrity": "sha512-DzF/+s++FtSfuBmONicBbgeKI8feiwDOm1iKWlcDlmHCPmHIoj1IbI0v2fGktzurnE37/vkyp6dvHO+FhwI87Q==",
-      "hasInstallScript": true,
+      "version": "4.16.3",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-4.16.3.tgz",
+      "integrity": "sha512-xcoZT9qZhF6pXvXx4bHxbmauLdEHng8pSlTK4F6asUkHNR5uzeSvY6znA1yJqK+8FFtsVILyvMQyGyhWw6WsOA==",
      "license": "ISC",
      "dependencies": {
-        "@sasjs/utils": "3.5.2",
-        "axios": "1.12.2",
+        "@sasjs/utils": "3.5.6",
+        "axios": "^1.13.5",
        "axios-cookiejar-support": "5.0.5",
        "form-data": "4.0.4",
        "https": "1.0.0",
        "tough-cookie": "4.1.3"
      }
    },
-    "node_modules/@sasjs/cli": {
-      "version": "4.12.15",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.12.15.tgz",
-      "integrity": "sha512-vA2YY+U9niquU7qkcDSxhOKcoased+4gePQyg8eWDRxCoWJx4mPv/y86J/cT2DmgAzjEt5JJAT8IUUtZcFJX+A==",
-      "hasInstallScript": true,
+    "node_modules/@sasjs/adapter/node_modules/@sasjs/utils": {
+      "version": "3.5.6",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-3.5.6.tgz",
+      "integrity": "sha512-jx8zWSOysDD66vTjA0BWiZ8bcFqmqh8F+56fUCgLmJhm89eDbKrGF3mDKMQx3UE7d2+gxp9xYhJCdaBWz0Dlxw==",
      "license": "ISC",
      "dependencies": {
-        "@sasjs/adapter": "^4.16.0",
-        "@sasjs/core": "4.59.9",
+        "@fast-csv/format": "4.3.5",
+        "@types/fs-extra": "11.0.4",
+        "@types/prompts": "2.0.13",
+        "chalk": "4.1.1",
+        "cli-table": "0.3.6",
+        "consola": "2.15.0",
+        "find": "0.3.0",
+        "fs-extra": "11.3.0",
+        "jwt-decode": "3.1.2",
+        "prompts": "2.4.1",
+        "valid-url": "1.0.9"
+      }
+    },
+    "node_modules/@sasjs/adapter/node_modules/chalk": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz",
+      "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==",
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/@sasjs/cli": {
+      "version": "4.13.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-4.13.1.tgz",
+      "integrity": "sha512-KvaKB551d3RjgD4upAy5c8iu7lxnL45pkUkueRf5cM1bneyyNIVR6lZ3E17UKnzFbeZxA/x3EPH9ObervpyHQw==",
+      "license": "ISC",
+      "dependencies": {
+        "@sasjs/adapter": "4.16.3",
+        "@sasjs/core": "4.60.1",
        "@sasjs/lint": "2.4.3",
        "@sasjs/utils": "3.5.2",
        "adm-zip": "0.5.10",
@@ -78,9 +111,9 @@
      }
    },
    "node_modules/@sasjs/core": {
-      "version": "4.59.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.59.9.tgz",
-      "integrity": "sha512-dHsEbMCHRjrUAGrYXH/a93F8iFogMEQuIZ2qFBFNzM0LwjwDdcTK0kCRMarBRc3oNo4JsaHqOltYbYaZ06GmCw==",
+      "version": "4.60.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-4.60.1.tgz",
+      "integrity": "sha512-PjHg0w8cV3q1ZLe1CxHUiSC+H44frTnovkFoGHxKPjNQ7KWYn5Hu7LkQuf2JsGoj85XCjeJjr/daB2yDrbJakQ==",
      "license": "MIT"
    },
    "node_modules/@sasjs/lint": {
@@ -239,14 +272,13 @@
      "license": "MIT"
    },
    "node_modules/axios": {
-      "version": "1.12.2",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz",
-      "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==",
+      "version": "1.13.5",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz",
+      "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==",
      "license": "MIT",
-      "peer": true,
      "dependencies": {
-        "follow-redirects": "^1.15.6",
-        "form-data": "^4.0.4",
+        "follow-redirects": "^1.15.11",
+        "form-data": "^4.0.5",
        "proxy-from-env": "^1.1.0"
      }
    },
@@ -269,6 +301,22 @@
        "tough-cookie": ">=4.0.0"
      }
    },
+    "node_modules/axios/node_modules/form-data": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
+      "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
    "node_modules/balanced-match": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
--- a/sas/package.json
+++ b/sas/package.json
@@ -28,7 +28,7 @@
  },
  "private": true,
  "dependencies": {
-    "@sasjs/cli": "^4.12.15",
-    "@sasjs/core": "^4.59.9"
+    "@sasjs/cli": "^4.13.1",
+    "@sasjs/core": "^4.60.0"
  }
 }
--- a/sas/sasjs/macros/bitemporal_closeouts.sas
+++ b/sas/sasjs/macros/bitemporal_closeouts.sas
@@ -19,6 +19,8 @@
    given that BUS_FROM should be supplied in the PK.
  @param [in] tech_from= (tx_from_dttm) Technical FROM datetime variable.
    Required on BASE table only.
+  @param [in] AUDITFOLDER= (0) Unquoted path to a directory into which a copy of
+    the generated delete program will be written

  <h4> Global Variables </h4>
  @li `dc_dttmtfmt`
@@ -29,6 +31,8 @@
  @li mf_existvar.sas
  @li mf_getattrn.sas
  @li mf_getengine.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
  @li mf_getuser.sas
  @li mf_getvartype.sas
  @li mp_lockanytable.sas
@@ -54,8 +58,7 @@
                              /* Should INCLUDE BUS_FROM field if relevant. */
      ,NOW=DEFINE
      ,FILTER=                /* supply a filter to limit the update */
-      ,outdest=               /* supply an unquoted filepath/filename.ext to get
-                                a text file containing the update statements */
+      ,AUDITFOLDER=0
      ,loadtype=
      ,loadtarget=YES      /* if <> YES will return without changing anything */
 );
@@ -126,6 +129,17 @@ run;
  msg=Aborted due to SYSCC=&SYSCC status
 )

+/* set up folder */
+%local tmplib;%let tmplib=%mf_getuniquelibref();
+%if "&AUDITFOLDER"="0" %then %do;
+  filename tmp temp lrecl=10000;
+  libname &tmplib (work);
+%end;
+%else %do;
+  filename tmp "&AUDITFOLDER/deleterecords.sas" lrecl=10000;
+  libname &tmplib "&AUDITFOLDER";
+%end;
+
 /**
  * Create closeout statements.  If UPDATE approach and CAS engine, use the
  * DeleteRows action (as regular SQL deletes are not supported).
@@ -137,22 +151,25 @@ run;
 %let update_cnt=0;
 %let etype=%mf_getengine(&base_lib);
 %put &=etype;
-filename tmp temp;
+
 %if &loadtype=UPDATE and &etype=CAS %then %do;
  /* create temp table for deletions */
  %local delds;%let delds=%mf_getuniquename(prefix=DC);
-  data casuser.&delds;
+  data casuser.&delds &tmplib..deleterecords;
    set work.___closeout1;
+    keep &pk;
  run;
  /* build the proc */
  data _null_;
    file tmp;
+    put "/* libname approve '&AUDITFOLDER'; */";
    put 'proc cas;table.deleteRows result=r/ table={' ;
    put "  caslib='&base_lib',name='&base_dsn',where='1=1',";
    put "  whereTable={caslib='CASUSER',name='&delds'}";
    put "};";
    put "call symputx('update_cnt',r.RowsDeleted);";
    put "quit;";
+    put "data;set casuser.&delds;putlog (_all_)(=);run;";
    put '%put &=update_cnt;';
    put "proc sql;drop table CASUSER.&delds;";
    stop;
@@ -193,12 +210,6 @@ filename tmp temp;
  run;
 %end;

-data _null_;
-  infile tmp;
-  input;
-  putlog _infile_;
-run;
-
 %if &loadtarget ne YES %then %return;

 /* ensure we have a lock */
--- a/sas/sasjs/macros/bitemporal_dataloader.sas
+++ b/sas/sasjs/macros/bitemporal_dataloader.sas
@@ -512,6 +512,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
      ,NOW=&dbnow
      ,loadtarget=&loadtarget
      ,loadtype=&loadtype
+      ,AUDITFOLDER=&dc_staging_area/&ETLSOURCE
    )
  %end;
 %end;
@@ -574,6 +575,7 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
      ,NOW=&dbnow
      ,loadtarget=&loadtarget
      ,loadtype=&loadtype
+      ,AUDITFOLDER=&dc_staging_area/&ETLSOURCE
    )
  %end;
 %end;
--- a/sas/sasjs/services/admin/demodata.sas
+++ b/sas/sasjs/services/admin/demodata.sas
@@ -20,7 +20,7 @@
  @li mp_abort.sas
  @li mp_binarycopy.sas
  @li mp_replace.sas
-  @li mx_createwebservice.sas
+  @li mx_createjob.sas

  @author 4GL Apps Ltd
  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
@@ -33,13 +33,16 @@

 %let dcdemoflag=1;
 %let demolib=PUBLIC;
+
 options dlcreatedir;
 %mpeinit()

-/* to enable - delete here */
+/* to enable - delete flag below and add libs to SETTINGS */
 %let dcdemoflag=0;
 libname &demolib "%sysfunc(pathname(&dc_libref))/&demolib";

+%let joblib=HOOKLIB;
+libname &joblib "%sysfunc(pathname(&dc_libref))/&joblib";

 %mp_abort(iftrue= (&dcdemoflag ne 1)
  ,mac=&_program
@@ -58,8 +61,6 @@ data work.cars_ext(index=(carspk=(make model PRODUCTIONDATE) /unique));
    NOTES                            length= $30
    CHECKBOXVAR                      length= $3
    PRODUCTIONDATE                   length= 8      format=DATE9.
-    EDITOR                           length= $30
-    APPROVER                         length= $30
  ;
  set sashelp.cars;
  retain comment 'n/a';
@@ -74,8 +75,6 @@ data work.cars_ext(index=(carspk=(make model PRODUCTIONDATE) /unique));
  else POTENTIALBUY='No';
  make=cats(make);
  model=cats(model);
-  EDITOR='SYSTEM';
-  APPROVER='n/a';

  array cntrs (4) $ 60 _temporary_ ( "Germany" "France" "Poland" "Italy");
  if origin='USA' then country='USA';
@@ -92,16 +91,12 @@ data work.COUNTRIES(index=(countriespk=(origin country) /unique));
  attrib
    ORIGIN                           length= $6
    COUNTRY                          length= $30
-    EDITOR                           length= $30
-    APPROVER                         length= $30
  ;
  infile cards dsd;
  input
    ORIGIN                           :$char.
    COUNTRY                          :$char.
  ;
-EDITOR='SYSTEM';
-APPROVER='n/a';
 datalines4;
 Europe,Germany
 Europe,France
@@ -116,6 +111,7 @@ run;
 data work.jobdata;
  length message job $100;
  call missing(of _all_);
+  stop;
 run;

 %let engine_type=%mf_getengine(&demolib);
@@ -129,22 +125,18 @@ run;
    table.tableExists result=r2 / name="COUNTRIES" caslib="PUBLIC";
    if r2.exists then
      table.dropTable / name="COUNTRIES" caslib="PUBLIC" quiet=TRUE;
-
-    table.tableExists result=r3 / name="JOBDATA" caslib="PUBLIC";
-    if r3.exists then
-      table.dropTable / name="JOBDATA" caslib="PUBLIC" quiet=TRUE;
  quit;
  proc casutil;
    load data=work.CARS_EXT outcaslib="PUBLIC" casout="CARS_EXT" promote;
    load data=work.COUNTRIES outcaslib="PUBLIC" casout="COUNTRIES" promote;
-    load data=work.COUNTRIES outcaslib="PUBLIC" casout="JOBDATA" promote;
  run;
+  data &joblib..JOBDATA; set work.JOBDATA;run;
 %end;
 %else %do;
  options replace;
  data &demolib..CARS_EXT; set work.cars_ext;
  data &demolib..COUNTRIES; set work.countries;
-  data &demolib..JOBDATA; set work.JOBDATA;run;
+  data &joblib..JOBDATA; set work.JOBDATA;run;
 %end;

 %let apploc=%mf_getapploc(&_program);
@@ -296,7 +288,7 @@ data _null_;
  put '  if first.display_index then forced_value=1;';
  put 'run;';
 run;
-%mx_createwebservice(path=&apploc/demo
+%mx_createjob(path=&apploc/demo
  ,name=origin,code=vldtr
 )
 proc sql;
@@ -317,58 +309,40 @@ parmcards4;
  proc sql;
  insert into XXXXXXXXXXX.JOBDATA values(
    "&orig_libds (%mf_nobs(work.out) obs) fetched for editing %trim(
-    )by %mf_getUser() at %mf_uid()","&pgmloc");
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
 ;;;;
 filename f1 temp;
 %mp_binarycopy(inref=ft15f001, outref=f1)
-%mp_replace("%sysfunc(pathname(f1))", findvar=fvar, replacevar=demolib)
-%mx_createwebservice(path=&apploc/demo,name=PREEDIT,code=f1)
+%mp_replace("%sysfunc(pathname(f1))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=PREEDIT,code=f1)
 filename ft15f001 clear;

 /* POST EDIT JOB */
 filename ft15f001 temp;
 parmcards4;
-
-  /* modify staging_ds to apply changes */
-  data work.staging_ds;
-    set work.staging_ds;
-    if "%scan(&orig_libds,1,.)"="XXXXXXXXXXX" then do;
-      if "%scan(&orig_libds,2,.)" in ('CARS_EXT','COUNTRIES')
-      then EDITOR="%mf_getuser()";
-    end;
-  run;
-
  proc sql;
  insert into XXXXXXXXXXX.JOBDATA values(
    "&orig_libds (%mf_nobs(work.staging_ds) obs) staged %trim(
-    )by %mf_getUser() at %mf_uid()","&pgmloc");
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
 ;;;;
 filename f2 temp;
 %mp_binarycopy(inref=ft15f001, outref=f2)
-%mp_replace("%sysfunc(pathname(f2))", findvar=fvar, replacevar=demolib)
-%mx_createwebservice(path=&apploc/demo,name=POSTEDIT,code=f2)
+%mp_replace("%sysfunc(pathname(f2))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=POSTEDIT,code=f2)
 filename ft15f001 clear;

 /* PRE APPROVE JOB */
 filename ft15f001 temp;
 parmcards4;
-  /* modify staging_ds to apply changes */
-  data work.staging_ds;
-    set work.staging_ds;
-    if "%scan(&orig_libds,1,.)"="XXXXXXXXXXX" then do;
-      if "%scan(&orig_libds,2,.)" in ('CARS_EXT','COUNTRIES')
-      then APPROVER="%mf_getuser()";
-    end;
-  run;
  proc sql;
  insert into XXXXXXXXXXX.JOBDATA values(
    "&orig_libds (%mf_nobs(work.staging_ds) obs) under review by %trim(
-    )by %mf_getUser() at %mf_uid()","&pgmloc");
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
 ;;;;
 filename f3 temp;
 %mp_binarycopy(inref=ft15f001, outref=f3)
-%mp_replace("%sysfunc(pathname(f3))", findvar=fvar, replacevar=demolib)
-%mx_createwebservice(path=&apploc/demo,name=PREAPPROVE,code=f3)
+%mp_replace("%sysfunc(pathname(f3))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=PREAPPROVE,code=f3)
 filename ft15f001 clear;

 /* POST APPROVE JOB */
@@ -377,10 +351,10 @@ parmcards4;
  proc sql;
  insert into XXXXXXXXXXX.JOBDATA values(
    "&orig_libds (%mf_nobs(work.staging_ds) obs) approved by %trim(
-    )by %mf_getUser() at %mf_uid()","&pgmloc");
+    )by %mf_getUser() at %sysfunc(datetime(),datetime19.)","&pgmloc");
 ;;;;
 filename f4 temp;
 %mp_binarycopy(inref=ft15f001, outref=f4)
-%mp_replace("%sysfunc(pathname(f4))", findvar=fvar, replacevar=demolib)
-%mx_createwebservice(path=&apploc/demo,name=POSTAPPROVE,code=f4)
+%mp_replace("%sysfunc(pathname(f4))", findvar=fvar, replacevar=joblib)
+%mx_createjob(path=&apploc/demo,name=POSTAPPROVE,code=f4)
 filename ft15f001 clear;