feat: capturing catalog specific information, closes #159

BREAKING CHANGE:  Introduction of 4 new tables for capturing information related to catalogs and their objects.  Migration script prepared and available in the DB folder (usual place)

sas/package.json

@@ -22,7 +22,7 @@
     "serverdata": "sasjs request services/admin/makedata -d deploy/makeDataServer.json -l sasjsresults/makedata_server.log -o sasjsresults/makedata_server.json -t server",
     "viya": "npm run viyacbd && sasjs test",
     "viyacbd": "sasjs folder delete /Public/app/viya && sasjs cbd -t viya && npm run viyadata",
-    "viyadata": "sasjs request services/admin/makedata -d deploy/makeDataViya.json -l sasjsresults/makedata.log -o sasjsresults/output.json",
+    "viyadata": "sasjs request services/admin/makedata -d deploy/makeDataViya.json -l sasjsresults/makedata.log -o sasjsresults/output.json -t viya",
     "v4data": "sasjs request services/admin/makedata -d deploy/makeDataV4.json -l sasjsresults/makedatav4.log -o sasjsresults/outputv4.json -t v4",
     "sasdocs": "sasjs doc && ./sasjs/utils/deploydocs.sh"
   },
@@ -31,4 +31,4 @@
     "@sasjs/cli": "^4.12.8",
     "@sasjs/core": "^4.59.1"
   }
-}
+}

sas/sasjs/db/datactrl/mpe_datastatus_cats.ddl

@@ -14,7 +14,7 @@ create table &curlib..mpe_datastatus_CATS(
     libref char(8) label='Library Name',
     memname char(64) label='Member Name',
     nobjs num label='Number of objects',
-    created num format=DATETIME. informat=DATETIME. label='Date Created',
-    modified num format=DATETIME. informat=DATETIME. label='Date Modified',
+    created num format=DATETIME. label='Date Created',
+    modified num format=DATETIME. label='Date Modified',
   constraint pk_mpe_datastatus_CATS
       primary key(libref,memname,tx_to));

sas/sasjs/db/datactrl/mpe_datastatus_objs.ddl

@@ -15,8 +15,8 @@ create table &curlib..mpe_datastatus_OBJS(
     memname char(64) label='Member Name',
     objname char(32) label='Object Name',
     objtype char(8) label='Object Type',
-    created num format=DATETIME. informat=DATETIME. label='Date Created',
-    modified num format=DATETIME. informat=DATETIME. label='Date Modified',
+    created num format=DATETIME. label='Date Created',
+    modified num format=DATETIME. label='Date Modified',
     level num label='Library Concatenation Level',
   constraint pk_mpe_datastatus_OBJS
       primary key(libref,memname,objname,objtype,tx_to));

sas/sasjs/db/datactrl/mpe_datastatus_tabs.ddl

@@ -14,8 +14,8 @@ create table &curlib..mpe_datastatus_tabs(
     libref char(8) label='Library Name',
     dsn char(64) label='Member Name',
     filesize num format=SIZEKMG. label='Size of file',
-    crdate num format=DATETIME. informat=DATETIME. label='Date Created',
-    modate num format=DATETIME. informat=DATETIME. label='Date Modified',
+    crdate num format=DATETIME. label='Date Created',
+    modate num format=DATETIME. label='Date Modified',
     nobs num label='Number of Physical (Actual, inc. deleted) Observations',
   constraint pk_mpe_datastatus_tabs
       primary key(libref,dsn,tx_to));

sas/sasjs/db/migrations/20200428_datacatalog.sas

@@ -52,7 +52,7 @@ create table dc.mpe_datacatalog_vars(
   constraint pk
       primary key(libref,dsn,name,tx_to));
 
-  create table dc.mpe_datastatus_libs(
+  create table dc.mpe_datastatus_cats(
     TX_FROM float format=datetime19.,
     TX_TO float format=datetime19.,
     libref char(8) label='Library Name',
@@ -67,8 +67,8 @@ create table dc.mpe_datacatalog_vars(
     libref char(8) label='Library Name',
     dsn char(64) label='Member Name',
     filesize num format=SIZEKMG. label='Size of file',
-    crdate num format=DATETIME. informat=DATETIME. label='Date Created',
-    modate num format=DATETIME. informat=DATETIME. label='Date Modified',
+    crdate num format=DATETIME. label='Date Created',
+    modate num format=DATETIME. label='Date Modified',
     nobs num label='Number of Physical (Actual, inc. deleted) Observations',
   constraint pk
       primary key(libref,dsn,tx_to));

sas/sasjs/db/migrations/20250611_v7.0_release.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief migration script to move from v6.8.2 to v7.0 of data controller
+
+  BREAKING CHANGE - 4 additional tables for capturing catalog information
+
+  Be sure to run this using the correct system account
+  (eg the regular DC account)
+
+  On SAS 9 you may wish to run proc metalib or refresh in DI Studio afterwards
+  to see the new tables in the VIEW page
+
+      proc metalib;
+        omr (library="&DC_LIBNAME");
+        folder="&root/data";
+        update_rule=(delete);
+      run;
+
+**/
+
+%let dclib=YOURDCLIB;
+
+libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH";
+
+proc sql;
+create table &dclib..mpe_datacatalog_CATS(
+    TX_FROM float format=datetime19.,
+    TX_TO float format=datetime19.,
+    libref char(8) label='Library Name',
+    memname char(64) label='Member Name',
+  constraint pk_mpe_datacatalog_CATS
+      primary key(libref,memname,tx_to));
+
+create table &dclib..mpe_datacatalog_OBJS(
+    TX_FROM float format=datetime19.,
+    TX_TO float format=datetime19.,
+    libref char(8) label='Library Name',
+    memname char(64) label='Member Name',
+    objname char(32) label='Object Name',
+    objtype char(8) label='Object Type',
+    objdesc char(256) label='Object Description',
+    alias char(32) label='Object Alias',
+  constraint pk_mpe_datacatalog_OBJS
+      primary key(libref,memname,objname,objtype,tx_to));
+
+create table &dclib..mpe_datastatus_CATS(
+    TX_FROM float format=datetime19.,
+    TX_TO float format=datetime19.,
+    libref char(8) label='Library Name',
+    memname char(64) label='Member Name',
+    nobjs num label='Number of objects',
+    created num format=DATETIME. label='Date Created',
+    modified num format=DATETIME. label='Date Modified',
+  constraint pk_mpe_datastatus_CATS
+      primary key(libref,memname,tx_to));
+
+create table &dclib..mpe_datastatus_OBJS(
+    TX_FROM float format=datetime19.,
+    TX_TO float format=datetime19.,
+    libref char(8) label='Library Name',
+    memname char(64) label='Member Name',
+    objname char(32) label='Object Name',
+    objtype char(8) label='Object Type',
+    created num format=DATETIME. label='Date Created',
+    modified num format=DATETIME. label='Date Modified',
+    level num label='Library Concatenation Level',
+  constraint pk_mpe_datastatus_OBJS
+      primary key(libref,memname,objname,objtype,tx_to));

sas/sasjs/macros/mpe_dsmeta.sas

@@ -14,14 +14,31 @@
 **/
 
 %macro mpe_dsmeta(libds, outds=dsmeta);
-  %local ddsd ddld notes lenstmt;
+  %local ddsd ddld notes lenstmt memname;
   %let lenstmt=length ods_table $18 name $100 value $1000;
   %let libds=%upcase(&libds);
-  %mp_dsmeta(&libds, outds=&outds)
+
+  %if "%scan(&libds,2,-)" ne "FC" %then %do;
+    %let memname=%scan(&libds,2,.);
+    %mp_dsmeta(&libds, outds=&outds)
+  %end;
+  %else %do;
+    %let memname=%scan(&libds,2,.-);
+    data &outds;
+      &lenstmt;
+      set sashelp.vcatalg;
+      ods_table=cats(objtype);
+      name=cats(objname);
+      value=catx(' ',objdesc,'(modified:',put(modified,datetime19.),')');
+      where libname="%scan(&libds,1,.)" and memname="&memname";
+      keep ods_table name value;
+    run;
+    proc sort; by ods_table name;run;
+  %end;
 
   data _null_;
     set &mpelib..mpe_datadictionary;
-    where &dc_dttmtfmt < tx_to & dd_source=%upcase("&libds") & dd_type='TABLE';
+    where &dc_dttmtfmt < tx_to & dd_source="&memname" & dd_type='TABLE';
     call symputx('ddsd',dd_shortdesc,'l');
     call symputx('ddld',dd_longdesc,'l');
   run;

sas/sasjs/macros/mpe_dsmeta.test.sas

@@ -34,4 +34,15 @@ run;
   outds=work.test_results
 )
 
+%mpe_dsmeta(FMTONLY.DCFMTS-FC,outds=test2)
+data work.test2;
+  set work.test2;
+  putlog (_all_)(=);
+run;
+
+%mp_assertdsobs(work.test2,
+  desc=Test 2 - records returned,
+  test=EQUALS 5,
+  outds=work.test_results
+)
 

sas/sasjs/macros/mpe_makedatamodel.sas

@@ -253,8 +253,8 @@ create table &lib..mpe_datastatus_tabs(
     libref char(8) label='Library Name',
     dsn char(64) label='Member Name',
     filesize num format=SIZEKMG. label='Size of file',
-    crdate num format=DATETIME. informat=DATETIME. label='Date Created',
-    modate num format=DATETIME. informat=DATETIME. label='Date Modified',
+    crdate num format=DATETIME. label='Date Created',
+    modate num format=DATETIME. label='Date Modified',
     nobs num label='Number of Physical (Actual, inc. deleted) Observations'
 );quit;
 proc datasets lib=&lib noprint;

sas/sasjs/macros/mpe_refreshcatalogs.sas

@@ -0,0 +1,140 @@
+/**
+  @file mpe_refreshtables.sas
+  @brief Refreshes the data catalog
+  @details Assumes library is already assigned.
+  Usage:
+
+      %mpe_refreshcatalogs(sashelp)
+
+  <h4> SAS Macros </h4>
+  @li bitemporal_dataloader.sas
+
+  @version 9.3
+  @author 4GL Apps Ltd
+**/
+
+%macro mpe_refreshcatalogs(lib,cat=#all);
+%let lib=%upcase(&lib);
+%let cat=%upcase(&cat);
+
+%put running &sysmacroname &lib for &cat;
+proc sql;
+create table work.catdata as
+  select libname as libref,
+    memname,
+    objname,
+    objtype,
+    objdesc,
+    created,
+    modified,
+    alias,
+    level
+  from dictionary.catalogs
+  where upcase(libname)="&lib"
+%if &cat ne #ALL %then %do;
+    and upcase(memname)="&cat"
+%end;
+  ;
+
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program
+  ,msg=%str(syscc=&syscc afer &lib objects extraction)
+)
+
+/* load mpe_datacatalog_CATS */
+proc sql;
+create table datacats as select distinct libref,memname from catdata;
+
+%bitemporal_dataloader(base_lib=&mpelib
+  ,base_dsn=mpe_datacatalog_CATS
+  ,append_dsn=datacats
+  ,PK=LIBREF MEMNAME
+  ,etlsource=&sysmacroname
+  ,loadtype=TXTEMPORAL
+  ,tech_from=TX_FROM
+  ,tech_to=TX_TO
+%if &cat = #ALL %then %do;
+    ,close_vars=LIBREF
+%end;
+  ,dclib=&mpelib
+)
+
+/* load mpe_datacatalog_objsS */
+proc sql;
+create table dataobjs as
+  select distinct libref,
+    memname,
+    objname,
+    objtype,
+    objdesc,
+    alias
+  from catdata;
+quit;
+
+%bitemporal_dataloader(base_lib=&mpelib
+  ,base_dsn=mpe_datacatalog_OBJS
+  ,append_dsn=dataobjs
+  ,PK=LIBREF MEMNAME OBJNAME OBJTYPE
+  ,etlsource=&sysmacroname
+  ,loadtype=TXTEMPORAL
+  ,tech_from=TX_FROM
+  ,tech_to=TX_TO
+%if &cat = #ALL %then %do;
+    ,close_vars=LIBREF MEMNAME
+%end;
+  ,dclib=&mpelib
+)
+
+%put load mpe_datastatus_OBJS;
+proc sql;
+create table statusobjs as
+  select distinct libref,
+    memname,
+    objname,
+    objtype,
+    created,
+    modified,
+    level
+  from catdata;
+
+%bitemporal_dataloader(base_lib=&mpelib
+  ,base_dsn=mpe_datastatus_OBJS
+  ,append_dsn=statusobjs
+  ,PK=LIBREF MEMNAME OBJNAME OBJTYPE
+  ,etlsource=&sysmacroname
+  ,loadtype=TXTEMPORAL
+  ,tech_from=TX_FROM
+  ,tech_to=TX_TO
+%if &cat = #ALL %then %do;
+    ,close_vars=LIBREF MEMNAME
+%end;
+  ,dclib=&mpelib
+)
+
+%put load mpe_datastatus_cats;
+proc sql;
+create table statuscats as
+  select libref,
+    memname,
+    count(*) as nobjs,
+    min(created) as created,
+    max(modified) as modified
+  from catdata
+  group by 1,2;
+
+%bitemporal_dataloader(base_lib=&mpelib
+  ,base_dsn=mpe_datastatus_cats
+  ,append_dsn=statuscats
+  ,PK=LIBREF MEMNAME
+  ,etlsource=&sysmacroname
+  ,loadtype=TXTEMPORAL
+  ,tech_from=TX_FROM
+  ,tech_to=TX_TO
+%if &cat = #ALL %then %do;
+    ,close_vars=LIBREF
+%end;
+  ,dclib=&mpelib
+)
+
+
+%mend mpe_refreshcatalogs;

sas/sasjs/macros/mpe_refreshcatalogs.test.sas

@@ -0,0 +1,38 @@
+/**
+  @file
+  @brief Testing mpe_refreshcatalogs macro
+
+  <h4> SAS Macros </h4>
+  @li mpe_refreshcatalogs.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_ds2md.sas
+
+  @author 4GL Apps Ltd
+  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
+    and may not be re-distributed or re-sold without the express permission of
+    4GL Apps Ltd.
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+%mpe_refreshcatalogs(FMTONLY)
+%mp_assertscope(COMPARE,
+  desc=Checking macro variables against previous snapshot
+)
+
+/* make sure that the process picks up the catalog */
+proc sql noprint;
+create table work.test1 as
+  select *
+  from &mpelib..mpe_datacatalog_cats(where=(&dc_dttmtfmt. lt tx_to))
+  where libref="FMTONLY";
+%let test1=0;
+select count(*) into: test1 from work.test1;
+
+%mp_assert(
+  iftrue=(&test1>0),
+  desc=Checking fmtonly.dcfmts was picked up
+)
+
+%mp_ds2md(work.test1)

sas/sasjs/macros/mpe_refreshtables.sas

@@ -48,11 +48,6 @@ create table cols as
   ,msg=%str(syscc=&syscc afer &lib cols extraction)
 )
 
-%mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&_program
-  ,msg=%str(syscc=&syscc afer &lib indexes extraction)
-)
-
 %if &engine=SQLSVR %then %do;
   proc sql;
   connect using &lib;
@@ -175,6 +170,11 @@ create table cols as
 
 %end;
 
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&_program
+  ,msg=%str(syscc=&syscc afer &lib indexes extraction)
+)
+
 /* load columns */
 %bitemporal_dataloader(base_lib=&mpelib
   ,base_dsn=mpe_datacatalog_vars

sas/sasjs/services/editors/getdata.sas

@@ -675,7 +675,7 @@ data xl_rules;
   keep xl_column xl_rule;
 run;
 
-%mpe_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&orig_libds, outds=dsmeta)
 
 %mpe_getversions(&mpelib,
   %scan(&orig_libds,1,.),

sas/sasjs/services/public/viewdata.sas

@@ -355,7 +355,7 @@ run;
 
 %mp_getcols(&libds, outds=cols)
 
-%mpe_dsmeta(&libds, outds=dsmeta)
+%mpe_dsmeta(&orig_libds, outds=dsmeta)
 
 %mpe_getversions(&mpelib,
   %scan(&orig_libds,1,.),

sas/sasjs/targets/sas9/macros_meta/dc_refreshcatalog.sas

@@ -9,6 +9,7 @@
   <h4> SAS Macros </h4>
   @li mpe_refreshlibs.sas
   @li dc_assignlib.sas
+  @li mpe_refreshcatalogs.sas
   @li mpe_refreshtables.sas
   @li mm_getrepos.sas
 
@@ -44,6 +45,7 @@ run;
   %dc_assignlib(WRITE,&libref) /* write just in order to assign direct lib */
   %mpe_refreshlibs(lib=&libref)
   %mpe_refreshtables(&libref)
+  %mpe_refreshcatalogs(&libref)
 %end;
 %else %do xx=1 %to &repocnt;
   options metarepository=&&repo&xx;
@@ -73,6 +75,7 @@ run;
   %do i=1 %to &libcnt;
     %dc_assignlib(WRITE,&&lib&i)
     %mpe_refreshtables(&&lib&i)
+    %mpe_refreshcatalogs(&&lib&i)
   %end;
 %end;
 

sas/sasjs/targets/server/macros_server/dc_refreshcatalog.sas

@@ -9,6 +9,7 @@
   <h4> SAS Macros </h4>
   @li mpe_refreshlibs.sas
   @li dc_assignlib.sas
+  @li mpe_refreshcatalogs.sas
   @li mpe_refreshtables.sas
 
 
@@ -34,6 +35,8 @@ data libraries;
   str=cats('%mpe_refreshtables(',libref,')');
   put str;
   putlog str;
+  str=cats('%mpe_refreshcatalogs(',libref,')');
+  put str;
 run;
 %inc executor/source2;
 

sas/sasjs/targets/viya/macros_viya/dc_refreshcatalog.sas

@@ -10,6 +10,7 @@
   @li mpe_refreshlibs.sas
   @li dc_assignlib.sas
   @li mpe_refreshtables.sas
+  @li mpe_refreshcatalogs.sas
 
 
   @version 3.4
@@ -21,15 +22,15 @@
 
 %macro dc_refreshcatalog(libref);
 
-%mpe_refreshlibs()
-
 %if #&libref# ne ## %then %do;
   %put &sysmacroname: assigning specific libref, &libref;
   %dc_assignlib(WRITE,&libref) /* write just in order to assign direct lib */
   %mpe_refreshlibs(lib=&libref)
   %mpe_refreshtables(&libref)
+  %mpe_refreshcatalogs(&libref)
 %end;
 %else %do;
+  %mpe_refreshlibs()
   filename executor catalog 'work.code.code.source';
   data libraries;
     set &mpelib..mpe_datacatalog_libs;
@@ -37,6 +38,8 @@
     file executor;
     str=cats('%mpe_refreshtables(',libref,')');
     put str;
+    str=cats('%mpe_refreshcatalogs(',libref,')');
+    put str;
     putlog str;
   run;
   %inc executor;