From e44a25dcc39ba4b9714257c60da84c2dfa613a85 Mon Sep 17 00:00:00 2001 From: allan Date: Wed, 11 Jun 2025 10:06:52 +0100 Subject: [PATCH] fix: showing catalog_cnt in libinfo Closes #160 --- sas/sasjs/db/datactrl/mpe_datastatus_libs.ddl | 1 + .../db/migrations/20200428_datacatalog.sas | 2 +- .../db/migrations/20250611_v7.0_release.sas | 5 +- sas/sasjs/macros/mpe_makedatamodel.sas | 3 +- sas/sasjs/macros/mpe_refreshtables.sas | 13 +- sas/sasjs/macros/mpe_refreshtables.test.sas | 3 +- sas/sasjs/services/public/refreshlibinfo.sas | 171 +++++++++--------- sas/sasjs/services/public/viewtables.sas | 3 +- 8 files changed, 110 insertions(+), 91 deletions(-) diff --git a/sas/sasjs/db/datactrl/mpe_datastatus_libs.ddl b/sas/sasjs/db/datactrl/mpe_datastatus_libs.ddl index b4516dc..28e5ce6 100644 --- a/sas/sasjs/db/datactrl/mpe_datastatus_libs.ddl +++ b/sas/sasjs/db/datactrl/mpe_datastatus_libs.ddl @@ -14,5 +14,6 @@ create table &curlib..mpe_datastatus_libs( libref char(8) label='Library Name', libsize num format=SIZEKMG. label='Size of file', table_cnt num label='Number of Tables', + catalog_cnt num label='Number of Catalogs', constraint pk_mpe_datastatus_libs primary key(libref,tx_to)); diff --git a/sas/sasjs/db/migrations/20200428_datacatalog.sas b/sas/sasjs/db/migrations/20200428_datacatalog.sas index d503a86..448bf67 100644 --- a/sas/sasjs/db/migrations/20200428_datacatalog.sas +++ b/sas/sasjs/db/migrations/20200428_datacatalog.sas @@ -29,7 +29,7 @@ create table dc.mpe_datacatalog_TABS( nvar num label='Number of Variables', compress char(8) label='Compression Routine', pk_fields char(512) - label='Primary Key Fields (identified by being in a constraint that is both Unique and Not Null)', + label='Primary Key Fields (in a constraint that is both Unique and Not Null)', constraint pk primary key(libref,dsn,tx_to)); diff --git a/sas/sasjs/db/migrations/20250611_v7.0_release.sas b/sas/sasjs/db/migrations/20250611_v7.0_release.sas index e5f2dae..d666d89 100644 --- a/sas/sasjs/db/migrations/20250611_v7.0_release.sas +++ b/sas/sasjs/db/migrations/20250611_v7.0_release.sas @@ -2,7 +2,7 @@ @file @brief migration script to move from v6.8.2 to v7.0 of data controller - BREAKING CHANGE - 4 additional tables for capturing catalog information + BREAKING CHANGE - 1 new column and 4 additional tables for capturing catalogs Be sure to run this using the correct system account (eg the regular DC account) @@ -23,6 +23,9 @@ libname &dclib "/YOUR/DATACONTROLLER/LIBRARY/PATH"; proc sql; +create table work.BACKUP as select * from &dclib..mpe_datastatus_libs; +alter table &dclib..mpe_datastatus_libs add catalog_cnt num; + create table &dclib..mpe_datacatalog_CATS( TX_FROM float format=datetime19., TX_TO float format=datetime19., diff --git a/sas/sasjs/macros/mpe_makedatamodel.sas b/sas/sasjs/macros/mpe_makedatamodel.sas index 25878db..f9862f1 100644 --- a/sas/sasjs/macros/mpe_makedatamodel.sas +++ b/sas/sasjs/macros/mpe_makedatamodel.sas @@ -220,7 +220,8 @@ create table &lib..mpe_datastatus_libs( TX_TO num ¬null format=datetime19.3, libref char(8) label='Library Name', libsize num format=SIZEKMG. label='Size of library', - table_cnt num label='Number of Tables' + table_cnt num label='Number of Tables', + catalog_cnt num label='Number of Catalogs' );quit; proc datasets lib=&lib noprint; modify mpe_datastatus_libs; diff --git a/sas/sasjs/macros/mpe_refreshtables.sas b/sas/sasjs/macros/mpe_refreshtables.sas index 171b1aa..0baab24 100644 --- a/sas/sasjs/macros/mpe_refreshtables.sas +++ b/sas/sasjs/macros/mpe_refreshtables.sas @@ -314,12 +314,23 @@ proc sql; %if &ds = #ALL %then %do; proc sql; - create table statuslibs as select + create table work.sumcat as + select libname as libref, + count(distinct memname) as catalog_cnt + from dictionary.catalogs + where upcase(libname)="&lib" + group by 1; + create table work.sumdsn as select libref ,sum(filesize) as libsize ,count(*) as table_cnt from statustabs group by 1; + create table work.statuslibs as + select a.*, b.catalog_cnt + from work.sumdsn a + full join work.sumcat b + on a.libref=b.libref; %bitemporal_dataloader(base_lib=&mpelib ,base_dsn=mpe_datastatus_libs diff --git a/sas/sasjs/macros/mpe_refreshtables.test.sas b/sas/sasjs/macros/mpe_refreshtables.test.sas index ba28673..d3ce789 100644 --- a/sas/sasjs/macros/mpe_refreshtables.test.sas +++ b/sas/sasjs/macros/mpe_refreshtables.test.sas @@ -32,7 +32,8 @@ create table work.libinfo as a.schemas, a.libid, b.libsize, - b.table_cnt + b.table_cnt, + b.catalog_cnt from &mpelib..mpe_datacatalog_libs(where=(&dc_dttmtfmt. lt tx_to)) a left join &mpelib..mpe_datastatus_libs(where=(&dc_dttmtfmt. lt tx_to)) b on a.libref=b.libref diff --git a/sas/sasjs/services/public/refreshlibinfo.sas b/sas/sasjs/services/public/refreshlibinfo.sas index b971b29..033a243 100644 --- a/sas/sasjs/services/public/refreshlibinfo.sas +++ b/sas/sasjs/services/public/refreshlibinfo.sas @@ -1,85 +1,86 @@ -/** - @file refreshlibinfo.sas - @brief Refresh the Data Catalog for a particular library - @details When showing library info in the VIEW menu, the data is taken from - the Data Catalog tables. These may be empty or outdated, and so this service - allows end users to run a refresh of the data. - -

Service Inputs

-
lib2refresh
- Should contain the libref to be refreshed. - |libref:$8.| - |---| - |SOMELIB| - -

Service Outputs

- -
libinfo
- - |engine $|libname $|paths $|perms $|owners $|schemas $ |libid $|libsize $|table_cnt | - |---|---|---|---|---|---|---|---|---| - |V9|SOMELIB|"some/path"|rwxrwxr-x|sassrv|` `|` `|636MB|33| - -

SAS Macros

- @li dc_assignlib.sas - @li dc_refreshcatalog.sas - @li mp_abort.sas - - @version 9.3 - @author 4GL Apps Ltd - @copyright 4GL Apps Ltd. This code may only be used within Data Controller - and may not be re-distributed or re-sold without the express permission of - 4GL Apps Ltd. -**/ - - -%mpeinit() - -%webout(FETCH) - -%mp_abort(iftrue= (&syscc ne 0) - ,msg=%str(syscc=&syscc Problem on startup) -) - -%let libref=; -data _null_; - set work.lib2refresh; - call symputx('libref',libref); -run; - -%mp_abort(iftrue= (&syscc ne 0) - ,msg=%str(syscc=&syscc Problem with inputs - was lib2refresh object sent?) -) - -%dc_assignlib(WRITE,&libref) - -%mp_abort(iftrue= (&syscc ne 0) - ,msg=%str(syscc=&syscc after lib assignment) -) - -%dc_refreshcatalog(&libref) - -%mp_abort(iftrue= (&syscc ne 0) - ,msg=%str(syscc=&syscc Problem when running the catalog refresh) -) - -/* get libinfo */ -proc sql; -create table work.libinfo as - select a.engine, - a.libname, - a.paths, - a.perms, - a.owners, - a.schemas, - a.libid, - b.libsize, - b.table_cnt - from &mpelib..mpe_datacatalog_libs(where=(&dc_dttmtfmt. lt tx_to)) a - inner join &mpelib..mpe_datastatus_libs(where=(&dc_dttmtfmt. lt tx_to)) b - on a.libref=b.libref - where a.libref="&libref"; - -%webout(OPEN) -%webout(OBJ,libinfo) -%webout(CLOSE) +/** + @file refreshlibinfo.sas + @brief Refresh the Data Catalog for a particular library + @details When showing library info in the VIEW menu, the data is taken from + the Data Catalog tables. These may be empty or outdated, and so this service + allows end users to run a refresh of the data. + +

Service Inputs

+
lib2refresh
+ Should contain the libref to be refreshed. + |libref:$8.| + |---| + |SOMELIB| + +

Service Outputs

+ +
libinfo
+ + |engine $|libname $|paths $|perms $|owners $|schemas $ |libid $|libsize $|table_cnt | + |---|---|---|---|---|---|---|---|---| + |V9|SOMELIB|"some/path"|rwxrwxr-x|sassrv|` `|` `|636MB|33| + +

SAS Macros

+ @li dc_assignlib.sas + @li dc_refreshcatalog.sas + @li mp_abort.sas + + @version 9.3 + @author 4GL Apps Ltd + @copyright 4GL Apps Ltd. This code may only be used within Data Controller + and may not be re-distributed or re-sold without the express permission of + 4GL Apps Ltd. +**/ + + +%mpeinit() + +%webout(FETCH) + +%mp_abort(iftrue= (&syscc ne 0) + ,msg=%str(syscc=&syscc Problem on startup) +) + +%let libref=; +data _null_; + set work.lib2refresh; + call symputx('libref',libref); +run; + +%mp_abort(iftrue= (&syscc ne 0) + ,msg=%str(syscc=&syscc Problem with inputs - was lib2refresh object sent?) +) + +%dc_assignlib(WRITE,&libref) + +%mp_abort(iftrue= (&syscc ne 0) + ,msg=%str(syscc=&syscc after lib assignment) +) + +%dc_refreshcatalog(&libref) + +%mp_abort(iftrue= (&syscc ne 0) + ,msg=%str(syscc=&syscc Problem when running the catalog refresh) +) + +/* get libinfo */ +proc sql; +create table work.libinfo as + select a.engine, + a.libname, + a.paths, + a.perms, + a.owners, + a.schemas, + a.libid, + b.libsize, + b.table_cnt, + b.catalog_cnt + from &mpelib..mpe_datacatalog_libs(where=(&dc_dttmtfmt. lt tx_to)) a + inner join &mpelib..mpe_datastatus_libs(where=(&dc_dttmtfmt. lt tx_to)) b + on a.libref=b.libref + where a.libref="&libref"; + +%webout(OPEN) +%webout(OBJ,libinfo) +%webout(CLOSE) diff --git a/sas/sasjs/services/public/viewtables.sas b/sas/sasjs/services/public/viewtables.sas index d95d0ef..6a88221 100644 --- a/sas/sasjs/services/public/viewtables.sas +++ b/sas/sasjs/services/public/viewtables.sas @@ -167,7 +167,8 @@ create table work.libinfo as a.schemas, a.libid, coalesce(b.libsize,0) as libsize, - coalesce(b.table_cnt,0) as table_cnt + coalesce(b.table_cnt,0) as table_cnt, + coalesce(b.catalog_cnt,0) as catalog_cnt from &mpelib..mpe_datacatalog_libs(where=(&dc_dttmtfmt. lt tx_to)) a left join &mpelib..mpe_datastatus_libs(where=(&dc_dttmtfmt. lt tx_to)) b on a.libref=b.libref