350 lines
9.8 KiB
SAS
350 lines
9.8 KiB
SAS
/**
|
|
@file
|
|
@brief fetch the metadata and server as dotlang
|
|
|
|
Some nice ideas for formatting are available here:
|
|
https://renenyffenegger.ch/notes/tools/Graphviz/examples/index
|
|
|
|
<h4> SAS Macros </h4>
|
|
@li mp_abort.sas
|
|
@li mf_getuser.sas
|
|
@li bitemporal_dataloader.sas
|
|
@li meta_mapper.sas
|
|
|
|
@version 9.4
|
|
@author 4GL Apps Ltd
|
|
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
|
|
and may not be re-distributed or re-sold without the express permission of
|
|
4GL Apps Ltd.
|
|
**/
|
|
|
|
%mpeinit()
|
|
|
|
%global column_id direction refresh;
|
|
|
|
/* enable col id and direction to be passed as url params */
|
|
%let exist=%sysfunc(exist(work.SASControlTable));
|
|
%let inds=%sysfunc(ifc(&exist=1,SASControlTable,_null_));
|
|
%let max_depth=50;
|
|
%put &=inds;
|
|
data _null_;
|
|
length max_depth $ 8;
|
|
set &inds;
|
|
call symputx('column_id',coluri);
|
|
call symputx('direction',direction);
|
|
call symputx('refresh',refresh);
|
|
if input(max_depth,8.)>0 then call symputx('max_depth',max_depth);
|
|
putlog (_all_)(=);
|
|
run;
|
|
%put &=max_depth &=refresh;
|
|
|
|
data info;
|
|
length coluri colname taburi tabname liburi libref $256;
|
|
call missing(of _all_);
|
|
if metadata_getattr("&column_id","Name",colname)<0 then do;
|
|
putlog "Col &column_id not found";
|
|
call symputx('syscc','1234');
|
|
stop;
|
|
end;
|
|
rc=metadata_getnasn("&column_id","Table",1,taburi);
|
|
rc=metadata_getattr(taburi,"Name",tabname);
|
|
rc=metadata_getnasn(taburi,"TablePackage",1,liburi);
|
|
rc=metadata_getattr(liburi,"Libref",libref);
|
|
call symputx('lib',libref);
|
|
call symputx('tab',tabname);
|
|
call symputx('col',colname);
|
|
run;
|
|
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program..sas
|
|
,msg=%str(syscc=&syscc)
|
|
)
|
|
|
|
|
|
%macro launcher();
|
|
/* check whether a lineage run already taken place */
|
|
proc sql noprint;
|
|
create table existing_data as
|
|
select * from &mpelib..mpe_lineage_cols
|
|
where col_id="&column_id"
|
|
and direction="%substr(&direction,1,1)";
|
|
|
|
/* no data, so make some, and append it */
|
|
%if &sqlobs=0 or &refresh=1 %then %do;
|
|
%meta_mapper(metaid=&column_id
|
|
, direction=&direction /* either REVERSE or FORWARDS */
|
|
, baseds=work.allmap
|
|
, levelcheck=%eval(&max_depth-1)
|
|
)
|
|
data append;
|
|
length col_id $32 direction $1 modified_by $64;
|
|
retain col_id "&column_id";
|
|
retain direction "%substr(&direction,1,1)";
|
|
%global modified_by modified_dttm;
|
|
%let modified_dttm=%sysfunc(datetime());
|
|
retain modified_dttm &modified_dttm;
|
|
retain modified_by "%mf_getuser()";
|
|
%let modified_by=%mf_getuser();
|
|
set allmap;
|
|
drop hash;
|
|
run;
|
|
proc sort data=append out=appendme nodupkey;
|
|
by col_id direction sourcecoluri targetcoluri map_type map_transform;
|
|
run;
|
|
|
|
%bitemporal_dataloader(base_lib=&mpelib
|
|
,base_dsn=mpe_lineage_cols
|
|
,append_dsn=appendme
|
|
,PK=col_id direction sourcecoluri targetcoluri map_type map_transform
|
|
,etlsource=&_program
|
|
,loadtype=UPDATE
|
|
,close_vars=col_id direction
|
|
,dclib=&mpelib
|
|
)
|
|
%end;
|
|
%else %do;
|
|
/* data exists, so use it */
|
|
data work.allmap(drop=modified_by modified_dttm);
|
|
set existing_data(drop=col_id direction );
|
|
if _n_=1 then do;
|
|
call symputx('modified_by',modified_by,'g');
|
|
call symputx('modified_dttm',modified_dttm,'g');
|
|
end;
|
|
where level < &max_depth;
|
|
run;
|
|
%end;
|
|
%mend launcher;
|
|
|
|
%launcher()
|
|
|
|
/* generate graphviz */
|
|
filename tmp "%sysfunc(pathname(work))\GraphViz%sysfunc(datetime()).txt"
|
|
lrecl=10000 encoding='utf-8';
|
|
options noquotelenmax;
|
|
|
|
%macro fcmpconditional();
|
|
%if &sysver=9.3 and &sysscp=WIN %then %do;
|
|
/* nothing - as the FCMP function causes an exception err in this case */
|
|
%end;
|
|
%else %do;
|
|
/* prepare quick func to enable word wrapping of transformations */
|
|
options cmplib=work.funcs;
|
|
proc fcmp outlib=work.funcs.macrocore;
|
|
function wordwrap(str $,cols,splitchar $) $;
|
|
length outstr $32767 curstr $5000;
|
|
base=0;
|
|
put str=;
|
|
do i=1 to countw(str,' ',' ');
|
|
curstr=scan(str,i,' ');
|
|
outstr=trim(outstr)!!' '!!curstr;
|
|
base=base+length(curstr)+1;
|
|
if base>cols then do;
|
|
outstr=cats(outstr,splitchar);
|
|
base=0;
|
|
end;
|
|
end;
|
|
return (outstr);
|
|
endsub;
|
|
run;
|
|
%end;
|
|
%mend fcmpconditional;
|
|
%fcmpconditional()
|
|
|
|
/* prepare label with metadata */
|
|
proc sql;
|
|
create table jobs as select distinct jobname as job from work.allmap ;
|
|
create table cols as select distinct upcase(scan(cat,1,'.-')) as tmplib
|
|
,cats(calculated tmplib,'.',upcase(scan(cat,2,'.'))) as tmptab
|
|
,cats(calculated tmptab,'.',upcase(col)) as col
|
|
from (select sourcetablename as cat, sourcecolname as col from work.allmap
|
|
union select targettablename as cat, targetcolname as col from work.allmap )
|
|
having findc(tmplib,'/\')=0 and tmplib ne 'WORK';
|
|
create table files as select distinct file
|
|
from (select sourcetablename as file from work.allmap
|
|
where findc(sourcetablename,'/\')>0
|
|
union select targettablename as file from work.allmap
|
|
where findc(targettablename,'/\')>0
|
|
) ;
|
|
create table libs as select distinct tmplib as lib from cols;
|
|
create table tabs as select distinct tmptab as tab from cols;
|
|
|
|
data _null_;
|
|
file tmp;
|
|
put 'digraph G {
|
|
concentrate=true;
|
|
node [style=filled,shape=plain];
|
|
labelloc = "t";
|
|
';
|
|
label= "label=<<table><tr>
|
|
<td align='text' colspan='4' >&direction Lineage for <b>&col</b></td></tr>
|
|
<tr><td align='right'>Library:<br /></td><td align='left'>&lib</td>
|
|
<td align='right'>Generated by:</td><td align='left'>&modified_by</td>
|
|
</tr>
|
|
<tr><td align='right'>Table:</td><td align='left'>&tab</td>
|
|
<td align='right'>Generated on:</td><td align='left'>
|
|
%sysfunc(round(&modified_dttm,2),datetime19.)</td></tr>
|
|
</table>>";
|
|
put label;
|
|
if "FORWARD"="&direction" then call symputx('dirdesc','Impacted');
|
|
else call symputx('dirdesc','Source');
|
|
|
|
/* close out if there is no lineage */
|
|
if nobs=0 then put 'x [label="No lineage found" shape=Mdiamond]}';
|
|
set work.allmap nobs=nobs;
|
|
stop;
|
|
run;
|
|
|
|
|
|
data graphviz1;
|
|
file tmp mod;
|
|
length line arrow $1000 stab ttab slib tlib $100 sbox tbox tooltip $500;
|
|
if _n_=1 then call missing(line, sbox, tbox, tooltip);
|
|
set work.allmap ;
|
|
|
|
sourceid=sourcecoluri;
|
|
targetid=targetcoluri;
|
|
|
|
if index(sourcetablename,':') then do;
|
|
slib='';
|
|
stab=sourcetablename;
|
|
end;
|
|
else if map_transform='File Reader' then do;
|
|
stab=scan(sourcetablename,-1,'/\');
|
|
slib=subpad(sourcetablename,1,length(sourcetablename)-length(stab));
|
|
end;
|
|
else do;
|
|
slib=scan(sourcetablename,1,'.');
|
|
stab=scan(sourcetablename,2,'.');
|
|
end;
|
|
|
|
if index(targettablename,':') then do;
|
|
tlib='';
|
|
ttab=targettablename;
|
|
end;
|
|
else if map_transform='File Reader' then do;
|
|
ttab=scan(targettablename,-1,'/\');
|
|
tlib=subpad(targettablename,1,length(targettablename)-length(ttab));
|
|
end;
|
|
else do;
|
|
tlib=scan(targettablename,1,'.');
|
|
ttab=scan(targettablename,2,'.');
|
|
end;
|
|
|
|
if trim(derived_rule) ne '' then do;
|
|
derived_rule=tranwrd(derived_rule,'"','\"');
|
|
%macro quick();
|
|
%if "&sysver"="9.3" and "&sysscp"="WIN" %then %do;
|
|
arrow=cats('[color=Red, fontcolor=Red, penwidth="3", arrowsize="2",'
|
|
,'label=">>',map_transform,'<<\n',derived_rule,'"]');
|
|
%end;
|
|
%else %do;
|
|
arrow=cats('[color=Red, fontcolor=Red, penwidth="3", arrowsize="2",'
|
|
,'label=">>',map_transform,'<<\n',wordwrap(derived_rule,24,'\n'),'"]');
|
|
%end;
|
|
%mend quick; %quick()
|
|
end;
|
|
else arrow=cats('[ label="',map_transform,'"]');
|
|
|
|
source=quote(strip(sourceid));
|
|
target=quote(strip(targetid));
|
|
|
|
put ' ' source ' -> ' target arrow;
|
|
|
|
run;
|
|
|
|
data graphviz2 (keep=id tab lib col tooltip map_transform);
|
|
set graphviz1 (rename=(source=id stab=tab slib=lib sourcecolname=col ))
|
|
graphviz1 (rename=(target=id ttab=tab tlib=lib targetcolname=col ));
|
|
|
|
if upcase(lib)=:'WORK' then tooltip=cats(',tooltip="Job:',jobname,'"');
|
|
else tooltip='';
|
|
run;
|
|
proc sort data=graphviz2 out=graphviz3 noduprec; by _all_; run;
|
|
|
|
data _null_;
|
|
length shape $100 ;
|
|
set graphviz3 end=last;
|
|
file tmp mod;
|
|
tab=tranwrd(tab,'\','\\');
|
|
tab=tranwrd(tab,'&','&');
|
|
lib=tranwrd(lib,'&','&');
|
|
if upcase(lib)=:'WORK' then do;
|
|
lib='WORK';
|
|
put id '[label=<<table border="0"><tr><td>Table</td><td>' tab
|
|
'</td></tr><tr><td>Column</td><td align="left">' col
|
|
'</td></tr></table>> ,fillcolor=lightgrey, shape=" " ' tooltip ']';
|
|
end;
|
|
else if map_transform='File Reader' then do;
|
|
put id '[label="Location: ' lib '\nFile:' tab '\nColumn: ' col
|
|
'",shape=parallelogram, fillcolor="#00b300"' tooltip ']';
|
|
end;
|
|
else do;
|
|
engine=scan(lib,2,'-');
|
|
lib=scan(lib,1,'-');
|
|
if engine='BASE' then fillcolour='lightyellow ';
|
|
else fillcolour='lightblue';
|
|
shape=' shape=cylinder, fillcolor= '!!fillcolour;
|
|
|
|
put id '[label=<<table border="0"><tr><td>Library</td><td align="left">' lib
|
|
'</td></tr><tr><td>Table</td><td align="left">' tab
|
|
'</td></tr><tr><td>Column</td><td align="left">' col
|
|
'</td></tr></table>> ,' shape tooltip ']';
|
|
end;
|
|
run;
|
|
|
|
data _null_;
|
|
file tmp mod;
|
|
/* close out if records exist */
|
|
set work.allmap;
|
|
put '}';
|
|
stop;
|
|
run;
|
|
|
|
data flatdata;
|
|
length type $8 item $256;
|
|
keep type item;
|
|
set cols(in=cols) tabs(in=tabs) files(in=files) libs(in=libs) jobs(in=jobs);
|
|
|
|
if cols then do;
|
|
type='Column';
|
|
item=col;
|
|
end;
|
|
else if tabs then do;
|
|
type='Table';
|
|
item=tab;
|
|
end;
|
|
else if files then do;
|
|
type='File';
|
|
item=file;
|
|
end;
|
|
else if libs then do;
|
|
type='Library';
|
|
item=lib;
|
|
end;
|
|
else if jobs then do;
|
|
type='Job';
|
|
item=job;
|
|
end;
|
|
run;
|
|
|
|
data fromSAS;
|
|
infile tmp end=last;
|
|
file tmp;
|
|
input ;
|
|
string=_infile_;
|
|
put string;
|
|
run;
|
|
filename tmp clear;
|
|
|
|
/* get list of IDs so frontend can make a clickable list */
|
|
proc sql;
|
|
create table ids as select distinct id from graphviz3;
|
|
|
|
%webout(OPEN)
|
|
%webout(OBJ,fromSAS,missing=STRING)
|
|
%webout(OBJ,ids,dslabel=clickableIDS)
|
|
%webout(OBJ,info)
|
|
%webout(OBJ,flatdata)
|
|
%webout(CLOSE)
|
|
%mpeterm()
|