/**
  @file refreshtablelineage.sas
  @brief updates the table level lineage
  @details extracts all sources/targets from every job

  <h4> SAS Macros </h4>
  @li mpeinit.sas
  @li bitemporal_dataloader.sas
  @li mp_binarycopy.sas
  @li mf_getuniquefileref.sas


  @version 9.3
  @author 4GL Apps Ltd
  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
    and may not be re-distributed or re-sold without the express permission of
    4GL Apps Ltd.
**/

%mpeinit()

/* get list of libraries */
%let fileref1=%mf_getuniquefileref();
filename &fileref1 temp;
proc metadata in=
"<GetMetadataObjects><Reposid>$METAREPOSITORY</Reposid>
  <Type>Job</Type><Objects/><NS>SAS</NS>
  <!-- OMI_DEPENDENCY_USES(8192) + OMI_GET_METADATA(256) + OMI_TEMPLATE(4) -->
  <Flags>8452</Flags>
  <Options>
    <Templates>
      <Job><JobActivities/></Job>
      <TransformationActivity><Steps/></TransformationActivity>
      <TransformationStep><Transformations/></TransformationStep>
      <ClassifierMap>
        <ClassifierSources/><ClassifierTargets/>
      </ClassifierMap>
      <Select>
        <ClassifierSources/><ClassifierTargets/>
      </Select>
      <Join>
        <ClassifierSources/><ClassifierTargets/>
      </Join>
    </Templates>
  </Options></GetMetadataObjects>"
  out=&fileref1
  ;
run;

/* create an XML map and extract dependencies from response */
%macro getTables(type=);
%local x;
%do x=1 %to 2;
  %local dir;
  %if &x=1 %then %let dir=Source;
  %else %let dir=Target;
  %local y;
  %do y=1 %to 3;
    %local maptype;
    %if &y=1 %then %let maptype=ClassifierMap;
    %else %if &y=2 %then %let maptype=Select;
    %else %if &y=3 %then %let maptype=Join;

    filename sxlemap temp;
    data _null_;
      file sxlemap;
      put '<SXLEMAP version="1.2" name="SASObjects">';
      put "<TABLE name='&dir&type'>";
      put "<TABLE-PATH syntax='XPath'>";
      put "/GetMetadataObjects/Objects/Job/JobActivities/TransformationActivi"@;
      put "ty/Steps/TransformationStep/Transformations/&maptype/Classifier"@;
      put "&dir.s/&type</TABLE-PATH>";
      put '<COLUMN name="jobid" retain="YES">';
      put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Job/@Id</PATH>";
      put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>17</LENGTH>";
      put '</COLUMN>';
      put '<COLUMN name="jobname" retain="YES">';
      put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Job/@Name</PATH>";
      put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>128</LENGTH>";
      put '</COLUMN>';
      put '<COLUMN name="TransformationActivityid" retain="YES">';
      put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Job/JobActivities"@;
      put "/TransformationActivity/@Id</PATH>";
      put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>17</LENGTH>";
      put '</COLUMN>';
      put '<COLUMN name="TransformationStepId" retain="YES">';
      put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Job/JobActivities"@;
      put "/TransformationActivity/Steps/TransformationStep/@Id</PATH>";
      put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>17</LENGTH>";
      put '</COLUMN>';
      put '<COLUMN name="mapid" retain="YES">';
      put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Job/JobActivities"@;
      put "/TransformationActivity/Steps/TransformationStep/Transformations"@;
      put "/&maptype/@Id</PATH>";
      put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>17</LENGTH>";
      put '</COLUMN>';
      put "<COLUMN name='tableid'>";
      put "<PATH syntax='XPath'>/GetMetadataObjects/Objects/Job/JobActivities"@;
      put "/TransformationActivity/Steps/TransformationStep/Transformations/"@;
      put "&maptype/Classifier&dir.s/&type/@Id</PATH>";
      put "<TYPE>character</TYPE><DATATYPE>string</DATATYPE><LENGTH>17</LENGTH>";
      put '</COLUMN>';
      put '</TABLE></SXLEMAP>';

    run;
    libname _XML_ xml xmlfileref=&fileref1 xmlmap=sxlemap;

    data work.&dir.&type/view=work.&dir.&type;
      set _xml_.&dir.&type;
      length tabletype maptype $16 tabledirection $1;
      jobname=upcase(jobname);
      tabletype="&type";
      maptype="&maptype";
      if "&dir"="Source" then tabledirection="R";
      else tabledirection='F';
    run;
    proc append base=work.alldata data=work.&dir.&type;
    run;

    libname _XML_ clear;
  %end;
%end;
%mend getTables;

/* list of types:
https://support.sas.com/documentation/cdl/en/omamodref/63903/HTML/default/viewer.htm#classifiermap.htm
*/
%getTables(type=PhysicalTable)
%getTables(type=ExternalTable)
%getTables(type=QueryTable)
%getTables(type=RelationalTable)
%getTables(type=Report)
%getTables(type=Cube)
%getTables(type=DataTable)

/* remove dups and get tables */
proc sort data=work.alldata(keep=jobid jobname tableid tabletype tabledirection)
  out=work.sorted nodupkey;
  by _all_;
run;
proc sort data=work.sorted(keep=tableid) out=work.tables nodupkey;
  by tableid;
run;

/* this bit is slow due to lookups but given the different table types
  it was the easiest approach */
data work.tables2;
  length tablename liburi puri $64 libref schemaname $8 ;
  if _n_=1 then call missing(tablename, liburi, puri, libref, schemaname);
  set work.tables;
  drop rc liburi;
  if metadata_getnasn(tableid,"TablePackage",1,liburi)>0 then do;
    rc= metadata_getattr(liburi, "SchemaName", SchemaName);
    if ^missing(SchemaName) then do;
      if metadata_getnasn(liburi,"UsedByPackages",1,puri)>0 then do;
        rc=metadata_getattr(puri,"Libref",libref);
      end;
    end;
    else rc=metadata_getattr(liburi,"Libref",libref);
    libref=upcase(libref);
  end;
  rc=metadata_getattr(tableid,"Name",tablename);
  tablename=upcase(tablename);
run;
proc sql;
create table work.augmented as
  select a.*
    ,b.tablename
    ,b.libref
  from work.sorted a
  left join work.tables2 b
  on a.tableid=b.tableid;

create table work.mpe_lineage_tabs as
  select distinct
    coalesce(a.jobid,b.jobid) as jobid,
    coalesce(a.jobname,b.jobname) as jobname,
    coalesce(a.tableid,"N/A") as srctableid,
    a.tabletype as srctabletype,
    a.tablename as srctablename,
    coalesce(a.libref,'nolib') as srclibref,
    coalesce(b.tableid,"N/A") as tgttableid,
    b.tabletype as tgttabletype,
    b.tablename as tgttablename,
    coalesce(b.libref,'nolib') as tgtlibref
  from work.augmented(where=(tabledirection='R')) a
  full join work.augmented(where=(tabledirection='F')) b
  on a.jobid=b.jobid
  ;


%bitemporal_dataloader(base_lib=&mpelib
  ,base_dsn=MPE_LINEAGE_TABS
  ,append_dsn=MPE_LINEAGE_TABS
  ,PK=jobid srctableid tgttableid
  ,etlsource=&_program
  ,loadtype=TXTEMPORAL
  ,tech_from=TX_FROM
  ,tech_to=TX_TO
  ,dclib=&mpelib
)

%mp_binarycopy(inref=&fileref1, outref=_webout)
