/**
  @file
  @brief Perform the metadata mapping

  @version 9.3
  @author 4GL Apps Ltd
  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
    and may not be re-distributed or re-sold without the express permission of
    4GL Apps Ltd.
**/
%macro meta_mapper(
      baseds=work.allmap /* base table to contain metamapping (two level) */
    , stageds=col_meta /* temp table to append to base*/
    , metaid=OMSOBJ:Column\A5HOSDWY.BF00LWQT
    , direction=REVERSE  /* either REVERSE or FORWARDS */
    , level=0 /* system var - show level of nesting */
    , job= /* system var - avoid looping same source */
    , levelcheck=50 /* system var - avoid going too deep down the rabbit hole */
    , append=NO /* system var - when YES means appending within nested loop */
  );
  %if &level>&levelcheck %then %return;
  %put &sysmacroname entry vars:;
  %put _local_;

  %if &direction=REVERSE %then %do;
    %let start=Target;
    %let finish=Source;
  %end;
  %else %do;
    %let start=Source;
    %let finish=Target;
  %end;

  %if &append=NO %then %do;
    proc datasets lib=work;
        delete %scan(&baseds,2,.);
    quit;
    %let index_statement=(index=(HASH/unique));
  %end;
  %else %let index_statement=;

  data &stageds &index_statement ;
    length HASH $32
      jobname sourcetablename sourcecolname sourcecoluri
      map_type map_transform  targettablename targetcolname targetcoluri
      uri targettableuri tfmuri sourcetableuri  scuri tpuri tmpuri mturi $256
      Derived_Rule $500 Marker_ID Name_ID N_Name LibRef engine sourcePublicType
      targetPublicType $64;
    keep HASH jobname sourcetablename sourcecolname sourcecoluri
      map_type map_transform
      targettablename targetcolname targetcoluri Derived_Rule level;

    /* proc transpose logic only */
    length  sourceshorttablename sourcemembertype sourcelocation
      assoc assocuri name sturi foundrefuri foundfinishuri
      targetshorttablename targetlocation targetmembertype trafoName $256
      sourceshorttableuri sourceshortcoluri checkrdm targetshorttableuri
      targetshortcoluri selected_direction $17
      lturi _location $200;

    call missing (of _all_);
    &start.coluri="&METAID";
    level=&level;
    /* first get table associated with the column */
    if metadata_getnasn(&start.coluri,'Table',1,&start.tableuri)=0 then do;
      putlog "ERR" "OR: Table not found";
      stop;
    end;
    rc=metadata_getattr(&start.tableuri,"Name",&start.tablename);
    rc=metadata_getattr(&start.coluri,"Name",&start.colname);
    rc=metadata_getattr(&start.tableuri,'PublicType',&start.PublicType);

    if (metadata_getnasn(&start.tableuri, "TablePackage",1, tpuri)>0) then
    do;
      rc=metadata_getattr(tpuri,"Libref",LibRef);
      rc=metadata_getattr(tpuri,"Engine",engine);
      if missing(libref) then do;
        rc=metadata_getnasn(tpuri, "UsedByPackages",1, tmpuri);
        rc=metadata_getattr(tmpuri,"Libref",LibRef);
        rc=metadata_getattr(tmpuri,"Engine",engine);
      end;
      &start.tablename=cats(
        upcase(LibRef),'-',upcase(engine),'.',&start.tablename
      );
    end;
    else if (&start.PublicType="ExternalFile") then do;
      rc=metadata_getnasn(&start.tableuri, "OwningFile",1, tmpuri);
      rc=metadata_getnasn(tmpuri, "FileRefs",1, tmpuri);
      rc=metadata_getnasn(tmpuri, "FileRefLocations",1, tmpuri);
      rc=metadata_getattr(tmpuri,'FileName',&start.tablename);
    end;
    else &start.tablename = 'work. '||trim(lowcase(&start.tablename));
    /* now loop the Source / TargetFeatureMaps */
    tfm=1;
    do while(metadata_getnasn(&start.coluri,"&start.FeatureMaps",tfm,tfmuri)>0);
      call missing(derived_rule);
      rc=metadata_getattr(tfmuri,'TransformRole',map_type);

      /* get job and step name */
      if (metadata_getnasn(tfmuri,'AssociatedClassifierMap',1,tmpuri)<1) then
      do;
        rc=metadata_getnasn(tfmuri,"&finish.Transformations",1,tmpuri);
        rc=metadata_getnasn(tmpuri,'AssociatedClassifierMap',1,tmpuri);
      end ;
      rc=metadata_getnasn(tmpuri,'Steps',1,tmpuri);
      rc=metadata_getattr(tmpuri,'Name',map_transform);
      rc=metadata_getnasn(tmpuri,'Activities',1,tmpuri);
      rc=metadata_getnasn(tmpuri,'Jobs',1,tmpuri);
      rc=metadata_getattr(tmpuri,'Name',jobname);

      if Map_Type = 'DERIVED' then do;
        if(metadata_getnasn(tfmuri,"SourceCode",1, scuri)>0) then do;
          /* standard */
          mturi=tfmuri;
        end;
        else do;
          /* some SQL joins store transform rules elsewhere */
          rc=metadata_getnasn(tfmuri,"Feature&start.s",1, tmpuri);
          if (metadata_getnasn(tmpuri,"Variables",1, tmpuri)>0)
          then rc=metadata_getnasn(tmpuri,"OwningTransformation",1, mturi);
          else rc=metadata_getnasn(tfmuri,"Transformation&start.s",1, mturi);
          rc=metadata_getnasn(mturi,"SourceCode",1, scuri);
        end;

        rc=metadata_getattr(scuri,"StoredText",Derived_Rule);
        Derived_Rule = compress(Derived_Rule,'0A'x);

        /* loop to generate derived rule (swap ref numbers for col descs) */
        sv=1;
        do while(metadata_getnasn(mturi,"SubstitutionVariables",sv,tmpuri)>0);
          rc=metadata_getattr(tmpuri,"Marker",Marker_ID);
          rc=metadata_getattr(tmpuri,"Name",Name_ID);
          N_Name = compress(scan(Name_ID,2,'-'));
          Derived_Rule=tranwrd(
            Derived_Rule,compress(Marker_ID),compress(N_Name)
          );
          sv+1;
        end;
      end;
      /* get source col attributes */
      fs=1;
      do while(metadata_getnasn(tfmuri,"Feature&finish.s",fs,&finish.coluri)>0);
        rc=metadata_getattr(&finish.coluri,'Name',&finish.colname);
        rc=metadata_getnasn(&finish.coluri,'Table',1,&finish.tableuri);
        rc=metadata_getattr(&finish.tableuri,'Name',&finish.tablename);
        rc=metadata_getattr(&finish.tableuri,'PublicType',&finish.PublicType);

        if (metadata_getnasn(&finish.tableuri,"TablePackage",1,tpuri)>0) then
        do;
          rc=metadata_getattr(tpuri,"Libref",LibRef);
          rc=metadata_getattr(tpuri,"Engine",engine);
          if missing(libref) then do;
            rc=metadata_getnasn(tpuri, "UsedByPackages",1, tmpuri);
            rc=metadata_getattr(tmpuri,"Libref",LibRef);
            rc=metadata_getattr(tmpuri,"Engine",engine);
          end;
          &finish.tablename=cats(
            upcase(LibRef),'-',upcase(engine),'.',&finish.tablename
          );
        end;
        else if (&finish.PublicType="ExternalFile") then do;
          rc=metadata_getnasn(&finish.tableuri, "OwningFile",1, tmpuri);
          rc=metadata_getnasn(tmpuri, "FileRefs",1, tmpuri);
          rc=metadata_getnasn(tmpuri, "FileRefLocations",1, tmpuri);
          rc=metadata_getattr(tmpuri,'FileName',&finish.tablename);
        end;
        else &finish.tablename=compress('work.'||lowcase(&finish.tablename));

        /* do a lookup to see if this record has been loaded before,
            IF base table exists */
        hash=put(md5(
          cats(jobname,sourcecoluri,map_type,map_transform,targetcoluri)
        ),$hex32.);
      %if %sysfunc(exist(&baseds)) %then %do;
        set &baseds(keep=hash) key=hash/unique;
        if _iorc_ ne 0 then do;
          /* hash did not exist, hence this is a new record */
          output;
          _error_=0;
        end;
      %end;
      %else %do;
        output;
      %end;
        fs+1;
      end;

      tfm+1;
    end;

    /* No finish URI found - so proceed to see if this is due to transpose */
    if missing(&finish.coluri)
      and (metadata_getnasn(&start.tableuri,"&start.ClassifierMaps",1,tmpuri)>0)
    then do;
      length trafoname $256;
      call missing(trafoName);
      rc=metadata_getattr(tmpuri,'Name',trafoName);

      /* get &finsh.tablename and jobname */
      rc=metadata_getnasn(tmpuri,"Classifier&finish.s",1,&finish.tableuri);
      rc=metadata_getattr(&finish.tableuri,'Name',&finish.tablename);
      &finish.shorttablename=&finish.tablename;
      rc=metadata_getnasn(tmpuri,'Steps',1,tmpuri);
      rc=metadata_getnasn(tmpuri,'Activities',1,tmpuri);
      rc=metadata_getnasn(tmpuri,'Jobs',1,tmpuri);
      rc=metadata_getattr(tmpuri,'Name',jobname);

      rc1=1;n1=1;
      do while(rc1>0);
        rc1=metadata_getnasl(&finish.tableuri,n1,assoc);
        if (assoc="Columns") then do;
          rc2=1;n2=1;
          do while(rc2>0 and missing(foundfinishuri));
            /* Walk through all column associations: SpecSourceTransformations*/
            rc2=metadata_getnasn(&finish.tableuri,trim(assoc),n2,assocuri);
          /* REVERSE */
          %if ("&direction." = "REVERSE") %then %do;
            if metadata_getnasn(assocuri,"SpecSourceTransformations",1,sturi)>0
            then do;
              rc=metadata_getattr(sturi,"Name",name);
              /* SAS Transpose: varColumns */
              if (name ="varColumns") then do;
                foundfinishuri = "true"; /* do a while exit */
                put "scource colname name=" name;
                put "scource colname uri=" assocuri;
                &finish.coluri=assocuri;
                &finish.shortcoluri=substr(
                  &finish.coluri,find(&finish.coluri, '\')+1
                );
                rc=metadata_getattr(&finish.coluri,'Name',&finish.colname);
                rc=metadata_getnasn(&finish.coluri,'Table',1,&finish.tableuri);
                rc=metadata_getattr(&finish.tableuri,'Name',&finish.tablename);
                &finish.shorttablename=&finish.tablename;
                map_type = "ONETOONE";

                /* get MemberType */
                rc=metadata_getattr(
                  &finish.tableuri,"MemberType",&finish.membertype
                );
                &finish.shorttableuri = substr(
                  &finish.tableuri,find(&finish.tableuri, '\')+1
                );
                rc=metadata_getattr(
                  &finish.shorttableuri,"PublicType",&finish.publictype
                );

                if metadata_getnasn(
                  &finish.shorttableuri,"TablePackage",1,tpuri
                )>0
                then do;
                  /* init LibRef to overwrite previous for &start!!! */
                  call missing (LibRef);
                  rc=metadata_getattr(tpuri,"Libref",LibRef);
                  rc=metadata_getattr(tpuri,"Engine",engine);
                  if missing(libref) then do;
                    rc=metadata_getnasn(tpuri, "UsedByPackages",1, tmpuri);
                    rc=metadata_getattr(tmpuri,"Libref",LibRef);
                    rc=metadata_getattr(tmpuri,"Engine",engine);
                  end;
                  &finish.shorttablename=&finish.tablename;
                  &finish.tablename=cats(
                    upcase(LibRef),'-',upcase(engine),'.',&finish.tablename
                  );
                end;
                /* get SAS Folder location of the table */
                if &finish.publictype eq "Table" then do;
                  lturi=&finish.shorttableuri;
                  rc=metadata_getnasn(lturi,"Trees",1,lturi);
                  rc=metadata_getattr(lturi,"Name",&finish.location);
                  tree=1;
                  do while (tree>0);
                    tree=metadata_getnasn(lturi,"ParentTree",1,lturi);
                    if tree > 0 then do;
                      rc=metadata_getattr(lturi,"Name",_location);
                      &finish.location=catx('/',_location,&finish.location);
                    end;
                  end;
                  &finish.location = '/'||&finish.location;
                end;
                map_transform = trafoName;
                derived_rule = "Transpose vertical";
                /* do a lookup to see if this record has been loaded before,
                  IF base table exists */
                hash=put(md5(
                  cats(jobname,sourcecoluri,map_type,map_transform,targetcoluri)
                ),$hex32.);
                %if %sysfunc(exist(&baseds)) %then %do;
                  set &baseds(keep=hash) key=hash/unique;
                  if _iorc_ ne 0 then do;
                    /* hash did not exist, hence this is a new record */
                    output;
                    _error_=0;
                  end;
                %end;
                %else %do;
                  output;
                %end;
              end; /*  (name = "_VALUE_COLUMN") */
            end; /* (metadata_getnasn(assocuri2,"Spec&finish.xxx",1,sturi)>0) */
          %end; /* &direction = "REVERSE" */
          /* FORWARDS: if TargetFeatureMaps not available: -> target! */
          %if ("&direction." = "FORWARDS") %then %do;
            if (metadata_getnasn(assocuri,"TargetFeatureMaps",1,sfuri)<0)
            then do;
              rc=metadata_getattr(assocuri,"Name",name);
              if not missing(assocuri) then do;
                put "target colname name=" name;
                put "target colname uri=" assocuri;
                &finish.coluri=assocuri;
                &finish.shortcoluri=substr(
                  &finish.coluri,find(&finish.coluri, '\')+1
                );
                rc=metadata_getattr(&finish.coluri,'Name',&finish.colname);
                rc=metadata_getnasn(&finish.coluri,'Table',1,&finish.tableuri);
                rc=metadata_getattr(&finish.tableuri,'Name',&finish.tablename);
                &finish.shorttablename=&finish.tablename;
                map_type = "ONETOMANY";

                /* get MemberType */
                rc=metadata_getattr(
                  &finish.tableuri,"MemberType",&finish.membertype
                );
                &finish.shorttableuri = substr(
                  &finish.tableuri,find(&finish.tableuri, '\')+1
                );
                rc=metadata_getattr(
                  &finish.shorttableuri,"PublicType",&finish.publictype
                );

                if metadata_getnasn(
                  &finish.shorttableuri,"TablePackage",1,tpuri
                )>0
                then do;
                  /* init LibRef to overwrite previous for &start!!! */
                  call missing (LibRef);
                  rc=metadata_getattr(tpuri,"Libref",LibRef);
                  rc=metadata_getattr(tpuri,"Engine",engine);
                  if missing(libref) then do;
                    rc=metadata_getnasn(tpuri, "UsedByPackages",1, tmpuri);
                    rc=metadata_getattr(tmpuri,"Libref",LibRef);
                    rc=metadata_getattr(tmpuri,"Engine",engine);
                  end;
                  &finish.shorttablename=&finish.tablename;
                  &finish.tablename=cats(
                    upcase(LibRef),'-',upcase(engine),'.',&finish.tablename
                  );
                end;
                /* get table's SAS Folder location */
                if &finish.publictype eq "Table" then do;
                  lturi=&finish.shorttableuri;
                  rc=metadata_getnasn(lturi,"Trees",1,lturi);
                  rc=metadata_getattr(lturi,"Name",&finish.location);
                  tree=1;
                  do while (tree>0);
                    tree=metadata_getnasn(lturi,"ParentTree",1,lturi);
                    if tree > 0 then do;
                      rc=metadata_getattr(lturi,"Name",_location);
                      &finish.location=catx('/',_location,&finish.location);
                    end;
                  end;
                  &finish.location = '/'||&finish.location;
                end;
                map_transform = trafoName;
                derived_rule = "Transpose horizontal";
                /* do a lookup to see if this record has been loaded before,
                  IF base table exists */
                hash=put(md5(
                  cats(jobname,sourcecoluri,map_type,map_transform,targetcoluri)
                ),$hex32.);
                %if %sysfunc(exist(&baseds)) %then %do;
                  set &baseds(keep=hash) key=hash/unique;
                  if _iorc_ ne 0 then do;
                    /* hash did not exist, hence this is a new record */
                    output;
                    _error_=0;
                  end;
                %end;
                %else %do;
                  output;
                %end;
              end;
            end;
          %end; /* (&direction = "FORWARDS" */

            call missing(assocuri);
            n2+1;
          end; /* while(rc1>0) */
        end; /* (assoc="Columns") */
        call missing(assoc);
        n1+1;
      end; /* while(rc1>0) */
    end; /* missing(&finish.coluri) */
    /* ################# end mapping for Transpose flat ################# */

    stop;
  run;

  proc append base=&baseds data=&stageds;
  run;

  data _null_;
    set &stageds;
    call execute('%meta_mapper(metaid='!!&finish.coluri
      !!",baseds=&baseds"
      !!",direction=&direction"
      !!",level=%eval(&level+1)"
      !!",levelcheck=&levelcheck"
      !!",job="!!jobname
      !!",append=YES)");
  run;

%mend meta_mapper;
