/**
  @file
  @brief Downloads data in a variety of formats
  @details To enable direct download, this service runs in a dedicated stream
  as a GET request using URL parameters as inputs.

  The inputs are:

  @li table - the libds of the table to be downloaded
  @li type - either SAS, CSV, EXCEL, MARKDOWN, WEBCSV or WEBTAB
  @li filter - the filter RK if used

  <h4> SAS Macros </h4>
  @li mf_verifymacvars.sas
  @li mf_getuser.sas
  @li mf_existfeature.sas
  @li dc_assignlib.sas
  @li mp_abort.sas
  @li mp_binarycopy.sas
  @li mp_cntlout.sas
  @li mp_ds2cards.sas
  @li mp_ds2csv.sas
  @li mp_streamfile.sas
  @li mp_validatecol.sas
  @li mpe_filtermaster.sas


  @version 9.2
  @author 4GL Apps Ltd
  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
    and may not be re-distributed or re-sold without the express permission of
    4GL Apps Ltd.

**/
%global table type filter ds format is_fmt txfrom txto;
%mpeinit()

%let user=%mf_getuser();
%let is_fmt=0;

/**
  * Validate inputs before they reach executable code.  table is used as a
  * dataset reference, in the output file path, and in the download filename,
  * so it must be a well-formed LIBREF.DATASET (the trailing -FC catalog
  * suffix is permitted); filter must be an integer.  Values are read with
  * symget (never re-resolved) and validated in a data step so no macro
  * content in the input can execute.
  */
%let is_libds=0;
%let is_int=0;
%let is_type=0;
data _null_;
  length _table $64 _filter $16 _ds $34 _type $16;
  _type=upcase(coalescec(symget('type'),''));
  _table=coalescec(symget('table'),'');
  _filter=coalescec(symget('filter'),'0');
  if missing(_table) then do;
    putlog 'ERR' 'OR: Missing table';
    stop;
  end;
  %mp_validatecol(_table,LIBDS,is_libds)
  /* permit the format-catalog form: LIBREF.CATALOGNAME-FC */
  if is_libds=0 then do;
    _ds=scan(_table,1,'-');
    %mp_validatecol(_ds,LIBDS,is_libds)
  end;
  /* an absent filter is a valid, unfiltered download */
  if missing(_filter) then _filter='0';
  %mp_validatecol(_filter,ISINT,is_int)
  /* type is validated against a fixed list of download formats */
  length _types_ok 8;
  _types_ok=0;
  if _type in ('SAS','CSV','EXCEL','MARKDOWN','WEBCSV','WEBTAB')
  then _types_ok=1;
  else putlog 'ERR' 'OR: Invalid type:' _type;
  if is_libds=0 then putlog 'ERR' 'OR: Invalid table:' _table;
  if is_int=0 then putlog 'ERR' 'OR: Invalid filter:' _filter;
  call symputx('is_libds',is_libds,'l');
  call symputx('is_int',is_int,'l');
  call symputx('is_type',_types_ok,'l');
  call symputx('filter',_filter,'l');
  if is_libds=1 then call symputx('table',upcase(_table),'l');
  if _types_ok=1 then call symputx('type',_type,'l');
run;

%mp_abort(iftrue= (&is_libds ne 1)
  ,mac=&_program..sas
  ,msg=%str(Invalid table)
)
%mp_abort(iftrue= (&is_int ne 1)
  ,mac=&_program..sas
  ,msg=%str(Invalid filter)
)
%mp_abort(iftrue= (&is_type ne 1)
  ,mac=&_program..sas
  ,msg=%str(Invalid type)
)

%let libds=%upcase(&table); /* actual source */
%let table=%upcase(&table); /* used as label for fmt catalogs */
%let lib=%scan(&table,1,.);
%let ds=%scan(&table,2,.);
%dc_assignlib(READ,&lib)

data _null_;
  set &mpelib..MPE_TABLES;
  where upcase(libref)="&lib" and upcase(dsn)="&ds";

  /* if a TXTEMPORAL table then filter as such */
  call symputx('txfrom',var_txfrom);
  call symputx('txto',var_txto);
run;

/* if a format, extract relevant info */
data _null_;
  ds=symget('ds');
  is_fmt=0;
  if subpad(cats(reverse(ds)),1,3)=:'CF-' then do;
    ds=scan(ds,1,'-');
    table=cats("&lib..",ds);
    putlog "Format Catalog Captured";
    is_fmt=1;
    call symputx('libds','work.fmtextract');
    call symputx('table',table);
  end;
  call symputx('is_fmt',is_fmt);
  putlog (_all_)(=);
run;

%mp_cntlout(
  iftrue=(&is_fmt=1)
  ,libcat=&table
  ,fmtlist=0
  ,cntlout=work.fmtextract
)

%put preparing query;
%mpe_filtermaster(DLOAD,&libds,
  dclib=&mpelib,
  filter_rk=&filter,
  outref=filtref,
  outds=work.query
)
%put printing generated filterquery:;
data _null_;
  infile filtref;
  input;
  putlog _infile_;
run;

options obs=200000;/* stop limit */
data staged(drop=&txfrom &txto);
  set &libds;
  where %inc filtref;;
run;
options obs=max;

options validvarname=upcase;

%macro mpestp_getrawdata();
  %local outfile;
  %if &type=SAS %then %do;
    %let outfile=%sysfunc(pathname(work))/&table..sas;
    %mp_ds2cards(base_ds=staged
        , tgt_ds=&table
        , cards_file= "&outfile"
        , maxobs=100000)
    %let ext=sas;
    %let mimetype=text;
  %end;
  %else %if &type=CSV or (&type=EXCEL and %mf_existfeature(EXPORTXLS) ne 1)
  /* cannot proc export excel if PC Files is not licensed */
  %then %do;
    %let outfile=%sysfunc(pathname(work))/&table..csv;
    /* cannot use PROC EXPORT as we need to wrap all char values in quotes */
    %mp_ds2csv(work.staged,outfile="&outfile",headerformat=NAME)
    %let ext=csv;
    %let mimetype=csv;
  %end;
  %else %if &type=EXCEL %then %do;
    %let ext=xlsx;
    %let outfile=%sysfunc(pathname(work))/&table..&ext;
    PROC EXPORT DATA= staged
        OUTFILE= "&outfile"
        DBMS=xlsx ;
    RUN;
    %let mimetype=XLSX;
  %end;
  %else %if &type=MARKDOWN %then %do;
    %let ext=md;
    %let outfile=%sysfunc(pathname(work))/&table..&ext;
    filename mdref "&outfile" lrecl=32767;

    %mp_ds2md(staged,outref=mdref,showlog=NO)

    %let mimetype=MARKDOWN;
  %end;
  %else %if &type=WEBCSV %then %do;
    PROC EXPORT DATA= staged
        OUTFILE= _webout
        DBMS=csv REPLACE;
    RUN;
    /* don't set headers */
    %return;
  %end;
  %else %if &type=WEBTAB %then %do;
    PROC EXPORT DATA= staged
        OUTFILE= _webout
        DBMS=tab REPLACE;
    RUN;
    /* don't set headers */
    %return;
  %end;
  %else %do;
    %mp_abort(msg=type &type not supported,mac=mpestp_getrawdata.sas);
  %end;

  %mp_abort(iftrue= (&syscc ne 0)
    ,mac=&_program..sas
    ,msg=%str(syscc=&syscc)
  )

  %mp_streamfile(contenttype=&mimetype
    ,inloc=%str(&outfile)
    ,outname=&table..&ext
  )

%mend mpestp_getrawdata;
%mpestp_getrawdata()

%mpeterm()