dc
16e9b31e65
Tests inferred whether a service call worked by reading an outlib that a failed call never touched, so assertions ran against the previous call's tables. This adds dc_execute, which wraps mx_execute and makes each call observable: - failfast: abort as soon as the child did not complete, with its message, so the first failure is the real one - a status row per call (outstatus) so a test can assert the outcome - the child's log behind the harness debug flag, quiet otherwise testsetup reseeds the data baseline before the test library is built, makedata honours a keepme switch and getdata initialises the variables it later tests with %length, plus desc= values reworded where a comma or apostrophe unbalanced the parameter. sasjs/macros/mx_execute.sas is a TEMPORARY copy from sasjs/core#476, which adds outlogds/outstatus and stops leaving work.results in the caller WORK. Delete it and bump the dependency once that PR is released.
917 lines
26 KiB
SAS
Executable File
917 lines
26 KiB
SAS
Executable File
/**
|
|
@file getdata.sas
|
|
@brief Returns a dataset to the editor front end
|
|
@details
|
|
|
|
<h4> Service Inputs </h4>
|
|
|
|
<h5> SASCONTROLTABLE </h5>
|
|
|LIBDS:$41.|FILTER_RK:$5.|
|
|
|---|---|
|
|
|DC258467.MPE_X_TEST|-1|
|
|
|
|
<h4> Service Outputs </h4>
|
|
<h5> sasdata </h5>
|
|
<h5> sasparams </h5>
|
|
Contains info on the request. One row is returned.
|
|
@li CLS_FLG - set to 0 if there are no CLS rules (everything editable)
|
|
else set to 1 (CLS rules exist)
|
|
@li ISMAP - set to 1 if the target DS is an excel map target, else 0
|
|
@li TRUNCATED - set to 1 if the selection was cut down to the
|
|
DC_MAXOBS_WEBEDIT / DC_MAXCELLS_WEBEDIT limit, else 0
|
|
@li TRUNC_MSG - the reason shown against the table name when TRUNCATED=1,
|
|
else empty
|
|
|
|
<h5> approvers </h5>
|
|
<h5> dqrules </h5>
|
|
<h5> dqdata </h5>
|
|
<h5> cols </h5>
|
|
Contains column level attributes.
|
|
@li NAME - column name
|
|
@li LABEL - var label. https://core.sasjs.io/mp__getcols_8sas.html
|
|
@li FMTNAME - derived format. https://core.sasjs.io/mp__getcols_8sas.html
|
|
@li DDTYPE - derived dropdown. C=CHARACTER, N=NUMERIC, else DATE / TIME /
|
|
DATETIME. https://core.sasjs.io/mp__getcols_8sas.html
|
|
@li CLS_RULE - values include:
|
|
- EDIT - the column is editable
|
|
- READ - the column should be readonly
|
|
- HIDE - the column should be hidden
|
|
@li memlabel
|
|
@li desc- augmented with MPE_DATADICTIONARY if exists, else label
|
|
@li longdesc - from MPE_DATADICTIONARY
|
|
@li coltype - front-end column formatting spec
|
|
|
|
|
|
<h5> maxvarlengths </h5>
|
|
<h5> xl_rules </h5>
|
|
<h5> query </h5>
|
|
|
|
<h5> versions </h5>
|
|
history of DC versions for this particular table
|
|
|
|
|
|
<h4> SAS Macros </h4>
|
|
@li dc_assignlib.sas
|
|
@li dc_casload.sas
|
|
@li dc_getgroupmembers.sas
|
|
@li mf_existvar.sas
|
|
@li mf_getattrn.sas
|
|
@li mf_getvarlist.sas
|
|
@li mf_existds.sas
|
|
@li mf_getquotedstr.sas
|
|
@li mf_getuniquefileref.sas
|
|
@li mf_getuniquename.sas
|
|
@li mf_getuser.sas
|
|
@li mf_nobs.sas
|
|
@li mf_verifymacvars.sas
|
|
@li mf_wordsinstr1butnotstr2.sas
|
|
@li mp_abort.sas
|
|
@li mp_cntlout.sas
|
|
@li mp_getcols.sas
|
|
@li mp_getmaxvarlengths.sas
|
|
@li mp_validatecol.sas
|
|
@li mpe_accesscheck.sas
|
|
@li mpe_columnlevelsecurity.sas
|
|
@li mpe_derived.sas
|
|
@li mpe_dsmeta.sas
|
|
@li mpe_getlabels.sas
|
|
@li mpe_getversions.sas
|
|
@li mpe_filtermaster.sas
|
|
@li mpe_runhook.sas
|
|
@li mpeinit.sas
|
|
@li mpeterm.sas
|
|
|
|
@version 9.2
|
|
@author 4GL Apps Ltd
|
|
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
|
|
and may not be re-distributed or re-sold without the express permission of
|
|
4GL Apps Ltd.
|
|
|
|
**/
|
|
|
|
%mpeinit()
|
|
|
|
/**
|
|
* Validate inputs
|
|
*/
|
|
data work.intest;
|
|
length filter_rk 8;
|
|
set work.SASCONTROLTABLE;
|
|
|
|
/* validate filter_rk */
|
|
if filter_rk le 0 then filter_rk=-1;
|
|
|
|
call symputx('orig_libds',upcase(libds));
|
|
|
|
is_fmt=0;
|
|
if substr(cats(reverse(libds)),1,3)=:'CF-' then do;
|
|
libds=scan(libds,1,'-');
|
|
putlog "Format Catalog Captured";
|
|
is_fmt=1;
|
|
libds='work.fmtextract';
|
|
call symputx('libds',libds);
|
|
end;
|
|
call symputx('is_fmt',is_fmt);
|
|
putlog (_all_)(=);
|
|
|
|
/* validate libds */
|
|
%mp_validatecol(LIBDS,LIBDS,is_libds)
|
|
|
|
if is_libds=0 then do;
|
|
putlog 'ERR' 'OR: Invalid libds:' libds;
|
|
stop;
|
|
end;
|
|
else do;
|
|
call symputx('filter_rk',filter_rk);
|
|
call symputx('libds',libds);
|
|
end;
|
|
output;
|
|
stop;
|
|
run;
|
|
|
|
%mp_abort(iftrue= (%mf_nobs(work.intest)=0)
|
|
,mac=&_program
|
|
,msg=%str(Some err with service inputs)
|
|
)
|
|
|
|
%mp_abort(
|
|
iftrue=(%mf_verifymacvars(libds filter_rk)=0)
|
|
,mac=&_program
|
|
,msg=%str(Missing: libds filter_rk)
|
|
)
|
|
|
|
/* export format catalog */
|
|
%mp_cntlout(
|
|
iftrue=(&is_fmt=1)
|
|
,libcat=&orig_libds
|
|
,fmtlist=0
|
|
,cntlout=work.fmtextract
|
|
)
|
|
|
|
/* stream back meta info, further calls will return col metadata and actual data
|
|
*/
|
|
%let libref=%upcase(%scan(&libds,1,.));
|
|
%let dsn=%upcase(%scan(&libds,2,.));
|
|
%dc_assignlib(WRITE,&libref)
|
|
%dc_casload(&libds)
|
|
|
|
/**
|
|
* First check user has access permission to edit the table
|
|
*/
|
|
%put checking access;
|
|
%let user=%mf_getuser();
|
|
%mpe_accesscheck(&orig_libds,outds=mw_auth,user=&user,access_level=EDIT)
|
|
|
|
|
|
%mp_abort(iftrue= (%mf_getattrn(work.mw_auth,NLOBS)=0)
|
|
,mac=mpestp_getdata.sas
|
|
,msg=&user is not authorised to edit &orig_libds %trim(
|
|
)in the &mpelib..MPE_SECURITY table
|
|
)
|
|
|
|
%mp_abort(iftrue= ( %mf_existds(libds=&libds) ne 1)
|
|
,mac=mpestp_getdata.sas
|
|
,msg=dataset &libds does not exist!!
|
|
)
|
|
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program..sas
|
|
,msg=%str(syscc=&syscc at line 60 )
|
|
)
|
|
|
|
|
|
%global loadtype var_txfrom var_txto var_processed filter_text pk coltype
|
|
sortpk var_busfrom var_busto rk_underlying buskey pre_edit_hook;
|
|
|
|
%put getting table attributes;
|
|
proc sql noprint;
|
|
select upcase(loadtype)
|
|
,var_txfrom,var_txto
|
|
,var_busfrom,var_busto
|
|
,var_processed,rk_underlying,buskey
|
|
,coalesce(rk_underlying,buskey)
|
|
,pre_edit_hook
|
|
,case when missing(rk_underlying) then buskey else rk_underlying end
|
|
into: loadtype,:var_txfrom,:var_txto
|
|
,:var_busfrom ,:var_busto
|
|
,:var_processed,:rk_underlying,:buskey, :sortPK, :pre_edit_hook,:pk
|
|
from &mpelib..mpe_tables
|
|
where &dc_dttmtfmt. lt TX_TO
|
|
and upcase(dsn)="%scan(&orig_libds,2,.)"
|
|
and upcase(libref)="%scan(&orig_libds,1,.)";
|
|
|
|
%put preparing filter query:;
|
|
%mpe_filtermaster(EDIT,&orig_libds,
|
|
dclib=&mpelib,
|
|
filter_rk=&filter_rk,
|
|
outref=filtref,
|
|
outds=work.query
|
|
)
|
|
|
|
/**
|
|
* Derived (computed) variables, configured in MPE_DERIVED. They are added
|
|
* via a view, so that the filter (applied by the step that reads the view)
|
|
* can reference them - a WHERE clause cannot see a variable that is created
|
|
* in the same data step.
|
|
*/
|
|
%mpe_derived(libds=&libds,
|
|
dclib=&mpelib,
|
|
outref=dercode,
|
|
outview=work.vw_derived,
|
|
outvars=work.derived_vars
|
|
)
|
|
%let derived_nobs=%mf_nobs(work.derived_vars);
|
|
%put &=derived_nobs;
|
|
|
|
%macro mpestp_getdata();
|
|
%if not %symexist(DC_MAXOBS_WEBEDIT) %then %do;
|
|
%put NOTE:;%put NOTE- DC_MAXOBS_WEBEDIT not found!;
|
|
%put NOTE- Please add to &mpelib..MPE_CONFIG table;
|
|
%put NOTE-;%put NOTE-;
|
|
%global DC_MAXOBS_WEBEDIT;
|
|
%let DC_MAXOBS_WEBEDIT=250;
|
|
%end;
|
|
%if not %symexist(DC_MAXCELLS_WEBEDIT) %then %do;
|
|
%put NOTE:;%put NOTE- DC_MAXCELLS_WEBEDIT not found!;
|
|
%put NOTE- Please add to &mpelib..MPE_CONFIG table;
|
|
%put NOTE-;%put NOTE-;
|
|
%global DC_MAXCELLS_WEBEDIT;
|
|
%let DC_MAXCELLS_WEBEDIT=200000;
|
|
%end;
|
|
/* for tables which use RKs/SKs then we just expose the business key to
|
|
users - this lets uploads be sent to multiple environments (with
|
|
potentially different RK/SK values for the same business key).
|
|
Note that the config table has the RK column in the buskey field in
|
|
this scenario. */
|
|
%if %length(&rk_underlying)>0 %then %let drop_rk=&buskey;
|
|
%else %let drop_rk=;
|
|
|
|
/* always remove the PROCESSED_DTTM column, if it exists */
|
|
%if %length(&var_processed)=0 %then %do;
|
|
%if %mf_existvar(&libds,PROCESSED_DTTM)>0 %then
|
|
%let var_processed=PROCESSED_DTTM;
|
|
%end;
|
|
|
|
/**
|
|
* Now get the slice of the actual table
|
|
*/
|
|
options obs=10000;
|
|
|
|
%if &loadtype=BITEMPORAL %then %do;
|
|
data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
|
|
%if &derived_nobs>0 %then %do;
|
|
set work.vw_derived;
|
|
%end;
|
|
%else %do;
|
|
set &libds;
|
|
%end;
|
|
where %inc filtref;;
|
|
run;
|
|
proc sort data=out;
|
|
by &pk &var_busfrom;
|
|
run;
|
|
data out;
|
|
set out;
|
|
by &pk &var_busfrom;
|
|
if last.%scan(&pk,-1);
|
|
run;
|
|
%end;
|
|
%else %do;
|
|
data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
|
|
%if &derived_nobs>0 %then %do;
|
|
set work.vw_derived;
|
|
%end;
|
|
%else %do;
|
|
set &libds;
|
|
%end;
|
|
where %inc filtref;;
|
|
run;
|
|
%end;
|
|
options obs=max;
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program
|
|
,msg=%str(Issue with filtering (line 165) )
|
|
)
|
|
|
|
/* Sort first - the PRE_EDIT_HOOK contract is that it runs on sorted rows -
|
|
then run the hook, then apply the row and cell limits (below). */
|
|
%let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
|
|
proc sort data=work.out; by &sortPK; run;
|
|
|
|
%mpe_runhook(PRE_EDIT_HOOK)
|
|
|
|
/* Serve the selection truncated, at whichever limit binds first, and carry
|
|
the reason in sasparams for the EDIT screen to show against the table
|
|
name. Bounding the browser's work is the point: a wide table exhausts the
|
|
cell budget long before it reaches the row limit - 250 rows of a 1000
|
|
column table is a quarter of a million cells - so the row cap is
|
|
min(DC_MAXOBS_WEBEDIT, DC_MAXCELLS_WEBEDIT / columns). Counting after the
|
|
PRE_EDIT_HOOK means the cap covers what the hook added to work.out. */
|
|
%let obscnt=%mf_getattrn(work.out,NLOBS);
|
|
%let colcnt=%sysfunc(countw(%mf_getvarlist(work.out)));
|
|
%let maxrows=&DC_MAXOBS_WEBEDIT;
|
|
%let limitvar=DC_MAXOBS_WEBEDIT;
|
|
%if &colcnt>0 %then %do;
|
|
%let cellrows=%sysfunc(floor(&DC_MAXCELLS_WEBEDIT/&colcnt));
|
|
%if %sysevalf(&cellrows lt &maxrows) %then %do;
|
|
%let maxrows=&cellrows;
|
|
%let limitvar=DC_MAXCELLS_WEBEDIT;
|
|
%end;
|
|
%end;
|
|
%if %sysevalf(&maxrows lt 1) %then %let maxrows=1;
|
|
%global truncated trunc_msg;
|
|
%let truncated=0;
|
|
%if %sysevalf(&obscnt gt &maxrows) %then %let truncated=1;
|
|
%let trunc_msg=;
|
|
%if &truncated=1 %then %do;
|
|
%let trunc_msg=Only the first &maxrows rows are loaded;
|
|
%let trunc_msg=&trunc_msg - the &limitvar limit has been reached.;
|
|
%let trunc_msg=&trunc_msg Filter the table to work with the rest.;
|
|
%end;
|
|
|
|
|
|
/* order pk fields at start of table. The delete flag is NOT added here:
|
|
_____DELETE__THIS__RECORD_____ is a client-owned column, created by the
|
|
editor itself (see deleteRecordColumnRule.ts), and getdata must never
|
|
emit it - a real table column of that name is refused by the frontend. */
|
|
%let sourcevars=%mf_wordsInStr1ButNotStr2(
|
|
Str1=%mf_getvarlist(work.out)
|
|
,Str2=&pk
|
|
);
|
|
%put sourcevars=&sourcevars;
|
|
/* the payload is capped at &maxrows - whichever limit that works out to be
|
|
- this is what goes to the browser. The cap is a data set option on the
|
|
SET, not `options obs=`, so no later step in this session can be cut by it:
|
|
a session option left set would truncate the metadata steps below, and a
|
|
table wide enough to hit the cell limit would lose columns rather than rows. */
|
|
data outdata;
|
|
/* pk fields come first */
|
|
%if %length(&pk)>0 %then %do;
|
|
attrib &pk label='';
|
|
%end;
|
|
/* keep remaining variable order */
|
|
%if %length(&sourcevars)>0 %then %do;
|
|
attrib &sourcevars label='';
|
|
%end;
|
|
%if %mf_nobs(work.out)=0 %then %do;
|
|
/* send empty row if empty table to help with hot rendering */
|
|
output;
|
|
%end;
|
|
set work.out(obs=&maxrows) ;
|
|
run;
|
|
|
|
|
|
|
|
/* get list of variables and their formats */
|
|
proc contents noprint data=outdata
|
|
out=vars(keep=name type length varnum format: label);
|
|
run;
|
|
|
|
proc sort;
|
|
by varnum;
|
|
run;
|
|
|
|
data vars3(keep=name type length format label pk varnum formatd);
|
|
set vars(rename=(format=format2 type=type2));
|
|
name=upcase(name);
|
|
/* not interested in transaction or processing dates
|
|
(append table must be supplied without them) */
|
|
if name not in ("&VAR_TXFROM","&VAR_TXTO","&VAR_PROCESSED");
|
|
if type2=2 or type2=6 then do;
|
|
length format $49.;
|
|
if format2='' then format=cats('$',length,'.');
|
|
else format=cats(format2,formatl,'.');
|
|
type='char';
|
|
end;
|
|
else do;
|
|
if format2='' then format=cats(length,'.');
|
|
else if upcase(format2)='DATETIME' and formatl=0 then format='DATETIME.';
|
|
else format=cats(format2,formatl,'.',formatd);
|
|
type='num';
|
|
end;
|
|
|
|
if name in ('',%upcase(%mf_getQuotedStr(&pk,dlm=%str(,),quote=S)))
|
|
then PK='YES';
|
|
run;
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program..sas
|
|
,msg=%str(syscc=&syscc at 242 (vars3 step) in &_program \n
|
|
%superq(syserrortext)
|
|
)
|
|
)
|
|
|
|
%global jsdttmvars jsdtvars jstmvars;
|
|
data vars4;
|
|
set vars3;
|
|
length coltype $500.;
|
|
format=upcase(format);
|
|
coltype=cats('{"data":"',name,'"');
|
|
if type='num' then do;
|
|
if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
|
|
then do;
|
|
colType=cats(coltype,',"type":"datetime"}');
|
|
/* build var list to reformat datetimes in javascript format */
|
|
call symput('jsdttmvars',symget('jsdttmvars')!!' '!!name);
|
|
end;
|
|
else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
|
|
or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
|
|
or format=:'MONYY' or format=:'NLDATE'
|
|
then do;
|
|
/* see bottom of file for more date formats!! */
|
|
/* also when updating, update stagedata.sas and mp_getcols.sas
|
|
and mpe_loader.sas */
|
|
colType=cats(coltype,',"type":"date"}');
|
|
/* build var list to reformat as javascript dates */
|
|
call symput('jsdtvars',symget('jsdtvars')!!' '!!name);
|
|
end;
|
|
else if format=:'TIME' or format=:'HHMM' then do;
|
|
colType=cats(coltype,',"type":"time"}');
|
|
/* build var list to reformat as javascript times */
|
|
call symput('jstmvars',symget('jstmvars')!!' '!!name);
|
|
end;
|
|
else do;
|
|
/* is standard numeric but need to ascertain precision */
|
|
retain base '000000000000000000';
|
|
if formatd>0 then numFormat=cats('.',substr(base,1,formatd));
|
|
colType=cats(coltype,',"type":"numeric","format":"0',numFormat,'"}');
|
|
end;
|
|
end;
|
|
else colType=cats(coltype,'}');
|
|
output;
|
|
putlog (_all_)(=);
|
|
run;
|
|
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program..sas
|
|
,msg=%str(syscc=&syscc at 283 (null step) in &_program)
|
|
)
|
|
|
|
PROC FORMAT;
|
|
picture yymmddThhmmss (default=28) other='%0Y-%0m-%0d %0H:%0M:%0s'
|
|
(datatype=datetime);
|
|
picture JSyymmdd other='%0Y-%0m-%0d' (datatype=date);
|
|
picture JShhmmss (default=16) other='%0H:%0M:%0s' (datatype=time);
|
|
RUN;
|
|
/* before we send the data, need to rebuild all date & datetime vars as char*/
|
|
%let finalvars=%mf_getvarlist(work.outdata);
|
|
data sasdata;
|
|
/* set formats & col order ahead of rename+import */
|
|
informat &finalvars ;
|
|
/* read dataset and rename date / datetime vars as necessary */
|
|
set outdata
|
|
%if %length(&jsdttmvars&jsdtvars&jstmvars)>0 %then %do;
|
|
(rename=(
|
|
%local dtvarnum dtvar tmvar;
|
|
/* temp datetime vars end in _____ */
|
|
%do dtvarnum=1 %to %sysfunc(countw(&jsdttmvars,%str( )));
|
|
%let dtvar=%scan(&jsdttmvars ,&dtvarnum);
|
|
&dtvar=_____&dtvarnum._____
|
|
%end;
|
|
/* temp date vars do not end in _____ */
|
|
%do dtvarnum=1 %to %sysfunc(countw(&jsdtvars,%str( )));
|
|
%let dtvar=%scan( &jsdtvars,&dtvarnum);
|
|
&dtvar=_____&dtvarnum
|
|
%end;
|
|
/* temp time vars end in ___tm */
|
|
%do tmvarnum=1 %to %sysfunc(countw(&jstmvars,%str( )));
|
|
%let tmvar=%scan( &jstmvars,&tmvarnum);
|
|
&tmvar=_____&tmvarnum.___tm
|
|
%end;
|
|
))
|
|
%end;
|
|
;
|
|
%if %length(&jsdttmvars)>0 %then %do ;
|
|
%do dtvarnum=1 %to %sysfunc(countw(&jsdttmvars,%str( )));
|
|
%let dtvar=%scan(&jsdttmvars,&dtvarnum);
|
|
&dtvar=cats(put(_____&dtvarnum._____,yymmddThhmmss28.));
|
|
if &dtvar="ERROR" then call missing(&dtvar);
|
|
drop _____&dtvarnum._____;
|
|
%end;
|
|
%end;
|
|
%if %length(&jsdtvars)>0 %then %do;
|
|
%do dtvarnum=1 %to %sysfunc(countw(&jsdtvars,%str( )));
|
|
%let dtvar=%scan(&jsdtvars,&dtvarnum);
|
|
&dtvar=cats(put(_____&dtvarnum,JSyymmdd.));
|
|
if &dtvar="ERROR" then call missing(&dtvar);
|
|
drop _____&dtvarnum;
|
|
%end;
|
|
%end;
|
|
%if %length(&jstmvars)>0 %then %do;
|
|
%do tmvarnum=1 %to %sysfunc(countw(&jstmvars,%str( )));
|
|
%let tmvar=%scan(&jstmvars,&tmvarnum);
|
|
&tmvar=cats(put(_____&tmvarnum.___tm,JShhmmss14.));
|
|
if &tmvar="ERROR" then call missing(&tmvar);
|
|
drop _____&tmvarnum.___tm;
|
|
%end;
|
|
%end;
|
|
output;
|
|
run;
|
|
|
|
/* get the relevant approvers for the drop down */
|
|
%put getting approvers;
|
|
%local sas_groups sas_i sas_group;
|
|
proc sql noprint;
|
|
select distinct sas_Group into: sas_groups separated by "|"
|
|
from &mpelib..mpe_security
|
|
where libref="%scan(&orig_libds,1,.)"
|
|
and dsn="%scan(&orig_libds,2,.)"
|
|
and access_level='APPROVE'
|
|
and &dc_dttmtfmt. lt TX_TO;
|
|
|
|
%if %length(&sas_groups)=0 %then %do;
|
|
%dc_getgroupmembers(&dc_admin_group,outds=work.access1)
|
|
%end;
|
|
%else %do sas_i=1 %to %sysfunc(countw(&sas_groups,%str(|)));
|
|
%let sas_group=%scan(&sas_Groups,&sas_i,%str(|));
|
|
%dc_getgroupmembers(&sas_group,outds=work.temp&sas_i)
|
|
proc append base=work.access1 data=work.temp&sas_i;run;
|
|
%end;
|
|
|
|
%mend mpestp_getdata;
|
|
|
|
%mpestp_getdata()
|
|
|
|
/* mp_abort cannot exit cleanly from inside a %include, so an abort there
|
|
(eg in a hook script run via mp_include) only records the error in
|
|
work.mp_abort_errds. mode=INCLUDE picks that up and aborts properly here. */
|
|
%mp_abort(mode=INCLUDE)
|
|
|
|
/* extract column level security rules */
|
|
%mpe_columnlevelsecurity(%scan(&libds,1,.),%scan(&libds,2,.),work.sasdata
|
|
,mode=EDIT
|
|
,clsds=&mpelib..mpe_column_level_security
|
|
,groupds=work.groups /* was created in mpe_filtermaster */
|
|
,outds=work.sasdata1
|
|
,outmeta=work.cls_rules
|
|
)
|
|
|
|
/* get labels - pull column names from the CLS-filtered WORK dataset but
|
|
join to mpe_datadictionary using the underlying LIB.DSN reference */
|
|
%mpe_getlabels(COLUMNS,sasdata1,tgt_ds=&libds,outds=spec)
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program
|
|
,msg=%str(syscc=&syscc extracting spec info)
|
|
)
|
|
|
|
/* extract col info */
|
|
%mp_getcols(&libds, outds=cols1)
|
|
|
|
/* derived variables are added to the view, so take their attributes from
|
|
there - this keeps type, length, format and label identical to the data */
|
|
%if &derived_nobs>0 %then %do;
|
|
%mp_getcols(work.vw_derived, outds=derived_cols_all)
|
|
proc sql;
|
|
create table work.derived_cols as
|
|
select a.*
|
|
from work.derived_cols_all a
|
|
where a.NAME in (select NAME from work.derived_vars);
|
|
quit;
|
|
data work.cols1;
|
|
set work.cols1 work.derived_cols;
|
|
run;
|
|
%end;
|
|
|
|
/* join with cls rules. Trim COLS payload in the SQL below - DDTYPE
|
|
C=CHARACTER N=NUMERIC (dates kept in full), VARNUM dropped (client
|
|
relies on column order, not position) */
|
|
proc sql;
|
|
create table work.cols as
|
|
select a.NAME
|
|
,coalesce(c.desc,a.LABEL,a.NAME) as LABEL
|
|
,a.FMTNAME
|
|
,case a.DDTYPE
|
|
when 'CHARACTER' then 'C'
|
|
when 'NUMERIC' then 'N'
|
|
else a.DDTYPE end as DDTYPE
|
|
,case b.cls_hide
|
|
when 1 then 'HIDE'
|
|
when 0 then 'EDIT'
|
|
else 'READ' end as CLS_RULE
|
|
,c.memlabel
|
|
,c.longdesc
|
|
,d.colType
|
|
from work.cols1(drop=varnum) a
|
|
left join work.cls_rules b
|
|
on a.NAME=b.CLS_VARIABLE_NM
|
|
left join work.spec c
|
|
on a.NAME=c.NAME
|
|
left join work.vars4 d
|
|
on a.NAME=d.NAME;
|
|
|
|
proc sql;
|
|
create table approvers as select distinct membername as personname
|
|
,membername as email, membername as userid
|
|
from work.access1;
|
|
/*
|
|
create table access3 as select b.userid,b.email
|
|
from access2 a
|
|
,support.users b
|
|
where a.personname=b.userid
|
|
and a.personname ne "%mf_getuser()"
|
|
and %sysfunc(datetime()) lt b.tx_to_dttm
|
|
order by 1;
|
|
*/
|
|
data _null_;
|
|
infile filtref end=eof;
|
|
input;
|
|
length filter_text $32767;
|
|
retain filter_text;
|
|
filter_text=catx(' ',filter_text,_infile_);
|
|
if eof then do;
|
|
if cats(filter_text)='1=1' then filter_text='';
|
|
call symputx('filter_text',filter_text);
|
|
end;
|
|
run;
|
|
|
|
%put params;
|
|
%let ismap=0;
|
|
proc sql noprint;
|
|
select count(*) into: ismap from &mpelib..mpe_xlmap_info
|
|
where XLMAP_TARGETLIBDS="&orig_libds" and &dc_dttmtfmt. le TX_TO;
|
|
|
|
data sasparams;
|
|
length colHeaders $20000 filter_text $32767 trunc_msg $200;
|
|
colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
|
|
pkCnt=countw("&pk");
|
|
pk="&pk";
|
|
dtvars=compbl("&jsdtvars");
|
|
dttmvars=compbl("&jsdttmvars");
|
|
tmvars=compbl("&jstmvars");
|
|
loadtype=symget('loadtype');
|
|
if trim(symget('rk_underlying')) ne '' then rk_flag=1;
|
|
else rk_flag=0;
|
|
filter_text=symget('filter_text');
|
|
/* set when the selection was cut down to the row / cell limit, so the EDIT
|
|
screen can say so against the table name */
|
|
truncated=&truncated;
|
|
trunc_msg="&trunc_msg";
|
|
if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
|
|
else cls_flag=1;
|
|
put (_all_)(=);
|
|
if "&orig_libds"="&mpelib..MPE_XLMAP_DATA" or &ismap ne 0 then ismap=1;
|
|
else ismap=0;
|
|
run;
|
|
|
|
|
|
/* Extract validation DQ Rules */
|
|
proc sort data=&mpelib..mpe_validations
|
|
(where=(&dc_dttmtfmt. le TX_TO
|
|
and BASE_LIB="%scan(&orig_libds,1,.)" and BASE_DS="%scan(&orig_libds,2,.)"
|
|
and rule_active=1))
|
|
out=dqrules (keep=base_col rule_type rule_value);
|
|
by base_col rule_type rule_value;
|
|
run;
|
|
|
|
/* merge with NOTNULL constraints in the physical table, and mark the derived
|
|
variables read-only: a derived column is computed at read time, so the grid
|
|
must not accept a value for it. COLS carries the same READ (a derived column
|
|
has no CLS rule row) but the client only applies COLS.CLS_RULE when CLS_FLAG
|
|
is set, which is driven by the table's column-level security - so the rule is
|
|
declared here, where it holds regardless.
|
|
|
|
work.derived_vars exists on every path that reaches this step: %mpe_derived
|
|
writes it before returning, empty when the table has no derived variables,
|
|
so the union below needs no guard */
|
|
proc sql;
|
|
create table _data_ as
|
|
select * from dqrules
|
|
union
|
|
select upcase(name) as base_col
|
|
,'NOTNULL' as rule_type
|
|
,'' as rule_value
|
|
from dictionary.columns
|
|
where upcase(libname)="%scan(&orig_libds,1,.)"
|
|
and upcase(memname)="%scan(&orig_libds,2,.)"
|
|
and upcase(name) in (select name from vars3)
|
|
and notnull='yes'
|
|
union
|
|
select upcase(name) as base_col
|
|
,'READONLY' as rule_type
|
|
,'' as rule_value
|
|
from work.derived_vars
|
|
order by 1,2,3;
|
|
data dqrules;
|
|
set &syslast;
|
|
by base_col rule_type rule_value;
|
|
if last.rule_type;
|
|
if rule_type in ('HARDSELECT','SOFTSELECT') and countw(rule_value)=3 then
|
|
do;
|
|
retain x 0; x+1;
|
|
call symputx(cats('source',x),rule_value);
|
|
%let sourcecnt=0;
|
|
call symputx('sourcecnt',x);
|
|
call symputx(cats('base_col',x),base_col);
|
|
end;
|
|
run;
|
|
|
|
proc sql;
|
|
create table dqdata as
|
|
select distinct base_column as base_col length=32
|
|
,upcase(base_column) as rule_value length=74 /* deprecated */
|
|
,selectbox_value as rule_data length=1000
|
|
,selectbox_order
|
|
from &mpelib..mpe_selectbox
|
|
where &dc_dttmtfmt. lt ver_to_dttm
|
|
and select_lib="%scan(&orig_libds,1,.)"
|
|
and select_ds="%scan(&orig_libds,2,.)";
|
|
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program
|
|
,msg=%str(syscc=&syscc during DQ rule validation)
|
|
)
|
|
|
|
/* extract selectbox data */
|
|
%macro dq_selects();
|
|
%local x source lib ds col filtref filterds srcview;
|
|
%do x=1 %to &sourcecnt;
|
|
%let source=&&source&x;
|
|
%let lib=%scan(&source,1,.);
|
|
%let ds=%scan(&source,2,.);
|
|
%let col=%scan(&source,3,.);
|
|
%put &=source;
|
|
%put &=lib;
|
|
%dc_assignlib(READ,&lib)
|
|
%dc_casload(&lib..&ds)
|
|
/* Selectbox values are read straight from the source table, so the same
|
|
Row Level Security that filters the grid must be applied here too -
|
|
otherwise values from rows the user is not permitted to see leak into
|
|
the cell dropdown. %mpe_filtermaster builds the filter; a data step
|
|
view is used because a fileref cannot be %inc'd inside a proc sql where
|
|
clause. EDIT mode mirrors the grid on this (editor) page. */
|
|
%let filtref=%mf_getuniquefileref();
|
|
%let filterds=%mf_getuniquename();
|
|
%mpe_filtermaster(EDIT,&lib..&ds,
|
|
dclib=&mpelib,
|
|
outref=&filtref,
|
|
outds=work.&filterds
|
|
)
|
|
%let srcview=%mf_getuniquename();
|
|
data work.&srcview / view=work.&srcview;
|
|
set &lib..&ds;
|
|
where %inc &filtref;;
|
|
run;
|
|
proc sql;
|
|
create table dqdata&x as
|
|
select distinct "&&base_col&x" as base_col length=32
|
|
,"&source" as rule_value length=74
|
|
,cats(&col) as rule_data length=1000
|
|
,&col as tmp_order
|
|
from work.&srcview
|
|
order by tmp_order;
|
|
/* ensure both numerics and char vals are ordered correctly */
|
|
data work.dqdata&x (drop=tmp_order);
|
|
set work.dqdata&x;
|
|
selectbox_order=_n_;
|
|
run;
|
|
%mp_abort(iftrue= (&syscc ne 0)
|
|
,mac=&_program
|
|
,msg=%str(syscc=&syscc when selecting &&base_col&x from &orig_libds)
|
|
)
|
|
proc append base=dqdata data=dqdata&x;run;
|
|
proc sql; drop table dqdata&x;
|
|
/* drop the per-source filter artifacts - WORK is session scoped, but the
|
|
loop should not leave a view and a filter table behind per source */
|
|
proc datasets lib=work nolist;
|
|
delete &srcview &filterds;
|
|
quit;
|
|
filename &filtref clear;
|
|
%end;
|
|
%mend dq_selects;
|
|
%dq_selects()
|
|
|
|
proc sort data=dqdata;
|
|
/* order by selectbox_order then the value */
|
|
by base_col selectbox_order rule_data;
|
|
run;
|
|
|
|
%mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)
|
|
|
|
data xl_rules;
|
|
set &mpelib..mpe_excel_config;
|
|
where &dc_dttmtfmt. lt tx_to;
|
|
where also upcase(xl_libref)="%scan(&orig_libds,1,.)";
|
|
where also upcase(xl_table)="%scan(&orig_libds,2,.)";
|
|
where also xl_active=1;
|
|
keep xl_column xl_rule;
|
|
run;
|
|
|
|
%mpe_dsmeta(&orig_libds, outds=dsmeta)
|
|
|
|
%mpe_getversions(&mpelib,
|
|
%scan(&orig_libds,1,.),
|
|
%scan(&orig_libds,2,.),
|
|
outds=versions
|
|
)
|
|
|
|
|
|
/* send to the client */
|
|
%webout(OPEN)
|
|
%webout(OBJ,approvers)
|
|
%webout(OBJ,cols)
|
|
%webout(OBJ,dqdata,missing=STRING)
|
|
%webout(OBJ,dqrules)
|
|
%webout(OBJ,dsmeta)
|
|
%webout(OBJ,maxvarlengths)
|
|
%webout(OBJ,query)
|
|
%webout(OBJ,sasdata1,fmt=N,missing=STRING,showmeta=YES,dslabel=sasdata)
|
|
%webout(OBJ,sasparams)
|
|
%webout(OBJ,versions)
|
|
%webout(OBJ,xl_rules)
|
|
%webout(CLOSE)
|
|
|
|
/*
|
|
$N8601Bw
|
|
$N8601BAw
|
|
$N8601Ew
|
|
$N8601EAw
|
|
$N8601EHw
|
|
$N8601EXw
|
|
$N8601Hw
|
|
$N8601Xw
|
|
B8601DAw
|
|
B8601DNw
|
|
B8601DTw
|
|
B8601DZw
|
|
B8601LZw
|
|
B8601TMw
|
|
B8601TZw
|
|
DATEw
|
|
DATEAMPMw
|
|
DATETIMEw
|
|
DAYw
|
|
DDMMYYw
|
|
DDMMYYxw
|
|
DOWNAMEw
|
|
DTDATEw
|
|
DTMONYYw
|
|
DTWKDATXw
|
|
DTYEARw
|
|
DTYYQCw
|
|
E8601DAw
|
|
E8601DNw
|
|
E8601DTw
|
|
E8601DZw
|
|
E8601LZw
|
|
E8601TMw
|
|
E8601TZw
|
|
HHMMw
|
|
HOURw
|
|
JULDAYw
|
|
JULIANw
|
|
MMDDYYw
|
|
MMDDYYxw
|
|
MMSSw
|
|
MMYYw
|
|
MMYYxw
|
|
MONNAMEw
|
|
MONTHw
|
|
MONYYw
|
|
PDJULGw
|
|
PDJULIw
|
|
QTRw
|
|
QTRRw
|
|
TIMEw
|
|
TIMEAMPMw
|
|
TODw
|
|
WEEKDATEw
|
|
WEEKDATXw
|
|
WEEKDAYw
|
|
WEEKUw
|
|
WEEKVw
|
|
WEEKWw
|
|
WORDDATEw
|
|
WORDDATXw
|
|
YEARw
|
|
YYMMw
|
|
YYMMxw
|
|
YYMMDDw
|
|
YYMMDDxw
|
|
YYMONw
|
|
YYQw
|
|
YYQxw
|
|
YYQRw
|
|
YYQRxw
|
|
$N8601BAw
|
|
$N8601Ew
|
|
$N8601EAw
|
|
$N8601EHw
|
|
$N8601EXw
|
|
$N8601Hw
|
|
$N8601Xw
|
|
B8601DAw
|
|
B8601DNw
|
|
B8601DTw
|
|
B8601DZw
|
|
B8601LZw
|
|
B8601TMw
|
|
B8601TZw
|
|
E8601DAw
|
|
E8601DNw
|
|
E8601DTw
|
|
E8601DZw
|
|
E8601LZw
|
|
E8601TMw
|
|
E8601TZw
|
|
*/
|
|
%mpeterm()
|