dc/sas/sasjs/services/editors/getdata.sas
Mihajlo Medjedovic f268de21a3
Some checks failed
Test / Build-and-test-development (push) Failing after 6m14s
Test / Build-and-test-development-latest-adapter (push) Failing after 6m13s
init
2023-07-13 13:44:05 +02:00

771 lines
20 KiB
SAS
Executable File

/**
@file getdata.sas
@brief Returns a dataset to the editor front end
@details
<h4> Service Inputs </h4>
<h5> SASCONTROLTABLE </h5>
|LIBDS:$41.|FILTER_RK:$5.|
|---|---|
|DC258467.MPE_X_TEST|-1|
<h4> Service Outputs </h4>
<h5> sasdata </h5>
<h5> sasparams </h5>
Contains info on the request. One row is returned.
* CLS_FLG - set to 0 if there are no CLS rules (everything should be editable)
else set to 1 (CLS rules exist)
<h5> approvers </h5>
<h5> dqrules </h5>
<h5> dqdata </h5>
<h5> cols </h5>
Contains column level attributes.
@li NAME - column name
@li VARNUM - variable position. Source: https://core.sasjs.io/mp__getcols_8sas.html
@li LABEL - variable label. Source: https://core.sasjs.io/mp__getcols_8sas.html
@li FMTNAME - derived format name. Source: https://core.sasjs.io/mp__getcols_8sas.html
@li DDTYPE - derived dropdown type. Source: https://core.sasjs.io/mp__getcols_8sas.html
@li CLS_RULE - values include:
- EDIT - the column is editable
- READ - the column should be readonly
- HIDE - the column should be hidden
@li memlabel
@li desc- augmented with MPE_DATADICTIONARY if exists, else label
@li longdesc - from MPE_DATADICTIONARY
<h5> maxvarlengths </h5>
<h5> xl_rules </h5>
<h5> query </h5>
<h4> SAS Macros </h4>
@li dc_assignlib.sas
@li dc_getgroupmembers.sas
@li mf_existvar.sas
@li mf_getattrn.sas
@li mf_getvarlist.sas
@li mf_existds.sas
@li mf_getquotedstr.sas
@li mf_getuser.sas
@li mf_nobs.sas
@li mf_verifymacvars.sas
@li mf_wordsinstr1butnotstr2.sas
@li mp_abort.sas
@li mp_cntlout.sas
@li mp_dsmeta.sas
@li mp_getcols.sas
@li mp_getmaxvarlengths.sas
@li mp_validatecol.sas
@li mpe_accesscheck.sas
@li mpe_columnlevelsecurity.sas
@li mpe_getlabels.sas
@li mpe_filtermaster.sas
@li mpe_runhook.sas
@version 9.2
@author 4GL Apps Ltd
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
and may not be re-distributed or re-sold without the express permission of
4GL Apps Ltd.
**/
%mpeinit()
/**
* Validate inputs
*/
data work.intest;
length filter_rk 8;
set work.SASCONTROLTABLE;
/* validate filter_rk */
if filter_rk le 0 then filter_rk=-1;
call symputx('orig_libds',upcase(libds));
is_fmt=0;
if substr(cats(reverse(libds)),1,3)=:'CF-' then do;
libds=scan(libds,1,'-');
putlog "Format Catalog Captured";
is_fmt=1;
libds='work.fmtextract';
call symputx('libds',libds);
end;
call symputx('is_fmt',is_fmt);
putlog (_all_)(=);
/* validate libds */
%mp_validatecol(LIBDS,LIBDS,is_libds)
if is_libds=0 then do;
putlog 'ERR' 'OR: Invalid libds:' libds;
stop;
end;
else do;
call symputx('filter_rk',filter_rk);
call symputx('libds',libds);
end;
output;
stop;
run;
%mp_abort(iftrue= (%mf_nobs(work.intest)=0)
,mac=&_program
,msg=%str(Some err with service inputs)
)
%mp_abort(
iftrue=(%mf_verifymacvars(libds filter_rk)=0)
,mac=&_program
,msg=%str(Missing: libds filter_rk)
)
/* export format catalog */
%mp_cntlout(
iftrue=(&is_fmt=1)
,libcat=&orig_libds
,fmtlist=0
,cntlout=work.fmtextract
)
/* stream back meta info, further calls will return col metadata and actual data
*/
%let libref=%upcase(%scan(&libds,1,.));
%let dsn=%upcase(%scan(&libds,2,.));
%dc_assignlib(WRITE,&libref)
/**
* First check user has access permission to edit the table
*/
%put checking access;
%let user=%mf_getuser();
%mpe_accesscheck(&orig_libds,outds=mw_auth,user=&user,access_level=EDIT)
%mp_abort(iftrue= (%mf_getattrn(work.mw_auth,NLOBS)=0)
,mac=mpestp_getdata.sas
,msg=&user is not authorised to edit &orig_libds %trim(
)in the &mpelib..MPE_SECURITY table
)
%mp_abort(iftrue= ( %mf_existds(libds=&libds) ne 1)
,mac=mpestp_getdata.sas
,msg=dataset &libds does not exist!!
)
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc at line 60 )
)
%global loadtype var_txfrom var_txto var_processed filter_text pk coltype
sortpk;
%put getting table attributes;
proc sql noprint;
select upcase(loadtype)
,var_txfrom,var_txto
,var_busfrom,var_busto
,var_processed,rk_underlying,buskey
,coalesce(rk_underlying,buskey)
,pre_edit_hook
,case when missing(rk_underlying) then buskey else rk_underlying end
into: loadtype,:var_txfrom,:var_txto
,:var_busfrom ,:var_busto
,:var_processed,:rk_underlying,:buskey, :sortPK, :pre_edit_hook,:pk
from &mpelib..mpe_tables
where &dc_dttmtfmt. lt TX_TO
and upcase(dsn)="%scan(&orig_libds,2,.)"
and upcase(libref)="%scan(&orig_libds,1,.)";
%put preparing filter query:;
%mpe_filtermaster(EDIT,&orig_libds,
dclib=&mpelib,
filter_rk=&filter_rk,
outref=filtref,
outds=work.query
)
%macro mpestp_getdata();
%if not %symexist(DC_MAXOBS_WEBEDIT) %then %do;
%put NOTE:;%put NOTE- DC_MAXOBS_WEBEDIT not found!;
%put NOTE- Please add to &mpelib..MPE_CONFIG table;
%put NOTE-;%put NOTE-;
%global DC_MAXOBS_WEBEDIT;
%let DC_MAXOBS_WEBEDIT=500;
%end;
/* for tables which use RKs/SKs then we just expose the business key to
users - this lets uploads be sent to multiple environments (with
potentially different RK/SK values for the same business key).
Note that the config table has the RK column in the buskey field in
this scenario. */
%if %length(&rk_underlying)>0 %then %let drop_rk=&buskey;
%else %let drop_rk=;
/* always remove the PROCESSED_DTTM column, if it exists */
%if %length(&var_processed)=0 %then %do;
%if %mf_existvar(&libds,PROCESSED_DTTM)>0 %then
%let var_processed=PROCESSED_DTTM;
%end;
/**
* Now get the slice of the actual table
*/
options obs=10000;
%if &loadtype=BITEMPORAL %then %do;
data out (drop=&var_txfrom &var_txto &var_processed &drop_rk );
_____DELETE__THIS__RECORD_____="No";
set &libds;
where %inc filtref;;
run;
proc sort data=out;
by &pk &var_busfrom;
run;
data out;
set out;
by &pk &var_busfrom;
if last.%scan(&pk,-1);
run;
%end;
%else %do;
data out (drop=&var_txfrom &var_txto &var_processed &drop_rk);
_____DELETE__THIS__RECORD_____="No";
set &libds;
where %inc filtref;;
run;
%end;
options obs=max;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program
,msg=%str(Issue with filtering (line 165) )
)
options obs=&DC_MAXOBS_WEBEDIT;
%let sortpk=%sysfunc(coalescec(&sortpk &var_busfrom,_ALL_));
proc sort data=work.out; by &sortPK; run;
options obs=max;
%mpe_runhook(PRE_EDIT_HOOK)
%let obscnt=%mf_getattrn(work.out,NLOBS);
%mp_abort(iftrue=(&obscnt>&DC_MAXOBS_WEBEDIT)
,mac=&_program
,msg=Table is too big (&obscnt rows) - please filter and try again!
)
/* order delete var and pk fields at start of table */
%let sourcevars=%mf_wordsInStr1ButNotStr2(
Str1=%mf_getvarlist(work.out)
,Str2= _____DELETE__THIS__RECORD_____ &pk
);
%put sourcevars=&sourcevars;
data outdata;
/* delete & pk fields come first */
attrib _____DELETE__THIS__RECORD_____ &pk label='';
/* keep remaining variable order */
%if %length(&sourcevars)>0 %then %do;
attrib &sourcevars label='';
%end;
_____DELETE__THIS__RECORD_____="No ";
%if %mf_nobs(work.out)=0 %then %do;
/* send empty row if empty table to help with hot rendering */
output;
%end;
set work.out ;
run;
/* get list of variables and their formats */
proc contents noprint data=outdata
out=vars(keep=name type length varnum format: label);
run;
proc sort;
by varnum;
run;
data vars3(keep=name type length format label pk varnum ctrloptions formatd);
set vars(rename=(format=format2 type=type2));
name=upcase(name);
/* not interested in transaction or processing dates
(append table must be supplied without them) */
if name not in ("&VAR_TXFROM","&VAR_TXTO","&VAR_PROCESSED");
if type2=2 or type2=6 then do;
length format $49.;
if format2='' then format=cats('$',length,'.');
else format=cats(format2,formatl,'.');
type='char';
end;
else do;
if format2='' then format=cats(length,'.');
else if upcase(format2)='DATETIME' and formatl=0 then format='DATETIME.';
else format=cats(format2,formatl,'.',formatd);
type='num';
end;
if name in ('',%upcase(%mf_getQuotedStr(&pk,dlm=%str(,),quote=S)))
then PK='YES';
length ctrlOptions $500;
if name="_____DELETE__THIS__RECORD_____" then ctrlOptions='["No","Yes"]';
else ctrlOptions='';
run;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc at 242 (vars3 step) in &_program \n
%superq(syserrortext)
)
)
%global jsdttmvars jsdtvars jstmvars;
data _null_;
set vars3 end=last;
if _n_>1 then comma=',';
length coltype $500.;
format=upcase(format);
coltype=cats(comma,'{"data":"',name,'"');
if ctrlOptions ne '' then
colType=cats(coltype,',"type":"dropdown","source":',ctrlOptions,"}");
else if type='num' then do;
if format=:'DATETIME' or format=:'E8601DT' then do;
colType=cats(coltype
,',"type":"date","dateFormat":"YYYY-MM-DD HH:mm:ss"'
,',"correctFormat":"true"}');
/* build var list to reformat datetimes in javascript format */
call symput('jsdttmvars',symget('jsdttmvars')!!' '!!name);
end;
else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
or format=:'MONYY'
then do;
/* see bottom of file for more date formats!! */
/* also when updating, update stagedata.sas and mp_getcols.sas
and mpe_loader.sas */
colType=cats(coltype,',"type":"date","dateFormat":"YYYY-MM-DD"'
/*colType=cats(coltype,',"type":"date","dateFormat":"MM/DD/YYYY"'*/
,',"correctFormat":"true"}');
/* build var list to reformat as javascript dates */
call symput('jsdtvars',symget('jsdtvars')!!' '!!name);
end;
else if format=:'TIME' or format=:'HHMM' then do;
colType=cats(coltype,',"type":"time","timeFormat":"HH:mm:ss"'
,',"correctFormat":"true"}');
/* build var list to reformat as javascript times */
call symput('jstmvars',symget('jstmvars')!!' '!!name);
end;
else do;
/* is standard numeric but need to ascertain precision */
retain base '000000000000000000';
if formatd>0 then numFormat=cats('.',substr(base,1,formatd));
colType=cats(coltype,',"type":"numeric","format":"0',numFormat,'"}');
end;
end;
else colType=cats(coltype,'}');
length concatcoltype $32767;
retain concatcoltype;
concatcoltype=cats(concatcoltype,coltype);
if last then call symputx('colType',strip(concatcoltype),'g');
putlog (_all_)(=);
run;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program..sas
,msg=%str(syscc=&syscc at 283 (null step) in &_program)
)
PROC FORMAT;
picture yymmddThhmmss (default=28) other='%0Y-%0m-%0d %0H:%0M:%0s'
(datatype=datetime);
picture JSyymmdd other='%0Y-%0m-%0d' (datatype=date);
picture JShhmmss (default=16) other='%0H:%0M:%0s' (datatype=time);
RUN;
/* before we send the data, need to rebuild all date & datetime vars as char*/
%let finalvars=%mf_getvarlist(work.outdata);
data sasdata;
/* set formats & col order ahead of rename+import */
informat &finalvars ;
/* read dataset and rename date / datetime vars as necessary */
set outdata
%if %length(&jsdttmvars&jsdtvars&jstmvars)>0 %then %do;
(rename=(
%local dtvarnum dtvar tmvar;
/* temp datetime vars end in _____ */
%do dtvarnum=1 %to %sysfunc(countw(&jsdttmvars,%str( )));
%let dtvar=%scan(&jsdttmvars ,&dtvarnum);
&dtvar=_____&dtvarnum._____
%end;
/* temp date vars do not end in _____ */
%do dtvarnum=1 %to %sysfunc(countw(&jsdtvars,%str( )));
%let dtvar=%scan( &jsdtvars,&dtvarnum);
&dtvar=_____&dtvarnum
%end;
/* temp time vars end in ___tm */
%do tmvarnum=1 %to %sysfunc(countw(&jstmvars,%str( )));
%let tmvar=%scan( &jstmvars,&tmvarnum);
&tmvar=_____&tmvarnum.___tm
%end;
))
%end;
;
%if %length(&jsdttmvars)>0 %then %do ;
%do dtvarnum=1 %to %sysfunc(countw(&jsdttmvars,%str( )));
%let dtvar=%scan(&jsdttmvars,&dtvarnum);
&dtvar=cats(put(_____&dtvarnum._____,yymmddThhmmss28.));
if &dtvar="ERROR" then call missing(&dtvar);
drop _____&dtvarnum._____;
%end;
%end;
%if %length(&jsdtvars)>0 %then %do;
%do dtvarnum=1 %to %sysfunc(countw(&jsdtvars,%str( )));
%let dtvar=%scan(&jsdtvars,&dtvarnum);
&dtvar=cats(put(_____&dtvarnum,JSyymmdd.));
if &dtvar="ERROR" then call missing(&dtvar);
drop _____&dtvarnum;
%end;
%end;
%if %length(&jstmvars)>0 %then %do;
%do tmvarnum=1 %to %sysfunc(countw(&jstmvars,%str( )));
%let tmvar=%scan(&jstmvars,&tmvarnum);
&tmvar=cats(put(_____&tmvarnum.___tm,JShhmmss14.));
if &tmvar="ERROR" then call missing(&tmvar);
drop _____&tmvarnum.___tm;
%end;
%end;
output;
run;
/* get the relevant approvers for the drop down */
%put getting approvers;
%local sas_groups sas_i sas_group;
proc sql noprint;
select distinct sas_Group into: sas_groups separated by "|"
from &mpelib..mpe_security
where libref="%scan(&orig_libds,1,.)"
and dsn="%scan(&orig_libds,2,.)"
and access_level='APPROVE'
and &dc_dttmtfmt. lt TX_TO;
%if %length(&sas_groups)=0 %then %do;
%dc_getgroupmembers(&dc_admin_group,outds=work.access1)
%end;
%else %do sas_i=1 %to %sysfunc(countw(&sas_groups,%str(|)));
%let sas_group=%scan(&sas_Groups,&sas_i,%str(|));
%dc_getgroupmembers(&sas_group,outds=work.temp&sas_i)
proc append base=work.access1 data=work.temp&sas_i;run;
%end;
%mend mpestp_getdata;
%mpestp_getdata()
%mp_abort(mode=INCLUDE)
/* extract column level security rules */
%mpe_columnlevelsecurity(%scan(&libds,1,.),%scan(&libds,2,.),work.sasdata
,mode=EDIT
,clsds=&mpelib..mpe_column_level_security
,groupds=work.groups /* was created in mpe_filtermaster */
,outds=work.sasdata1
,outmeta=work.cls_rules
)
/* get labels */
%mpe_getlabels(COLUMNS,sasdata1,outds=spec)
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program
,msg=%str(syscc=&syscc extracting spec info)
)
/* extract col info */
%mp_getcols(&libds, outds=cols1)
/* join with cls rules */
proc sql;
create table work.cols as
select a.NAME
,a.VARNUM
,a.LABEL
,a.FMTNAME
,a.DDTYPE
,case b.cls_hide
when 1 then 'HIDE'
when 0 then 'EDIT'
else 'READ' end as CLS_RULE
,c.memlabel
,c.desc
,c.longdesc
from work.cols1 a
left join work.cls_rules b
on a.NAME=b.CLS_VARIABLE_NM
left join work.spec c
on a.NAME=c.NAME;
proc sql;
create table approvers as select distinct membername as personname
,membername as email, membername as userid
from work.access1;
/*
create table access3 as select b.userid,b.email
from access2 a
,support.users b
where a.personname=b.userid
and a.personname ne "%mf_getuser()"
and %sysfunc(datetime()) lt b.tx_to_dttm
order by 1;
*/
data _null_;
infile filtref end=eof;
input;
length filter_text $32767;
retain filter_text;
filter_text=catx(' ',filter_text,_infile_);
if eof then do;
if cats(filter_text)='1=1' then filter_text='';
call symputx('filter_text',filter_text);
end;
run;
%put params;
data sasparams;
length colHeaders $20000 filter_text $32767;
colHeaders=cats(upcase("%mf_getvarlist(sasdata1,dlm=%str(,))"));
pkCnt=countw("&pk");
pk="&pk";
dtvars=compbl("&jsdtvars");
dttmvars=compbl("&jsdttmvars");
tmvars=compbl("&jstmvars");
length coltype $32000;
coltype=symget('coltype');
loadtype=symget('loadtype');
if trim(symget('rk_underlying')) ne '' then rk_flag=1;
else rk_flag=0;
filter_text=symget('filter_text');
if %mf_nobs(work.cls_rules)=0 then cls_flag=0;
else cls_flag=1;
put (_all_)(=);
run;
/* Extract validation DQ Rules */
proc sort data=&mpelib..mpe_validations
(where=(&dc_dttmtfmt. le TX_TO
and BASE_LIB="%scan(&orig_libds,1,.)" and BASE_DS="%scan(&orig_libds,2,.)"
and rule_active=1))
out=dqrules (keep=base_col rule_type rule_value);
by base_col rule_type rule_value;
run;
/* merge with NOTNULL constraints in the physical table */
proc sql;
create table _data_ as
select * from dqrules
union
select upcase(name) as base_col
,'NOTNULL' as rule_type
,'' as rule_value
from dictionary.columns
where upcase(libname)="%scan(&orig_libds,1,.)"
and upcase(memname)="%scan(&orig_libds,2,.)"
and upcase(name) in (select name from vars3)
and notnull='yes'
order by 1,2,3;
data dqrules;
set &syslast;
by base_col rule_type rule_value;
if last.rule_type;
if rule_type in ('HARDSELECT','SOFTSELECT') and countw(rule_value)=3 then
do;
retain x 0; x+1;
call symputx(cats('source',x),rule_value);
%let sourcecnt=0;
call symputx('sourcecnt',x);
call symputx(cats('base_col',x),base_col);
end;
run;
proc sql;
create table dqdata as
select distinct base_column as base_col length=32
,upcase(base_column) as rule_value length=74 /* deprecated */
,selectbox_value as rule_data length=1000
,selectbox_order
from &mpelib..mpe_selectbox
where &dc_dttmtfmt. lt ver_to_dttm
and select_lib="%scan(&orig_libds,1,.)"
and select_ds="%scan(&orig_libds,2,.)";
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program
,msg=%str(syscc=&syscc during DQ rule validation)
)
/* extract selectbox data */
%macro dq_selects();
%local x source lib ds col;
%do x=1 %to &sourcecnt;
%let source=&&source&x;
%let lib=%scan(&source,1,.);
%let ds=%scan(&source,2,.);
%let col=%scan(&source,3,.);
%put &=source;
%put &=lib;
%dc_assignlib(READ,&lib)
proc sql;
create table dqdata&x as
select distinct "&&base_col&x" as base_col length=32
,"&source" as rule_value length=74
,cats(&col) as rule_data length=1000
,0 as selectbox_order
from &lib..&ds
order by 1;
%mp_abort(iftrue= (&syscc ne 0)
,mac=&_program
,msg=%str(syscc=&syscc when selecting &&base_col&x from &orig_libds)
)
proc append base=dqdata data=dqdata&x;run;
proc sql; drop table dqdata&x;
%end;
%mend dq_selects;
%dq_selects()
proc sort data=dqdata;
by base_col selectbox_order;
run;
%mp_getmaxvarlengths(work.sasdata1,outds=maxvarlengths)
data maxvarlengths;
set maxvarlengths;
if name='_____DELETE__THIS__RECORD_____' then mAXLEN=3;
run;
data xl_rules;
set &mpelib..mpe_excel_config;
where &dc_dttmtfmt. lt tx_to;
where also upcase(xl_libref)="%scan(&orig_libds,1,.)";
where also upcase(xl_table)="%scan(&orig_libds,2,.)";
where also xl_active=1;
keep xl_column xl_rule;
run;
%mp_dsmeta(&libds, outds=dsmeta)
/* send to the client */
%webout(OPEN)
%webout(OBJ,approvers)
%webout(OBJ,cols)
%webout(OBJ,dqdata)
%webout(OBJ,dqrules)
%webout(OBJ,dsmeta)
%webout(OBJ,maxvarlengths)
%webout(OBJ,query)
%webout(OBJ,sasdata1,fmt=N,missing=STRING,showmeta=YES,dslabel=sasdata)
%webout(OBJ,sasparams)
%webout(OBJ,xl_rules)
%webout(CLOSE)
/*
$N8601Bw
$N8601BAw
$N8601Ew
$N8601EAw
$N8601EHw
$N8601EXw
$N8601Hw
$N8601Xw
B8601DAw
B8601DNw
B8601DTw
B8601DZw
B8601LZw
B8601TMw
B8601TZw
DATEw
DATEAMPMw
DATETIMEw
DAYw
DDMMYYw
DDMMYYxw
DOWNAMEw
DTDATEw
DTMONYYw
DTWKDATXw
DTYEARw
DTYYQCw
E8601DAw
E8601DNw
E8601DTw
E8601DZw
E8601LZw
E8601TMw
E8601TZw
HHMMw
HOURw
JULDAYw
JULIANw
MMDDYYw
MMDDYYxw
MMSSw
MMYYw
MMYYxw
MONNAMEw
MONTHw
MONYYw
PDJULGw
PDJULIw
QTRw
QTRRw
TIMEw
TIMEAMPMw
TODw
WEEKDATEw
WEEKDATXw
WEEKDAYw
WEEKUw
WEEKVw
WEEKWw
WORDDATEw
WORDDATXw
YEARw
YYMMw
YYMMxw
YYMMDDw
YYMMDDxw
YYMONw
YYQw
YYQxw
YYQRw
YYQRxw
$N8601BAw
$N8601Ew
$N8601EAw
$N8601EHw
$N8601EXw
$N8601Hw
$N8601Xw
B8601DAw
B8601DNw
B8601DTw
B8601DZw
B8601LZw
B8601TMw
B8601TZw
E8601DAw
E8601DNw
E8601DTw
E8601DZw
E8601LZw
E8601TMw
E8601TZw
*/
%mpeterm()