dc/sas/sasjs/macros/mpe_makedatamodel.sas

633 lines
19 KiB
SAS

/**
@file
@brief Creates the empty Data Controller tables
@details Creates the empty DC tables during the initial deployment.
Usage:
%mpe_makedatamodel(lib=DCxxxx)
<h4> SAS Macros </h4>
@li mf_existfeature.sas
@version 9.2
@author 4GL Apps Ltd
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
and may not be re-distributed or re-sold without the express permission of
4GL Apps Ltd.
**/
%macro mpe_makedatamodel(lib=);
%if &syscc ne 0 %then %do;
%put syscc=&syscc exiting &sysmacroname;
%return;
%end;
%local notnull;
%if %mf_existfeature(COLCONSTRAINTS)=1 %then %let notnull=not null;
%put &=notnull;
proc sql;
create table &lib..mpe_alerts(
tx_from num format=datetime19.3,
alert_event char(20),
alert_lib char(8),
alert_ds char(32),
alert_user char(100) ,
tx_to num &notnull format=datetime19.3
);quit;
proc datasets lib=&lib noprint;
modify mpe_alerts;
index create
pk_mpealerts=(tx_from alert_event alert_lib alert_ds alert_user)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_audit(
load_ref char(36) label='unique load reference',
libref char(8) label='Library Reference (8 chars)',
dsn char(32) label='Dataset Name (32 chars)',
key_hash char(32) label=
'MD5 Hash of primary key values (pipe seperated)',
tgtvar_nm char(32) label='Target variable name (32 chars)',
move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
is_pk num label='Is Primary Key Field? (1/0)',
is_diff num label=
'Did value change? (1/0/-1). Always -1 for appends and deletes.',
tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
oldval_num num format=best32. label='Old (numeric) value',
newval_num num format=best32. label='New (numeric) value',
oldval_char char(32765) label='Old (character) value',
newval_char char(32765) label='New (character) value'
);quit;
proc datasets lib=&lib noprint;
modify mpe_audit;
index create
pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_column_level_security(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
CLS_SCOPE char(4) &notnull,
CLS_GROUP char(64) &notnull,
CLS_LIBREF char(8) &notnull,
CLS_TABLE char(32) &notnull,
CLS_VARIABLE_NM char(32) &notnull,
CLS_ACTIVE num &notnull,
CLS_HIDE num
);quit;
proc datasets lib=&lib noprint;
modify mpe_column_level_security;
index create
pk_mpe_column_level_security=
(tx_to CLS_SCOPE CLS_GROUP CLS_LIBREF CLS_TABLE CLS_VARIABLE_NM)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_config(
tx_from num &notnull format=datetime19.3
,tx_to num &notnull format=datetime19.3
,var_scope varchar(10) &notnull
,var_name varchar(32) &notnull
,var_value varchar(5000)
,var_active num
,var_desc varchar(300)
);quit;
proc datasets lib=&lib noprint;
modify mpe_config;
index create
pk_mpe_config=(tx_to var_scope var_name)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_libs(
TX_FROM num &notnull format=datetime19.3,
TX_TO num &notnull format=datetime19.3,
libref char(8) label='Library Ref',
engine char(32) label='Library Engine',
libname char(256) format=$256. label='Library Name',
paths char(8192) label='Library Paths',
perms char(500) label='Library Permissions (if BASE)',
owners char(500) label='Library Owners (if BASE)',
schemas char(500) label='Library Schemas (if DB)',
libid char(17) label='LibraryId'
);quit;
proc datasets lib=&lib noprint;
modify mpe_datacatalog_libs;
index create
pk_mpe_datacatalog_libs=(libref tx_to)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_TABS(
TX_FROM num &notnull format=datetime19.3,
TX_TO num &notnull format=datetime19.3,
libref char(8) label='Library Name',
dsn char(64) label='Member Name',
memtype char(8) label='Member Type',
dbms_memtype char(32) label='DBMS Member Type',
memlabel char(512) label='Data Set Label',
typemem char(8) label='Data Set Type',
nvar num label='Number of Variables',
compress char(8) label='Compression Routine',
pk_fields char(512)
label='Primary Key Fields (identified by being in a constraint that is both Unique and Not Null)'
);quit;
proc datasets lib=&lib noprint;
modify mpe_datacatalog_TABS;
index create
pk_mpe_datacatalog_TABS=(libref dsn tx_to)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_vars(
TX_FROM num &notnull format=datetime19.3,
TX_TO num &notnull format=datetime19.3,
libref char(8) label='Library Name',
dsn char(64) label='Table Name',
name char(64) label='Column Name',
memtype char(8) label='Member Type',
type char(16) label='Column Type',
length num label='Column Length',
varnum num label='Column Number in Table',
label char(512) label='Column Label',
format char(49) label='Column Format',
idxusage char(9) label='Column Index Type',
notnull char(3) label='Not NULL?',
pk_ind num label='Primary Key Indicator (1=Primary Key field)'
);quit;
proc datasets lib=&lib noprint;
modify mpe_datacatalog_vars;
index create
pk_mpe_datacatalog_vars=(libref dsn name tx_to)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_datastatus_libs(
TX_FROM num &notnull format=datetime19.3,
TX_TO num &notnull format=datetime19.3,
libref char(8) label='Library Name',
libsize num format=SIZEKMG. label='Size of library',
table_cnt num label='Number of Tables'
);quit;
proc datasets lib=&lib noprint;
modify mpe_datastatus_libs;
index create
pk_mpe_datastatus_libs=(libref tx_to)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_datastatus_tabs(
TX_FROM num &notnull format=datetime19.3,
TX_TO num &notnull format=datetime19.3,
libref char(8) label='Library Name',
dsn char(64) label='Member Name',
filesize num format=SIZEKMG. label='Size of file',
crdate num format=DATETIME. informat=DATETIME. label='Date Created',
modate num format=DATETIME. informat=DATETIME. label='Date Modified',
nobs num label='Number of Physical (Actual, inc. deleted) Observations'
);quit;
proc datasets lib=&lib noprint;
modify mpe_datastatus_tabs;
index create
pk_mpe_datastatus_tabs=(libref dsn tx_to)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_datadictionary
(
TX_FROM num &notnull format=datetime19.3,
TX_TO num &notnull format=datetime19.3,
DD_TYPE char(16),
DD_SOURCE char(1024),
DD_SHORTDESC char(256),
DD_LONGDESC char(32767),
DD_OWNER char(128),
DD_RESPONSIBLE char(128),
DD_SENSITIVITY char(64)
);quit;
proc datasets lib=&lib noprint;
modify mpe_datadictionary;
index create
pk_mpe_datadictionary=(tx_to dd_type dd_source)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_dataloads(
libref varchar(8) &notnull,
dsn varchar(32) &notnull,
etlsource varchar(100) &notnull,
loadtype varchar(20) &notnull,
changed_records int,
new_records int,
deleted_records int,
duration num,
user_nm varchar(50) &notnull,
processed_dttm num format=datetime19.3,
mac_ver varchar(5)
);quit;
proc datasets lib=&lib noprint;
modify mpe_dataloads;
index create
pk_mpe_dataloads=(processed_dttm libref dsn etlsource)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_emails(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
user_name char(50) &notnull,
user_displayname char(100),
user_email char(100) &notnull
);quit;
proc datasets lib=&lib noprint;
modify mpe_emails;
index create
pk_mpe_emails=(tx_to user_name)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_excel_config(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
xl_libref char(8),
xl_table char(32),
xl_column char(32),
xl_rule char(32),
xl_active num
);quit;
proc datasets lib=&lib noprint;
modify mpe_excel_config;
index create
pk_mpe_excel_config=(tx_to xl_libref xl_table xl_column)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_excel_map(
tx_from num &notnull,
tx_to num &notnull,
XLMAP_ID char(32) &notnull,
XLMAP_RANGE_ID char(32) &notnull,
XLMAP_SHEET char(32) &notnull,
XLMAP_START char(1000) &notnull,
XLMAP_FINISH char(1000)
);quit;
proc datasets lib=&lib noprint;
modify mpe_excel_map;
index create
pk_mpe_excel_map=(tx_to xlmap_id xlmap_range_id)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_excel_upload(
LOAD_REF char(32) &notnull,
XLMAP_ID char(32) &notnull,
XLMAP_RANGE_ID char(32) &notnull,
ROW_NO num &notnull,
COL_NO num &notnull,
VALUE_TXT char(4000)
);quit;
proc datasets lib=&lib noprint;
modify mpe_excel_upload;
index create
pk_mpe_excel_upload=(load_ref xlmap_id xlmap_range_id row_no col_no)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_filteranytable(
filter_rk num &notnull,
filter_hash char(32) &notnull,
filter_table char(41) &notnull,
processed_dttm num &notnull format=datetime19.
);quit;
proc datasets lib=&lib noprint;
modify mpe_filteranytable;
index create filter_rk /nomiss unique;
quit;
proc sql;
create table &lib..mpe_filtersource(
filter_hash char(32) &notnull,
filter_line num &notnull,
group_logic char(3) &notnull,
subgroup_logic char(3) &notnull,
subgroup_id num &notnull,
variable_nm varchar(32) &notnull,
operator_nm varchar(12) &notnull,
raw_value varchar(4000) &notnull,
processed_dttm num &notnull format=datetime19.
);quit;
proc datasets lib=&lib noprint;
modify mpe_filtersource;
index create
pk_mpe_filtersource=(filter_hash filter_line)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_groups(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
group_name char(100) &notnull,
user_name char(50) &notnull,
group_desc char(256)
);quit;
proc datasets lib=&lib noprint;
modify mpe_groups;
index create
pk_mpe_groups=(tx_to group_name user_name)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_lineage_cols
(
col_id char(32),
direction char(1),
sourcecoluri char(256),
map_type char(256),
map_transform char(256),
jobname char(256),
sourcetablename char(256),
sourcecolname char(256),
targettablename char(256),
targetcolname char(256),
targetcoluri char(256),
Derived_Rule char(500),
level int,
modified_dttm num format=datetime19.3,
modified_by char(64)
);quit;
proc datasets lib=&lib noprint;
modify mpe_lineage_cols;
index create
pk_mpe_lineage_cols=(col_id direction sourcecoluri targetcoluri map_type map_transform)
/nomiss unique;
quit;
proc sql;
create table &lib..MPE_LINEAGE_TABS
(
tx_from num &notnull format=datetime19.3,
jobid char(17),
srctableid char(17),
tgttableid char(17),
jobname char(128),
srctabletype char(16),
srctablename char(64),
srclibref char(8),
tgttabletype char(16),
tgttablename char(64),
tgtlibref char(8),
tx_to num &notnull format=datetime19.3
);quit;
proc datasets lib=&lib noprint;
modify mpe_lineage_tabs;
index create
pk_mpe_lineage_tabs=(tx_to jobid srctableid tgttableid)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_loads(
csv_dir char(255),
user_nm char(50) ,
status char(15) ,
duration num ,
processed_dttm num format=datetime19.3,
reason_txt char(2048) ,
approvals char(64)
);quit;
proc datasets lib=&lib noprint;
modify mpe_loads;
index create csv_dir /nomiss unique;
quit;
proc sql;
create table &lib..mpe_lockanytable(
lock_lib varchar(8) &notnull ,
lock_ds varchar(32) &notnull,
lock_status_cd varchar(10) &notnull,
lock_user_nm varchar(100) &notnull ,
lock_ref varchar(200),
lock_pid varchar(10),
lock_start_dttm num format=E8601DT26.6,
lock_end_dttm num format=E8601DT26.6
);quit;
proc datasets lib=&lib noprint;
modify mpe_lockanytable;
index create
pk_mpe_lockanytable=(lock_lib lock_ds)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_maxkeyvalues(
keytable varchar(41) label='Base table in libref.dataset format',
keycolumn char(32) format=$32.
label='The Surrogate / Retained key field containing the key values.',
max_key num label=
'Integer value representing current max RK or SK value in the KEYTABLE',
processed_dttm num format=E8601DT26.6
label='Datetime this value was last updated'
);quit;
proc datasets lib=&lib noprint;
modify mpe_maxkeyvalues;
index create keytable /nomiss unique;
quit;
/* no PK defined as it is a transaction table */
proc sql;
create table &lib..mpe_requests(
request_dttm num &notnull format=datetime19.,
request_user char(64) &notnull,
request_service char(64) &notnull,
request_params char(128)
);
proc sql;
create table &lib..mpe_review(
table_id varchar(32) &notnull,
reviewed_by_nm varchar(100) &notnull,
base_table varchar(41) &notnull,
review_status_id varchar(10) &notnull,
reviewed_on_dttm num &notnull format=datetime19.3,
review_reason_txt varchar(400)
);quit;
proc datasets lib=&lib noprint;
modify mpe_review;
index create
pk_mpe_review=(table_id reviewed_by_nm)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_row_level_security(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
RLS_RK num &notnull,
RLS_SCOPE char(8) &notnull,
RLS_GROUP char(128) &notnull,
RLS_LIBREF char(8) &notnull,
RLS_TABLE char(32) &notnull,
RLS_GROUP_LOGIC char(3) &notnull,
RLS_SUBGROUP_LOGIC char(3) &notnull,
RLS_SUBGROUP_ID num &notnull,
RLS_VARIABLE_NM varchar(32) &notnull,
RLS_OPERATOR_NM varchar(12) &notnull,
RLS_RAW_VALUE varchar(4000) &notnull,
RLS_ACTIVE num &notnull
);quit;
proc datasets lib=&lib noprint;
modify mpe_row_level_security;
index create
pk_mpe_row_level_security=(tx_to RLS_RK)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_security(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
libref char(8) &notnull,
dsn char(32) &notnull,
access_level char(10) &notnull,
sas_group char(100) &notnull
);quit;
proc datasets lib=&lib noprint;
modify mpe_security;
index create
pk_mpe_security=(tx_to libref dsn access_level sas_group)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_selectbox(
ver_from_dttm num &notnull format=datetime19.3,/* timestamp for versioning*/
ver_to_dttm num &notnull format=datetime19.3, /* timestamp for versioning */
selectbox_rk num &notnull, /* surrogate key */
select_lib varchar(17) &notnull, /* libref (big enough for uri)*/
select_ds varchar(32) &notnull,
base_column varchar(36) &notnull, /* variable name against which to apply selectbox */
selectbox_value varchar(500) &notnull, /* selectbox value */
selectbox_order num , /* optional ordering (1 comes before 2) */
selectbox_type varchar(32) /* column type (blank for default, else
sas or js to indicate relevant system functions)*/
);quit;
proc datasets lib=&lib noprint;
modify mpe_selectbox;
index create
pk_mpe_selectbox=(ver_to_dttm selectbox_rk)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_signoffs(
tech_from_dttm num &notnull format=datetime19.3,
tech_to_dttm num &notnull format=datetime19.3,
signoff_table varchar(50) &notnull,
signoff_section_rk num &notnull,
signoff_version_rk num &notnull,
signoff_name varchar(100) &notnull
);quit;
proc datasets lib=&lib noprint;
modify mpe_signoffs;
index create
pk_mpe_signoffs=(tech_to_dttm signoff_table signoff_section_rk)
/nomiss unique;
quit;
/* mpe_submit */
proc sql;
create table &lib..mpe_submit(
table_id varchar(32) &notnull,
submit_status_cd varchar(10) &notnull,
base_lib char(8) &notnull,
base_ds char(32) &notnull,
submitted_by_nm varchar(100) &notnull,
submitted_on_dttm num &notnull format=datetime19.3,
submitted_reason_txt varchar(400),
input_obs num,
input_vars num,
num_of_approvals_required num &notnull ,
num_of_approvals_remaining num &notnull ,
reviewed_by_nm char(100),
reviewed_on_dttm num
);quit;
proc datasets lib=&lib noprint;
modify mpe_submit;
index create table_id /nomiss unique;
quit;
proc sql;
create table &lib..mpe_tables(
tx_from num &notnull format=datetime19.3,
tx_to num &notnull format=datetime19.3,
libref char(8) &notnull,
dsn char(32) &notnull,
num_of_approvals_required int,
loadtype char(12) ,
buskey char(1000) ,
var_txfrom char(32) ,
var_txto char(32) ,
var_busfrom char(32) ,
var_busto char(32) ,
var_processed char(32) ,
close_vars varchar(500),
pre_edit_hook char(200),
post_edit_hook char(200),
pre_approve_hook char(200) ,
post_approve_hook char(200) ,
signoff_cols varchar(500),
signoff_hook varchar(200),
notes char(1000) ,
rk_underlying char(1000) ,
audit_libds char(41)
);quit;
proc datasets lib=&lib noprint;
modify mpe_tables;
index create
pk_mpe_tables=(tx_to libref dsn)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_users(
user_id char(50) &notnull,
last_seen_dt num &notnull format=date9.,
registered_dt num &notnull format=date9.
);quit;
proc datasets lib=&lib noprint;
modify mpe_users;
index create user_id /nomiss unique;
quit;
proc sql;
create table &lib..MPE_VALIDATIONS
(
TX_FROM num &notnull format=datetime19.3,
BASE_LIB varchar(8),
BASE_DS varchar(32),
BASE_COL varchar(32),
RULE_TYPE varchar(32),
RULE_VALUE varchar(128),
RULE_ACTIVE num ,
TX_TO num &notnull format=datetime19.3
);quit;
proc datasets lib=&lib noprint;
modify mpe_validations;
index create
pk_mpe_validations=(tx_from base_lib base_ds base_col rule_type)
/nomiss unique;
quit;
proc sql;
create table &lib..mpe_x_test(
primary_key_field num &notnull,
some_char char(32767) ,
some_dropdown char(128),
some_num num ,
some_date num format=date9.,
some_datetime num format=datetime19. informat=ANYDTDTM19.,
some_time num format=time8.,
some_shortnum num length=4,
some_bestnum num format=best.
);quit;
proc datasets lib=&lib noprint;
modify mpe_x_test;
index create primary_key_field /nomiss unique;
quit;
%mend mpe_makedatamodel;