/**
  @file
  @brief Creates the empty Data Controller tables
  @details Creates the empty DC tables during the initial deployment.

  Usage:

      %mpe_makedatamodel(lib=DCxxxx)

  <h4> SAS Macros </h4>
  @li mf_existfeature.sas

  @version 9.2
  @author 4GL Apps Ltd
  @copyright 4GL Apps Ltd.  This code may only be used within Data Controller
    and may not be re-distributed or re-sold without the express permission of
    4GL Apps Ltd.
**/

%macro mpe_makedatamodel(lib=);
%if &syscc ne 0 %then %do;
  %put syscc=&syscc exiting &sysmacroname;
  %return;
%end;

%local notnull;
%if %mf_existfeature(COLCONSTRAINTS)=1 %then %let notnull=not null;
%put &=notnull;

proc sql;
create table &lib..mpe_alerts(
    tx_from num format=datetime19.3,
    alert_event char(20),
    alert_lib char(8),
    alert_ds char(32),
    alert_user char(100) ,
    tx_to num &notnull format=datetime19.3
);quit;
proc datasets lib=&lib noprint;
  modify mpe_alerts;
  index create
    pk_mpealerts=(tx_from alert_event alert_lib alert_ds alert_user)
    /nomiss unique;
quit;

proc sql;
create table &lib..mpe_audit(
  load_ref char(36) label='unique load reference',
  libref char(8) label='Library Reference (8 chars)',
  dsn char(32) label='Dataset Name (32 chars)',
  key_hash char(32) label=
    'MD5 Hash of primary key values (pipe seperated)',
  tgtvar_nm char(32) label='Target variable name (32 chars)',
  move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified',
  processed_dttm num format=E8601DT26.6 label='Processed at timestamp',
  is_pk num label='Is Primary Key Field? (1/0)',
  is_diff num label=
    'Did value change? (1/0/-1).  Always -1 for appends and deletes.',
  tgtvar_type char(1) label='Either (C)haracter or (N)umeric',
  oldval_num num format=best32. label='Old (numeric) value',
  newval_num num format=best32. label='New (numeric) value',
  oldval_char char(32765) label='Old (character) value',
  newval_char char(32765) label='New (character) value'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_audit;
  index create
    pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
    /nomiss unique;
quit;

proc sql;
create table &lib..mpe_column_level_security(
    tx_from num &notnull format=datetime19.3,
    tx_to num &notnull format=datetime19.3,
    CLS_SCOPE char(4) &notnull,
    CLS_GROUP char(64) &notnull,
    CLS_LIBREF char(8) &notnull,
    CLS_TABLE char(32) &notnull,
    CLS_VARIABLE_NM char(32) &notnull,
    CLS_ACTIVE num &notnull,
    CLS_HIDE num
);quit;
proc datasets lib=&lib noprint;
  modify mpe_column_level_security;
  index create
    pk_mpe_column_level_security=
      (tx_to CLS_SCOPE CLS_GROUP CLS_LIBREF CLS_TABLE CLS_VARIABLE_NM)
    /nomiss unique;
quit;

proc sql;
create table &lib..mpe_config(
        tx_from num &notnull format=datetime19.3
        ,tx_to num &notnull format=datetime19.3
        ,var_scope varchar(10) &notnull
        ,var_name varchar(32) &notnull
        ,var_value varchar(5000)
        ,var_active num
        ,var_desc varchar(300)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_config;
  index create
    pk_mpe_config=(tx_to var_scope var_name)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_CATS(
    TX_FROM float &notnull format=datetime19.,
    TX_TO float format=datetime19.,
    libref char(8) label='Library Name',
    memname char(64) label='Member Name'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datacatalog_CATS;
  index create
    pk_mpe_datacatalog_CATS=(tx_to libref memname)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_libs(
    TX_FROM num &notnull format=datetime19.3,
    TX_TO num &notnull format=datetime19.3,
    libref char(8) label='Library Ref',
    engine char(32) label='Library Engine',
    libname char(256) format=$256. label='Library Name',
    paths char(8192) label='Library Paths',
    perms char(500) label='Library Permissions (if BASE)',
    owners char(500) label='Library Owners (if BASE)',
    schemas char(500) label='Library Schemas (if DB)',
    libid char(17) label='LibraryId'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datacatalog_libs;
  index create
    pk_mpe_datacatalog_libs=(libref tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_OBJS(
    TX_FROM num &notnull format=datetime19.,
    TX_TO num &notnull format=datetime19.,
    libref char(8) &notnull label='Library Name',
    memname char(64) &notnull label='Member Name',
    objname char(32) &notnull label='Object Name',
    objtype char(8) &notnull label='Object Type',
    objdesc char(256) label='Object Description',
    alias char(32) label='Object Alias'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datacatalog_OBJS;
  index create
    pk_mpe_datacatalog_OBJS=(libref memname objname objtype tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_TABS(
    TX_FROM num &notnull format=datetime19.3,
    TX_TO num &notnull format=datetime19.3,
    libref char(8) label='Library Name',
    dsn char(64) label='Member Name',
    memtype char(8) label='Member Type',
    dbms_memtype char(32) label='DBMS Member Type',
    memlabel char(512) label='Data Set Label',
    typemem char(8) label='Data Set Type',
    nvar num label='Number of Variables',
    compress char(8) label='Compression Routine',
    pk_fields char(512)
    label='Primary Key Fields - in a constraint being both Unique and Not Null'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datacatalog_TABS;
  index create
    pk_mpe_datacatalog_TABS=(libref dsn tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datacatalog_vars(
    TX_FROM num &notnull format=datetime19.3,
    TX_TO num &notnull format=datetime19.3,
    libref char(8) label='Library Name',
    dsn char(64) label='Table Name',
    name char(64) label='Column Name',
    memtype char(8) label='Member Type',
    type char(16) label='Column Type',
    length num label='Column Length',
    varnum num label='Column Number in Table',
    label char(512) label='Column Label',
    format char(49) label='Column Format',
    idxusage char(9) label='Column Index Type',
    notnull char(3) label='Not NULL?',
    pk_ind num label='Primary Key Indicator (1=Primary Key field)'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datacatalog_vars;
  index create
    pk_mpe_datacatalog_vars=(libref dsn name tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datastatus_CATS(
    TX_FROM float format=datetime19.,
    TX_TO float format=datetime19.,
    libref char(8) label='Library Name',
    memname char(64) label='Member Name',
    nobjs num &notnull label='Number of objects',
    created num &notnull format=DATETIME.  label='Date Created',
    modified num format=DATETIME. label='Date Modified'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datastatus_CATS;
  index create
    pk_mpe_datastatus_cats=(libref memname tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datastatus_libs(
    TX_FROM num &notnull format=datetime19.3,
    TX_TO num &notnull format=datetime19.3,
    libref char(8) label='Library Name',
    libsize num format=SIZEKMG. label='Size of library',
    table_cnt num label='Number of Tables',
    catalog_cnt num label='Number of Catalogs'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datastatus_libs;
  index create
    pk_mpe_datastatus_libs=(libref tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datastatus_OBJS(
    TX_FROM float &notnull format=datetime19.,
    TX_TO float &notnull format=datetime19.,
    libref char(8) label='Library Name',
    memname char(64) label='Member Name',
    objname char(32) label='Object Name',
    objtype char(8) label='Object Type',
    created num &notnull format=DATETIME. label='Date Created',
    modified num format=DATETIME. label='Date Modified',
    level num label='Library Concatenation Level'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datastatus_OBJS;
  index create
    pk_mpe_datastatus_OBJS=(libref memname objname objtype tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datastatus_tabs(
    TX_FROM num &notnull format=datetime19.3,
    TX_TO num &notnull format=datetime19.3,
    libref char(8) label='Library Name',
    dsn char(64) label='Member Name',
    filesize num format=SIZEKMG. label='Size of file',
    crdate num format=DATETIME. label='Date Created',
    modate num format=DATETIME. label='Date Modified',
    nobs num label='Number of Physical (Actual, inc. deleted) Observations'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datastatus_tabs;
  index create
    pk_mpe_datastatus_tabs=(libref dsn tx_to)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_datadictionary
  (
    TX_FROM num &notnull format=datetime19.3,
    TX_TO num &notnull format=datetime19.3,
    DD_TYPE char(16),
    DD_SOURCE char(1024),
    DD_SHORTDESC char(256),
    DD_LONGDESC char(32767),
    DD_OWNER char(128),
    DD_RESPONSIBLE char(128),
    DD_SENSITIVITY char(64)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_datadictionary;
  index create
    pk_mpe_datadictionary=(tx_to dd_type dd_source)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_dataloads(
          libref varchar(8) &notnull,
          dsn varchar(32) &notnull,
          etlsource varchar(100) &notnull,
          loadtype varchar(20) &notnull,
          changed_records int,
          new_records int,
          deleted_records int,
          duration num,
          user_nm varchar(50) &notnull,
          processed_dttm num format=datetime19.3,
          mac_ver varchar(5)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_dataloads;
  index create
    pk_mpe_dataloads=(processed_dttm libref dsn etlsource)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_emails(
        tx_from num &notnull format=datetime19.3,
        tx_to num &notnull format=datetime19.3,
        user_name char(50) &notnull,
        user_displayname char(100),
        user_email char(100) &notnull
);quit;
proc datasets lib=&lib noprint;
  modify mpe_emails;
  index create
    pk_mpe_emails=(tx_to user_name)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_excel_config(
        tx_from num &notnull format=datetime19.3,
        tx_to num &notnull format=datetime19.3,
        xl_libref char(8),
        xl_table char(32),
        xl_column char(32),
        xl_rule char(32),
        xl_active num
);quit;
proc datasets lib=&lib noprint;
  modify mpe_excel_config;
  index create
    pk_mpe_excel_config=(tx_to xl_libref xl_table xl_column)
    /nomiss unique;
quit;

proc sql;
create table &lib..MPE_XLMAP_DATA(
        LOAD_REF char(32) &notnull,
        XLMAP_ID char(32) &notnull,
        XLMAP_RANGE_ID char(32) &notnull,
        ROW_NO num &notnull,
        COL_NO num &notnull,
        VALUE_TXT char(4000)
);quit;
proc datasets lib=&lib noprint;
  modify MPE_XLMAP_DATA;
  index create
    pk_MPE_XLMAP_DATA=(load_ref xlmap_id xlmap_range_id row_no col_no)
    /nomiss unique;
quit;

proc sql;
create table &lib..mpe_xlmap_info(
        tx_from num &notnull,
        tx_to num &notnull,
        XLMAP_ID char(32) &notnull,
        XLMAP_DESCRIPTION char(1000) &notnull,
        XLMAP_TARGETLIBDS char(41) &notnull
);quit;
proc datasets lib=&lib noprint;
  modify mpe_xlmap_info;
  index create
    pk_mpe_xlmap_info=(tx_to xlmap_id)
    /nomiss unique;
quit;

proc sql;
create table &lib..mpe_xlmap_rules(
        tx_from num &notnull,
        tx_to num &notnull,
        XLMAP_ID char(32) &notnull,
        XLMAP_RANGE_ID char(32) &notnull,
        XLMAP_SHEET char(32) &notnull,
        XLMAP_START char(1000) &notnull,
        XLMAP_FINISH char(1000)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_xlmap_rules;
  index create
    pk_mpe_xlmap_rules=(tx_to xlmap_id xlmap_range_id)
    /nomiss unique;
quit;

proc sql;
create table &lib..mpe_filteranytable(
        filter_rk num &notnull,
        filter_hash char(32) &notnull,
        filter_table char(41) &notnull,
        processed_dttm num &notnull format=datetime19.
);quit;
proc datasets lib=&lib noprint;
  modify mpe_filteranytable;
  index create filter_rk /nomiss unique;
quit;
proc sql;
create table &lib..mpe_filtersource(
        filter_hash char(32) &notnull,
        filter_line num &notnull,
        group_logic char(3) &notnull,
        subgroup_logic char(3) &notnull,
        subgroup_id num &notnull,
        variable_nm varchar(32) &notnull,
        operator_nm varchar(12) &notnull,
        raw_value varchar(4000) &notnull,
        processed_dttm num &notnull format=datetime19.
);quit;
proc datasets lib=&lib noprint;
  modify mpe_filtersource;
  index create
    pk_mpe_filtersource=(filter_hash filter_line)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_groups(
      tx_from num &notnull format=datetime19.3,
      tx_to num &notnull format=datetime19.3,
      group_name char(100) &notnull,
      user_name char(50) &notnull,
      group_desc char(256)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_groups;
  index create
    pk_mpe_groups=(tx_to group_name user_name)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_lineage_cols
    (
        col_id char(32),
        direction char(1),
        sourcecoluri char(256),
        map_type char(256),
        map_transform char(256),
        jobname char(256),
        sourcetablename char(256),
        sourcecolname char(256),
        targettablename char(256),
        targetcolname char(256),
        targetcoluri char(256),
        Derived_Rule char(500),
        level int,
        modified_dttm num format=datetime19.3,
        modified_by char(64)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_lineage_cols;
  index create
    pk_mpe_lineage_cols=(col_id direction sourcecoluri targetcoluri map_type map_transform)
    /nomiss unique;
quit;
proc sql;
create table &lib..MPE_LINEAGE_TABS
    (
      tx_from num &notnull format=datetime19.3,
      jobid char(17),
      srctableid char(17),
      tgttableid char(17),
      jobname char(128),
      srctabletype char(16),
      srctablename char(64),
      srclibref char(8),
      tgttabletype char(16),
      tgttablename char(64),
      tgtlibref char(8),
      tx_to num &notnull format=datetime19.3
);quit;
proc datasets lib=&lib noprint;
  modify mpe_lineage_tabs;
  index create
    pk_mpe_lineage_tabs=(tx_to jobid srctableid tgttableid)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_loads(
          csv_dir char(255),
          user_nm char(50) ,
          status char(15) ,
          duration num ,
          processed_dttm num format=datetime19.3,
          reason_txt char(2048) ,
          approvals char(64)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_loads;
  index create csv_dir /nomiss unique;
quit;
proc sql;
create table &lib..mpe_lockanytable(
          lock_lib varchar(8) &notnull ,
          lock_ds varchar(32)  &notnull,
          lock_status_cd varchar(10) &notnull,
          lock_user_nm varchar(100) &notnull ,
          lock_ref varchar(200),
          lock_pid varchar(10),
          lock_start_dttm num format=E8601DT26.6,
          lock_end_dttm num format=E8601DT26.6
);quit;
proc datasets lib=&lib noprint;
  modify mpe_lockanytable;
  index create
    pk_mpe_lockanytable=(lock_lib lock_ds)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_maxkeyvalues(
      keytable varchar(41) label='Base table in libref.dataset format',
      keycolumn char(32) format=$32.
        label='The Surrogate / Retained key field containing the key values.',
      max_key num label=
        'Integer value representing current max RK or SK value in the KEYTABLE',
      processed_dttm num format=E8601DT26.6
        label='Datetime this value was last updated'
);quit;
proc datasets lib=&lib noprint;
  modify mpe_maxkeyvalues;
  index create keytable /nomiss unique;
quit;
/* no PK defined as it is a transaction table */
proc sql;
create table &lib..mpe_requests(
    request_dttm num &notnull  format=datetime19.,
    request_user char(64) &notnull,
    request_service char(64) &notnull,
    request_params char(128)
  );
proc sql;
create table &lib..mpe_review(
          table_id varchar(32) &notnull,
          reviewed_by_nm varchar(100) &notnull,
          base_table varchar(41) &notnull,
          review_status_id varchar(10) &notnull,
          reviewed_on_dttm num &notnull format=datetime19.3,
          review_reason_txt varchar(400)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_review;
  index create
    pk_mpe_review=(table_id reviewed_by_nm)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_row_level_security(
    tx_from num &notnull format=datetime19.3,
    tx_to num &notnull format=datetime19.3,
    RLS_RK num &notnull,
    RLS_SCOPE char(8) &notnull,
    RLS_GROUP char(128) &notnull,
    RLS_LIBREF char(8) &notnull,
    RLS_TABLE char(32) &notnull,
    RLS_GROUP_LOGIC char(3) &notnull,
    RLS_SUBGROUP_LOGIC char(3) &notnull,
    RLS_SUBGROUP_ID num &notnull,
    RLS_VARIABLE_NM varchar(32) &notnull,
    RLS_OPERATOR_NM varchar(12) &notnull,
    RLS_RAW_VALUE varchar(4000) &notnull,
    RLS_ACTIVE num &notnull
);quit;
proc datasets lib=&lib noprint;
  modify mpe_row_level_security;
  index create
    pk_mpe_row_level_security=(tx_to RLS_RK)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_security(
        tx_from num &notnull format=datetime19.3,
        tx_to num &notnull format=datetime19.3,
        libref char(8) &notnull,
        dsn char(32) &notnull,
        access_level char(10) &notnull,
        sas_group char(100) &notnull
);quit;
proc datasets lib=&lib noprint;
  modify mpe_security;
  index create
    pk_mpe_security=(tx_to libref dsn access_level sas_group)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_selectbox(
    ver_from_dttm num &notnull format=datetime19.3,/* timestamp for versioning*/
    ver_to_dttm num &notnull format=datetime19.3, /* timestamp for versioning */
    selectbox_rk num &notnull, /* surrogate key */
    select_lib varchar(17) &notnull, /* libref (big enough for uri)*/
    select_ds varchar(32) &notnull,
    base_column varchar(36) &notnull, /* variable name against which to apply selectbox */
    selectbox_value varchar(500) &notnull, /* selectbox value */
    selectbox_order num , /* optional ordering (1 comes before 2) */
    selectbox_type varchar(32) /* column type (blank for default, else
                            sas or js to indicate relevant system functions)*/
);quit;
proc datasets lib=&lib noprint;
  modify mpe_selectbox;
  index create
    pk_mpe_selectbox=(ver_to_dttm selectbox_rk)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_signoffs(
          tech_from_dttm num &notnull format=datetime19.3,
          tech_to_dttm num &notnull format=datetime19.3,
          signoff_table varchar(50) &notnull,
          signoff_section_rk num &notnull,
          signoff_version_rk num &notnull,
          signoff_name varchar(100) &notnull
);quit;
proc datasets lib=&lib noprint;
  modify mpe_signoffs;
  index create
    pk_mpe_signoffs=(tech_to_dttm signoff_table signoff_section_rk)
    /nomiss unique;
quit;
/* mpe_submit */
proc sql;
create table &lib..mpe_submit(
      table_id varchar(32) &notnull,
      submit_status_cd varchar(10) &notnull,
      base_lib char(8) &notnull,
      base_ds char(32) &notnull,
      submitted_by_nm varchar(100) &notnull,
      submitted_on_dttm num &notnull format=datetime19.3,
      submitted_reason_txt varchar(400),
      input_obs num,
      input_vars num,
      num_of_approvals_required num &notnull ,
      num_of_approvals_remaining num &notnull ,
      reviewed_by_nm char(100),
      reviewed_on_dttm num
);quit;
proc datasets lib=&lib noprint;
  modify mpe_submit;
  index create table_id /nomiss unique;
quit;
proc sql;
create table &lib..mpe_tables(
        tx_from num &notnull format=datetime19.3,
        tx_to num &notnull format=datetime19.3,
        libref char(8) &notnull,
        dsn char(32) &notnull,
        num_of_approvals_required int,
        loadtype char(12) ,
        buskey char(1000) ,
        var_txfrom char(32) ,
        var_txto char(32) ,
        var_busfrom char(32) ,
        var_busto char(32) ,
        var_processed char(32) ,
        close_vars varchar(500),
        pre_edit_hook char(200),
        post_edit_hook char(200),
        pre_approve_hook char(200) ,
        post_approve_hook char(200) ,
        signoff_cols varchar(500),
        signoff_hook varchar(200),
        notes char(1000) ,
        rk_underlying char(1000) ,
        audit_libds char(41)
);quit;
proc datasets lib=&lib noprint;
  modify mpe_tables;
  index create
    pk_mpe_tables=(tx_to libref dsn)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_users(
        user_id char(50) &notnull,
        last_seen_dt num &notnull format=date9.,
        registered_dt num &notnull format=date9.
);quit;
proc datasets lib=&lib noprint;
  modify mpe_users;
  index create user_id /nomiss unique;
quit;
proc sql;
create table &lib..MPE_VALIDATIONS
  (
    TX_FROM num &notnull format=datetime19.3,
    BASE_LIB varchar(8),
    BASE_DS varchar(32),
    BASE_COL varchar(32),
    RULE_TYPE varchar(32),
    RULE_VALUE varchar(128),
    RULE_ACTIVE num ,
    TX_TO num &notnull format=datetime19.3
);quit;
proc datasets lib=&lib noprint;
  modify mpe_validations;
  index create
    pk_mpe_validations=(tx_from base_lib base_ds base_col rule_type)
    /nomiss unique;
quit;
proc sql;
create table &lib..mpe_x_test(
          primary_key_field num &notnull,
          some_char char(32767) ,
          some_dropdown char(128),
          some_num num ,
          some_date num  format=date9.,
          some_datetime num  format=datetime19. informat=ANYDTDTM19.,
          some_time num format=time8.,
          some_shortnum num length=4,
          some_bestnum num format=best.
);quit;
proc datasets lib=&lib noprint;
  modify mpe_x_test;
  index create primary_key_field /nomiss unique;
quit;

%mend mpe_makedatamodel;
