feat: new hash algorithm - allows 1000+ cols. Closes #269
This commit is contained in:
1 parent
efcccd7d11
commit
08947f0fc8
2 files changed
+149
-24
No files matched your search
@@ -111,6 +111,7 @@ Areas for optimisation
|
||||
@li mp_lockfilecheck.sas
|
||||
@li mp_retainedkey.sas
|
||||
@li mp_storediffs.sas
|
||||
@li mpe_rowhash.sas
|
||||
|
||||
@version 9.3
|
||||
@author 4GL Apps Ltd.
|
||||
@@ -193,7 +194,7 @@ Areas for optimisation
|
||||
|
||||
/* when changing this macro, update the version num here */
|
||||
%local ver;
|
||||
%let ver=32;
|
||||
%let ver=33;
|
||||
%put &sysmacroname entry vars:;
|
||||
%put _local_;
|
||||
|
||||
@@ -246,7 +247,7 @@ run;
|
||||
)
|
||||
|
||||
|
||||
%local cols idx_pk md5_col ;
|
||||
%local cols idx_pk md5_col hash_char_vars hash_num_vars ;
|
||||
%let md5_col=___TMP___md5;
|
||||
%let check_uniqueness=%upcase(&check_uniqueness);
|
||||
%let RK_UPDATE_MAXKEYTABLE=%upcase(&RK_UPDATE_MAXKEYTABLE);
|
||||
@@ -321,14 +322,20 @@ select name into: cols separated by ','
|
||||
(%upcase("&bus_from","&bus_to"
|
||||
,"&tech_from","&tech_to"
|
||||
,"&processed","&delete_col")) ;
|
||||
select case when type in (2,6) then cats('put(md5(trim(',name,')),$hex32.)')
|
||||
/* multiply by 1 to strip precision errors (eg 0 != 0) */
|
||||
/* but ONLY if not missing, else will lose any special missing values */
|
||||
else cats('put(md5(trim(put(ifn(missing('
|
||||
,name,'),',name,',',name,'*1),binary64.))),$hex32.)') end
|
||||
into: stripcols separated by '||'
|
||||
/* Character variables are hashed separately from numerics so that the
|
||||
iterative hash can be built with arrays rather than by concatenating
|
||||
hundreds of 32-byte hex strings. */
|
||||
select name into: hash_char_vars separated by ' '
|
||||
from work.bitemp_cols
|
||||
where upcase(name) not in
|
||||
where type in (2,6)
|
||||
and upcase(name) not in
|
||||
(%upcase("&bus_from","&bus_to"
|
||||
,"&tech_from","&tech_to"
|
||||
,"&processed","&delete_col")) ;
|
||||
select name into: hash_num_vars separated by ' '
|
||||
from work.bitemp_cols
|
||||
where type not in (2,6)
|
||||
and upcase(name) not in
|
||||
(%upcase("&bus_from","&bus_to"
|
||||
,"&tech_from","&tech_to"
|
||||
,"&processed","&delete_col")) ;
|
||||
@@ -342,7 +349,8 @@ select case when type in (2,6) then cats('put(md5(trim(',name,')),$hex32.)')
|
||||
%let tech_to_fmt = format=datetime19.;
|
||||
|
||||
|
||||
%put &=stripcols;
|
||||
%put &=hash_char_vars;
|
||||
%put &=hash_num_vars;
|
||||
%put &=pk;
|
||||
|
||||
data _null_;
|
||||
@@ -368,8 +376,8 @@ data _null_;
|
||||
call symputx('processed_fmt',fmt,'L');
|
||||
run;
|
||||
|
||||
%if %index(%quote(&cols),___TMP___) %then %do;
|
||||
%let msg=%str(Table contains a variable name containing "___TMP___".%trim(
|
||||
%if %index(%quote(&cols),___TMP___) or %index(%quote(&cols),_____) %then %do;
|
||||
%let msg=%str(Table contains a variable name containing "___TMP___" or "_____".%trim(
|
||||
) This may conflict with temp variable generation!!);
|
||||
%mp_abort(msg=&msg,mac=bitemporal_dataloader);
|
||||
%let syscc=5;
|
||||
@@ -485,7 +493,10 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
|
||||
&bus_to= %unquote(&bus_to_override) ;
|
||||
%end;
|
||||
length &md5_col $32;
|
||||
&md5_col=put(md5(&stripcols),hex32.);
|
||||
%mpe_rowhash(md5_col=&md5_col
|
||||
,char_vars=&hash_char_vars
|
||||
,num_vars=&hash_num_vars
|
||||
);
|
||||
%if %length(&processed)>0 %then %do;
|
||||
format &processed &processed_fmt;
|
||||
&processed=&now;
|
||||
@@ -772,13 +783,17 @@ or &engine_type=SNOW or &engine_type=SASIOSNF
|
||||
,msg=%str(syscc=&syscc sqlrc=&sqlrc)
|
||||
)
|
||||
|
||||
%put hashcols2=&stripcols;
|
||||
proc sql;
|
||||
create table work.bitemp1_current(drop=___TMP___NEW_FLG) as
|
||||
select *
|
||||
, put(md5(&stripcols),$hex32.) as &md5_col
|
||||
from work.bitemp0_base (drop=&md5_col)
|
||||
where ___TMP___NEW_FLG=0;
|
||||
%put hash_char_vars=&hash_char_vars;
|
||||
%put hash_num_vars=&hash_num_vars;
|
||||
data work.bitemp1_current(drop=___TMP___NEW_FLG);
|
||||
set work.bitemp0_base(drop=&md5_col);
|
||||
if ___TMP___NEW_FLG=0;
|
||||
length &md5_col $32;
|
||||
%mpe_rowhash(md5_col=&md5_col
|
||||
,char_vars=&hash_char_vars
|
||||
,num_vars=&hash_num_vars
|
||||
);
|
||||
run;
|
||||
|
||||
/**
|
||||
* NEW records were identified in ___TMP___NEW_FLG in bitemp0_base
|
||||
@@ -1061,7 +1076,9 @@ run;
|
||||
*/
|
||||
|
||||
%if &loadtype=BITEMPORAL %then %do;
|
||||
%let cat_string=catx('|' ,&bus_from,&bus_to);
|
||||
/* For bitemporal we also include the business dates in the comparison hash.
|
||||
The business dates are passed as prefix numerics so that both lookup and
|
||||
update tables use exactly the same hashing order. */
|
||||
|
||||
data work.bitemp5a_lkp (keep=&md5_col)
|
||||
%if "%substr(&sysver,1,1)" ne "4" & "%substr(&sysver,1,1)" ne "5" %then %do;
|
||||
@@ -1069,8 +1086,11 @@ run;
|
||||
%end;
|
||||
;
|
||||
set work.bitemp0_base;
|
||||
/* for BITEMPORAL we need to compare business dates also */
|
||||
&md5_col=put(md5(&cat_string!!'|'!!&stripcols),$hex32.);
|
||||
%mpe_rowhash(md5_col=&md5_col
|
||||
,char_vars=&hash_char_vars
|
||||
,num_vars=&hash_num_vars
|
||||
,prefix_num_vars=&bus_from &bus_to
|
||||
);
|
||||
run;
|
||||
|
||||
data bitemp5b_updates;
|
||||
@@ -1081,7 +1101,11 @@ run;
|
||||
md5_lkp.definedone();
|
||||
end;
|
||||
/* drop old md5 col as will rebuild with new business dates */
|
||||
&md5_col=put(md5(&cat_string!!'|'!!&stripcols),$hex32.) ;
|
||||
%mpe_rowhash(md5_col=&md5_col
|
||||
,char_vars=&hash_char_vars
|
||||
,num_vars=&hash_num_vars
|
||||
,prefix_num_vars=&bus_from &bus_to
|
||||
);
|
||||
if md5_lkp.check()=0 then delete;
|
||||
run;
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
@file
|
||||
@brief Iterative row-level MD5 hash generator
|
||||
@details Generates DATA step statements that compute a deterministic,
|
||||
row-level MD5 hash using a Merkle-style construction. Each variable is
|
||||
hashed independently and the resulting raw 16-byte digests are combined
|
||||
iteratively. This avoids concatenating the per-variable hex digests into a
|
||||
single SAS character expression, which overflows when datasets contain
|
||||
many variables (>1000).
|
||||
|
||||
All temporary variables use five leading and five trailing underscores
|
||||
to comply with the internal naming standard for system variables.
|
||||
|
||||
This macro is called from inside a DATA step. The caller is responsible
|
||||
for declaring the output hash column (default `&md5_col`) before the call
|
||||
if it does not already exist on the input dataset.
|
||||
|
||||
@param [in] md5_col= Name of the output hash column.
|
||||
@param [in] char_vars= Space separated list of character variables to hash.
|
||||
@param [in] num_vars= Space separated list of numeric variables to hash.
|
||||
@param [in] prefix_num_vars= Optional space separated list of numeric
|
||||
variables to hash before `num_vars` (e.g. business dates in bitemporal
|
||||
mode).
|
||||
|
||||
<h4> SAS Macros </h4>
|
||||
@li bitemporal_dataloader.sas
|
||||
|
||||
@version 9.3M5
|
||||
@author 4GL Apps Ltd.
|
||||
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
|
||||
and may not be re-distributed or re-sold without the express permission of
|
||||
4GL Apps Ltd.
|
||||
**/
|
||||
|
||||
%macro mpe_rowhash(
|
||||
md5_col=
|
||||
,char_vars=
|
||||
,num_vars=
|
||||
,prefix_num_vars=
|
||||
);
|
||||
|
||||
/* Internal system variables: five underscores before and after. */
|
||||
length _____state_____ _____digest_____ $16
|
||||
_____pair_____ $32
|
||||
_____numtext_____ $64
|
||||
_____normal_____ 8;
|
||||
drop _____state_____ _____digest_____ _____pair_____
|
||||
_____numtext_____ _____normal_____
|
||||
%if %length(&char_vars &prefix_num_vars &num_vars)>0 %then _____i_____;;
|
||||
|
||||
/* Versioned seed prevents confusion with other hashing schemes. */
|
||||
_____state_____ = md5('DC HASH v2');
|
||||
|
||||
%if %length(&char_vars)>0 %then %do;
|
||||
array _____chars_____{*} $ &char_vars;
|
||||
do _____i_____ = 1 to dim(_____chars_____);
|
||||
/* Leading blanks are retained. */
|
||||
_____digest_____ = md5(trimn(_____chars_____[_____i_____]));
|
||||
substr(_____pair_____, 1, 16) = _____state_____;
|
||||
substr(_____pair_____, 17, 16) = _____digest_____;
|
||||
_____state_____ = md5(_____pair_____);
|
||||
end;
|
||||
%end;
|
||||
|
||||
%if %length(&prefix_num_vars)>0 %then %do;
|
||||
array _____prefix_____{*} &prefix_num_vars;
|
||||
do _____i_____ = 1 to dim(_____prefix_____);
|
||||
/* multiply-by-one for cross-system precision handling */
|
||||
_____normal_____ = ifn(
|
||||
missing(_____prefix_____[_____i_____]),
|
||||
_____prefix_____[_____i_____],
|
||||
_____prefix_____[_____i_____] * 1
|
||||
);
|
||||
_____numtext_____ = put(_____normal_____, binary64.);
|
||||
_____digest_____ = md5(trim(_____numtext_____));
|
||||
substr(_____pair_____, 1, 16) = _____state_____;
|
||||
substr(_____pair_____, 17, 16) = _____digest_____;
|
||||
_____state_____ = md5(_____pair_____);
|
||||
end;
|
||||
%end;
|
||||
|
||||
%if %length(&num_vars)>0 %then %do;
|
||||
array _____nums_____{*} &num_vars;
|
||||
do _____i_____ = 1 to dim(_____nums_____);
|
||||
/* multiply-by-one for cross-system precision handling */
|
||||
_____normal_____ = ifn(
|
||||
missing(_____nums_____[_____i_____]),
|
||||
_____nums_____[_____i_____],
|
||||
_____nums_____[_____i_____] * 1
|
||||
);
|
||||
_____numtext_____ = put(_____normal_____, binary64.);
|
||||
_____digest_____ = md5(trim(_____numtext_____));
|
||||
substr(_____pair_____, 1, 16) = _____state_____;
|
||||
substr(_____pair_____, 17, 16) = _____digest_____;
|
||||
_____state_____ = md5(_____pair_____);
|
||||
end;
|
||||
%end;
|
||||
|
||||
&md5_col = put(_____state_____, $hex32.);
|
||||
|
||||
%mend mpe_rowhash;
|
||||
Reference in new issue
Block a user