feat: new hash algorithm - allows 1000+ cols. Closes #269
Build / Build-and-ng-test (pull_request) Successful in 5m49s
Lighthouse Checks / lighthouse (pull_request) Successful in 22m52s
Build / Build-and-test-development (pull_request) Successful in 15m34s

This commit is contained in:
dcbot committed 2026-07-17 22:53:53 +01:00
1 parent efcccd7d11
commit 08947f0fc8
2 files changed
+149 -24

No files matched your search

+48 -24
View File
@@ -111,6 +111,7 @@ Areas for optimisation
@li mp_lockfilecheck.sas
@li mp_retainedkey.sas
@li mp_storediffs.sas
@li mpe_rowhash.sas
@version 9.3
@author 4GL Apps Ltd.
@@ -193,7 +194,7 @@ Areas for optimisation
/* when changing this macro, update the version num here */
%local ver;
%let ver=32;
%let ver=33;
%put &sysmacroname entry vars:;
%put _local_;
@@ -246,7 +247,7 @@ run;
)
%local cols idx_pk md5_col ;
%local cols idx_pk md5_col hash_char_vars hash_num_vars ;
%let md5_col=___TMP___md5;
%let check_uniqueness=%upcase(&check_uniqueness);
%let RK_UPDATE_MAXKEYTABLE=%upcase(&RK_UPDATE_MAXKEYTABLE);
@@ -321,14 +322,20 @@ select name into: cols separated by ','
(%upcase("&bus_from","&bus_to"
,"&tech_from","&tech_to"
,"&processed","&delete_col")) ;
select case when type in (2,6) then cats('put(md5(trim(',name,')),$hex32.)')
/* multiply by 1 to strip precision errors (eg 0 != 0) */
/* but ONLY if not missing, else will lose any special missing values */
else cats('put(md5(trim(put(ifn(missing('
,name,'),',name,',',name,'*1),binary64.))),$hex32.)') end
into: stripcols separated by '||'
/* Character variables are hashed separately from numerics so that the
iterative hash can be built with arrays rather than by concatenating
hundreds of 32-byte hex strings. */
select name into: hash_char_vars separated by ' '
from work.bitemp_cols
where upcase(name) not in
where type in (2,6)
and upcase(name) not in
(%upcase("&bus_from","&bus_to"
,"&tech_from","&tech_to"
,"&processed","&delete_col")) ;
select name into: hash_num_vars separated by ' '
from work.bitemp_cols
where type not in (2,6)
and upcase(name) not in
(%upcase("&bus_from","&bus_to"
,"&tech_from","&tech_to"
,"&processed","&delete_col")) ;
@@ -342,7 +349,8 @@ select case when type in (2,6) then cats('put(md5(trim(',name,')),$hex32.)')
%let tech_to_fmt = format=datetime19.;
%put &=stripcols;
%put &=hash_char_vars;
%put &=hash_num_vars;
%put &=pk;
data _null_;
@@ -368,8 +376,8 @@ data _null_;
call symputx('processed_fmt',fmt,'L');
run;
%if %index(%quote(&cols),___TMP___) %then %do;
%let msg=%str(Table contains a variable name containing "___TMP___".%trim(
%if %index(%quote(&cols),___TMP___) or %index(%quote(&cols),_____) %then %do;
%let msg=%str(Table contains a variable name containing "___TMP___" or "_____".%trim(
) This may conflict with temp variable generation!!);
%mp_abort(msg=&msg,mac=bitemporal_dataloader);
%let syscc=5;
@@ -485,7 +493,10 @@ data work.bitemp0_append &keepvars &outds_del(drop=&md5_col )
&bus_to= %unquote(&bus_to_override) ;
%end;
length &md5_col $32;
&md5_col=put(md5(&stripcols),hex32.);
%mpe_rowhash(md5_col=&md5_col
,char_vars=&hash_char_vars
,num_vars=&hash_num_vars
);
%if %length(&processed)>0 %then %do;
format &processed &processed_fmt;
&processed=&now;
@@ -772,13 +783,17 @@ or &engine_type=SNOW or &engine_type=SASIOSNF
,msg=%str(syscc=&syscc sqlrc=&sqlrc)
)
%put hashcols2=&stripcols;
proc sql;
create table work.bitemp1_current(drop=___TMP___NEW_FLG) as
select *
, put(md5(&stripcols),$hex32.) as &md5_col
from work.bitemp0_base (drop=&md5_col)
where ___TMP___NEW_FLG=0;
%put hash_char_vars=&hash_char_vars;
%put hash_num_vars=&hash_num_vars;
data work.bitemp1_current(drop=___TMP___NEW_FLG);
set work.bitemp0_base(drop=&md5_col);
if ___TMP___NEW_FLG=0;
length &md5_col $32;
%mpe_rowhash(md5_col=&md5_col
,char_vars=&hash_char_vars
,num_vars=&hash_num_vars
);
run;
/**
* NEW records were identified in ___TMP___NEW_FLG in bitemp0_base
@@ -1061,7 +1076,9 @@ run;
*/
%if &loadtype=BITEMPORAL %then %do;
%let cat_string=catx('|' ,&bus_from,&bus_to);
/* For bitemporal we also include the business dates in the comparison hash.
The business dates are passed as prefix numerics so that both lookup and
update tables use exactly the same hashing order. */
data work.bitemp5a_lkp (keep=&md5_col)
%if "%substr(&sysver,1,1)" ne "4" & "%substr(&sysver,1,1)" ne "5" %then %do;
@@ -1069,8 +1086,11 @@ run;
%end;
;
set work.bitemp0_base;
/* for BITEMPORAL we need to compare business dates also */
&md5_col=put(md5(&cat_string!!'|'!!&stripcols),$hex32.);
%mpe_rowhash(md5_col=&md5_col
,char_vars=&hash_char_vars
,num_vars=&hash_num_vars
,prefix_num_vars=&bus_from &bus_to
);
run;
data bitemp5b_updates;
@@ -1081,7 +1101,11 @@ run;
md5_lkp.definedone();
end;
/* drop old md5 col as will rebuild with new business dates */
&md5_col=put(md5(&cat_string!!'|'!!&stripcols),$hex32.) ;
%mpe_rowhash(md5_col=&md5_col
,char_vars=&hash_char_vars
,num_vars=&hash_num_vars
,prefix_num_vars=&bus_from &bus_to
);
if md5_lkp.check()=0 then delete;
run;
+101
View File
@@ -0,0 +1,101 @@
/**
@file
@brief Iterative row-level MD5 hash generator
@details Generates DATA step statements that compute a deterministic,
row-level MD5 hash using a Merkle-style construction. Each variable is
hashed independently and the resulting raw 16-byte digests are combined
iteratively. This avoids concatenating the per-variable hex digests into a
single SAS character expression, which overflows when datasets contain
many variables (>1000).
All temporary variables use five leading and five trailing underscores
to comply with the internal naming standard for system variables.
This macro is called from inside a DATA step. The caller is responsible
for declaring the output hash column (default `&md5_col`) before the call
if it does not already exist on the input dataset.
@param [in] md5_col= Name of the output hash column.
@param [in] char_vars= Space separated list of character variables to hash.
@param [in] num_vars= Space separated list of numeric variables to hash.
@param [in] prefix_num_vars= Optional space separated list of numeric
variables to hash before `num_vars` (e.g. business dates in bitemporal
mode).
<h4> SAS Macros </h4>
@li bitemporal_dataloader.sas
@version 9.3M5
@author 4GL Apps Ltd.
@copyright 4GL Apps Ltd. This code may only be used within Data Controller
and may not be re-distributed or re-sold without the express permission of
4GL Apps Ltd.
**/
%macro mpe_rowhash(
md5_col=
,char_vars=
,num_vars=
,prefix_num_vars=
);
/* Internal system variables: five underscores before and after. */
length _____state_____ _____digest_____ $16
_____pair_____ $32
_____numtext_____ $64
_____normal_____ 8;
drop _____state_____ _____digest_____ _____pair_____
_____numtext_____ _____normal_____
%if %length(&char_vars &prefix_num_vars &num_vars)>0 %then _____i_____;;
/* Versioned seed prevents confusion with other hashing schemes. */
_____state_____ = md5('DC HASH v2');
%if %length(&char_vars)>0 %then %do;
array _____chars_____{*} $ &char_vars;
do _____i_____ = 1 to dim(_____chars_____);
/* Leading blanks are retained. */
_____digest_____ = md5(trimn(_____chars_____[_____i_____]));
substr(_____pair_____, 1, 16) = _____state_____;
substr(_____pair_____, 17, 16) = _____digest_____;
_____state_____ = md5(_____pair_____);
end;
%end;
%if %length(&prefix_num_vars)>0 %then %do;
array _____prefix_____{*} &prefix_num_vars;
do _____i_____ = 1 to dim(_____prefix_____);
/* multiply-by-one for cross-system precision handling */
_____normal_____ = ifn(
missing(_____prefix_____[_____i_____]),
_____prefix_____[_____i_____],
_____prefix_____[_____i_____] * 1
);
_____numtext_____ = put(_____normal_____, binary64.);
_____digest_____ = md5(trim(_____numtext_____));
substr(_____pair_____, 1, 16) = _____state_____;
substr(_____pair_____, 17, 16) = _____digest_____;
_____state_____ = md5(_____pair_____);
end;
%end;
%if %length(&num_vars)>0 %then %do;
array _____nums_____{*} &num_vars;
do _____i_____ = 1 to dim(_____nums_____);
/* multiply-by-one for cross-system precision handling */
_____normal_____ = ifn(
missing(_____nums_____[_____i_____]),
_____nums_____[_____i_____],
_____nums_____[_____i_____] * 1
);
_____numtext_____ = put(_____normal_____, binary64.);
_____digest_____ = md5(trim(_____numtext_____));
substr(_____pair_____, 1, 16) = _____state_____;
substr(_____pair_____, 17, 16) = _____digest_____;
_____state_____ = md5(_____pair_____);
end;
%end;
&md5_col = put(_____state_____, $hex32.);
%mend mpe_rowhash;