From 70598074820b53bc67a6f32fec5696d3aa046425 Mon Sep 17 00:00:00 2001 From: Chris Marooney Date: Thu, 12 Dec 2024 10:28:25 +0000 Subject: [PATCH 01/21] add and test hash serialize --- .../store2020_1.mat | Bin 0 -> 2888 bytes .../test_unique_objects.m | 10 ++++ .../@serializable/private/from_bare_struct_.m | 1 + .../private/check_combo_arg_.m | 7 +-- .../unique_objects_container.m | 45 ++++++++++++++++-- 5 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 _test/test_unique_objects_container/store2020_1.mat diff --git a/_test/test_unique_objects_container/store2020_1.mat b/_test/test_unique_objects_container/store2020_1.mat new file mode 100644 index 0000000000000000000000000000000000000000..0986498cef3edff20b9707e1bc6eab2d7dbb4cbd GIT binary patch literal 2888 zcmV-O3%B%5K~zjZLLfCRFd$7qR4ry{Y-KDUP;6mzW^ZzBIv`L(S4mDbG%O%Pa%Ew3 zWn>_4ZaN@WWn>^kWn&;QF(5HGIxsdmH8~(MFfueCARr(hARr(hARr(hARr(hARr(h zARr(hARr(hARr(hARr(hARr*<00000000000ZB~{0001B0001Zoa19)V3+{JY(UHb z#DWYEz{tSNP@10%q*)kzo&AFufnpLsT$-0zSehE2pOlrFTv8mLoS#>cn3Py6$vgf$e$U@~wx7q_1AvRy0owTN zVCh03D+)KTJlrgvQkn`wqrW9_56hdIrG9gYzAi<(lfP@IK3f~_Y->AU*m+v@Z;YQ# z-dEyPEN}3Dy_e89W(grm|6rrkFUKoz&*CBR&6UIbetxCezp2_<_T?!& zDS?oVP%VwJ#Kq?x4Y)`ABpT_uh*XV>eS!uUNtqzvMK}m^cZnN~1`(mj4w0-h5%72_ zXedp7z&$Y}Fl2mZC`=kTgTfkWR)~Y2u=g_-y88qY>hTPTx>>SE3n$Zn3!?1i^)#j~ z50L^3S+avl@sOt2@irGih5_x5BY^?j3GiK<1=vM42MA&eU69=wW50r9niNFEkh{Af zOOcOYqgYzZ8Q}m$2ti;duxAlg^pwgUCisYfKN2AB%D!P11o<>T7IJiyM1$MR6)8qD z6qKVA_6tanI`atnHDn06cZa|Lopb{#*@VQ);()t2mW)*t?%_np$xB7T=!l>{VT7HS zPqP4LpA=nEj*YozQ>AI)RSQ33;b$#;$-vk3 zP7@}5eZjDA;tu?O(lef&-pS4douda>QkHNUdw`^?aH2DV*^RO=UhtA33bzb3btazEx(iuLFluW5g z-^i<&R!bsfI>kPI*`DWdRWA;SqCQTwvR>v|mEtGkK1;l{Upn6Bi+ES2eG&1lqu>7M zi;d5|yIfn}jDx8^Ee>s{#zA7f>(V4v-(AOy--7D+7qNt2efsMC-!80!CA083XYr93 z4?sUkM!EjWNAS^N@|ia7ym35J`uF^GJMQ+pX))(bk1LZ*oqQwL-7NX$u$*zyeDI{j zUt;qscJ4UNkq^H8`@z=W=B0zW`k%+2Jb3uyPp>aRzwSCX>AAD2@O`k(>B=!}u7vWY zAVG0gPzJ{r1_{b5g9hr(P+3cr#Z)&FFInt6(4$&+zy2cS z-`eXYmxGUMkN@F??{EBmSIjm3U;Lli`Gdgsn-aQL}nZg1mqUrjtP+rHQL*MfI}%fAjvJM-@Q|5r2pm{_K-8fnc;9a8WB*yhw3?3&w+X#n_mq> zgZHdQ!^n&qxYYb(i+|womr(LQwbpIRw}ZYr@E`vB+sAMI(ENY*SN#1A{6l5g6N77e z1>qD$kfE4>E5Zd$SWYLBj0dFP$tNv+FW2}<%Pr-Xe$x2&FCfdJ2vR0w2Y$=<`E9SA z-QTRS+qP9AJ}0uE2|0P45fI6s?b`okytiHVVZZltzjw`VKfTv%-#;?= zkD$8m3hb>{Fhha_5fY+6Ge*^>!({f71X2JY6HYK(Ah95dg5qmX1Fhe8Yy7J9c<%W1 zd2J~eBupa82;a@$ci+1FK7AfJ`ePhXIsaM`IG0B7nYnSj+`>L z+5Y~SF@?b*N!9#n zE>1AvNTrJ^d5jp=qc{24f*~al7BIha9Jv~bd!jaRg(8Y!$dee>hh+ji)KuO;wJ?D~ zU1#|)PL$Flc%>3eJWHUu=-|pvU7R=N>fFa&$r~7%UCA37neII}>R08Jwv_VvIkwyV z*@gZGhNmlekIbBFmzzIayGE8>oBU|!oxVAN+wk=&e7)tpaM9Acs`?^s+Lu?+^NYgA*$wQa7(P}D9J=JR%W!EktNW=A%DdMI^ z-9E|51r1&?A}U!{k~q~BcqPet(_xb&2lhM9U9Qo)j4fMljjLUe2!A|ZocJ$P4FccgP=fOjvq&3b+o6Iyl>>KzI{BpuRipi3&8oG533nBS8`MtdN zfAIhSCt^52eH}}i`Am$y7@pquTF(1hxMPb07T5cS`)-dmoL!fz|81-5wBermc~4dF zExDDu;oKIkBeLu{=ctS)(bIi2j2{R6K|gKpmt9v+tl|bA_u)d#eK~*g(bxK_ z<_qA<1yW9lvet_%hh=H}Tfd8z{LU@Eq@E)_Tyiwy48dZ|Xb~|6mnvS^60d8;>zbrW zTr6~w`{Z?D#gSUIEl4Yz^J`23Ml+g4YWxPx>N;nknkJ)|o-QE3h_39AL m(cEX9Ysmlk7pF9E?j}us?(?2HTFZGddYG9w^7{ddp4h^Y(Wox~ literal 0 HcmV?d00001 diff --git a/_test/test_unique_objects_container/test_unique_objects.m b/_test/test_unique_objects_container/test_unique_objects.m index 279f1ca989..b71889adf7 100644 --- a/_test/test_unique_objects_container/test_unique_objects.m +++ b/_test/test_unique_objects_container/test_unique_objects.m @@ -496,5 +496,15 @@ function test_arrays_of_containers(obj) assertTrue(isa(arr{2},'unique_objects_container')); end + function test_hashing_preserved_over_save_and_load(obj) + uoc = unique_objects_container('baseclass','IX_inst'); + uoc{1} = obj.mi1; + uoc{2} = IX_null_inst(); + save('store2020_1.mat','uoc'); + zzz = load('store2020_1.mat'); + assertEqual(uoc.stored_hashes, zzz.uoc.stored_hashes); + + end + end end diff --git a/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m b/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m index 79e870bf68..14b2038237 100644 --- a/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m +++ b/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m @@ -79,6 +79,7 @@ obj(i).do_check_combo_arg_ = true; % Check interdependent properties. If the object is invalid, an % exception is thrown + obj(i) = obj(i).check_combo_arg(); end if nobj > 1 diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m index 0420901a61..a83e510b0f 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m @@ -45,6 +45,7 @@ obj.baseclass,disp2str(non_type_ind),class(invalid_obj)) end end -if do_rehashify - obj = obj.rehashify_all(with_checks); -end +% not doing rehashify here as hashes are now loaded from a saved object +%if do_rehashify +% obj = obj.rehashify_all(with_checks); +%end diff --git a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m index b3e6299988..0c8e5faef4 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m +++ b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m @@ -195,6 +195,17 @@ error('HERBERT:unique_objects_container:invalid_set', ... 'attempt to set unique objects in container outside of loadobj'); end + %--{ + if ~isempty(self.stored_hashes_) + for ii = 1:numel(self.stored_hashes_) + hash = self.stored_hashes_{ii}; + hash2 = Hashing.hashify_obj(self.unique_objects_{ii}); + if hash ~= hash2 + error("bad hash"); + end + end + end + %--} end % function x = get.stored_hashes(self) @@ -202,6 +213,31 @@ % objects. Only really useful for debugging. x = self.stored_hashes_; end + + function self = set.stored_hashes(self, val) + %GET.STORED_HASHES - list the hashes corresponding to the unique + % objects. Only really useful for debugging. + if ~self.do_check_combo_arg_ + if ~iscell(val) + val = {val}; + end + self.stored_hashes_ = val; + else + error('HERBERT:unique_objects_container:invalid_set', ... + 'attempt to set stored hashes in container outside of loadobj'); + end + %--{ + if ~isempty(self.unique_objects_) + for ii = 1:numel(self.unique_objects) + obj = self.unique_objects_{ii}; + hash = Hashing.hashify_obj(obj); + if hash ~= self.stored_hashes_{ii} + error("bad hash"); + end + end + end + %--} + end % function x = get.idx(self) %GET.IDX - get the indices of each stored object in the container @@ -795,6 +831,7 @@ function list(self,field) fields_to_save_ = { 'baseclass', ... 'unique_objects',... + 'stored_hashes', ... 'idx', ... 'conv_func_string'}; end @@ -847,9 +884,11 @@ function list(self,field) % save-able class obj = unique_objects_container(); obj = loadobj@serializable(S,obj); - if obj.do_check_combo_arg - obj.check_combo_arg(true,true); - end + % not doing a check combo arg here as it is done in the loadobj + % of serializable above + %if obj.do_check_combo_arg + % obj.check_combo_arg(true,true); + %end end function out = concatenate(objs, type) From bd847e7c3b3ecd5a45e26834da7dcf6dbcd91050 Mon Sep 17 00:00:00 2001 From: Chris Marooney Date: Thu, 12 Dec 2024 17:12:21 +0000 Subject: [PATCH 02/21] fix required hashing --- .../@unique_objects_container/private/check_combo_arg_.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m index a83e510b0f..890704ff25 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m @@ -49,3 +49,6 @@ %if do_rehashify % obj = obj.rehashify_all(with_checks); %end +if isempty(obj.stored_hashes_) && ~isempty(obj.unique_objects) + obj = obj.rehashify_all(with_checks); +end From a4de98647c3f828d78c9454cde96ec49285a7a92 Mon Sep 17 00:00:00 2001 From: abuts Date: Fri, 13 Dec 2024 12:37:25 +0000 Subject: [PATCH 03/21] Re #1788 suggested hashable implementation (incomplete) --- herbert_core/utilities/classes/@hashable/eq.m | 28 ++++ .../utilities/classes/@hashable/hashable.m | 125 ++++++++++++++++++ herbert_core/utilities/classes/@hashable/ne.m | 32 +++++ .../utilities/classes/@hashable/private/eq_.m | 44 ++++++ .../@hashable/private/to_hashable_array_.m | 33 +++++ .../private/add_single_.m | 2 +- .../unique_objects_container.m | 95 ++----------- herbert_core/utilities/misc/build_hash.m | 53 ++++++++ 8 files changed, 327 insertions(+), 85 deletions(-) create mode 100644 herbert_core/utilities/classes/@hashable/eq.m create mode 100644 herbert_core/utilities/classes/@hashable/hashable.m create mode 100644 herbert_core/utilities/classes/@hashable/ne.m create mode 100644 herbert_core/utilities/classes/@hashable/private/eq_.m create mode 100644 herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m create mode 100644 herbert_core/utilities/misc/build_hash.m diff --git a/herbert_core/utilities/classes/@hashable/eq.m b/herbert_core/utilities/classes/@hashable/eq.m new file mode 100644 index 0000000000..90097930f4 --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/eq.m @@ -0,0 +1,28 @@ +function [iseq, mess] = eq (obj1, obj2, varargin) +% Return a logical variable stating if two serializable objects are equal or not +% +% >> [iseq, mess] = eq (obj1, obj2) +% >> [iseq, mess] = eq (obj1, obj2, p1, p2, ...) +% +% Input: +% ------ +% obj1 Object on left-hand side +% +% obj2 Object on right-hand side +% +% Optional: +% p1, p2,... Any set of parameters that the equal_to_tol function accepts +% +% See also equal_to_tol + + +names = cell(2,1); +if nargout == 2 + names{1} = inputname(1); + names{2} = inputname(2); + [iseq, mess] = eq_ (obj1, obj2, nargout, names, varargin{:}); +else + iseq = eq_ (obj1, obj2, nargout, names, varargin{:}); +end + +end diff --git a/herbert_core/utilities/classes/@hashable/hashable.m b/herbert_core/utilities/classes/@hashable/hashable.m new file mode 100644 index 0000000000..b733e1776d --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/hashable.m @@ -0,0 +1,125 @@ +classdef hashable < serializable + % + + properties (Access=protected) + hash_value_ = [] + end + + %--------------------------------------------------------------------------- + % Constructor + %--------------------------------------------------------------------------- + methods + function obj = hashable() + % Class constructor. + % Does nothing except enable methods of the base serializable class + % to be accessed. + end + end + + %--------------------------------------------------------------------------- + % INTERFACE + %--------------------------------------------------------------------------- + % Convert object or array of objects to/from a structure + %--------------------------------------------------------------------------- + methods + function flds = hashingFields (obj) + % function provides set of fields which define hash. By + % default, eqyak ti saveableFields, but different to give + % possibility to overload + flds = obj.saveableFields(); + end + + function S = to_struct (obj) + % overload to_struct to add hash to it if hash was available + S = to_struct@serializable(obj); + if ~isempty(obj.hash_value_) + S.hash_value = obj.hash_value_; + end + end + function [obj,bytestream] = to_hashable_array(obj) + % Function which extracts distignuishable information from the + % object to use as basis for the hash which describes this + % object. + [obj,bytestream] = to_hashable_array_(obj); + end + + + function [obj,hash] = build_hash(obj) + % calculate hash if it not available + if ~isempty(obj.hash_value_) + hash = obj.hash_value_; + return; + end + [obj,bytestream] = to_hashable_array(obj); + use_mex = config_store.instance().get_value('hor_config','use_mex'); + persistent Engine; + % In case the java engine is going to be used, initialise it as + % a persistent object + + if use_mex + % mex version to be used, use it + hash = GetMD5(bytestream); + else + + if isempty(Engine) + Engine = java.security.MessageDigest.getInstance('MD5'); + end + + + % mex version not to be used, manually construct from the + % Java engine + Engine.update(bytestream); + hash0 = Engine.digest; + + %using the following typecast to remedy that dec2hex + %does not work with negative numbers before Matlab 2020b. + %the typecast moves negative numbers to twos-complement + %positive representation, as is automatically done by the + %later dec2hex + hash1 = typecast(hash0,'uint8'); + + hash2 = dec2hex(hash1); + hash3 = cellstr(hash2); + hash4 = horzcat(hash3{:}); + hash = lower(hash4); % reduces hash ! + end + end + end + + methods (Static) + function obj = from_struct (S, varargin) + % overload from_struct to restore hash if available + obj = serializable.from_struct(S,varargin{:}); + if isfield(S,'hash_value') + obj.hash_value_ = S.hash_value; + end + end + end + + + %--------------------------------------------------------------------------- + % Testing equality of serializable objects + %--------------------------------------------------------------------------- + methods + % Return logical variable stating if two serializable objects are equal + % or not + [iseq, mess] = eq (obj1, obj2, varargin) + + % Return logical variable stating if two serializable objects are + % unequal or not + [isne, mess] = ne (obj1, obj2, varargin) + end + + %--------------------------------------------------------------------------- + % Object validation + %--------------------------------------------------------------------------- + methods + function obj = check_combo_arg (obj) + % overload check_combo_arg. Normally arguments have changed + % so existihg hashes should be destroyed. If they are not, + % overload this function for your class appropriately + obj = check_combo_arg@serializable(obj); + obj.hash_value_ = []; + end + end +end diff --git a/herbert_core/utilities/classes/@hashable/ne.m b/herbert_core/utilities/classes/@hashable/ne.m new file mode 100644 index 0000000000..cba336a43d --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/ne.m @@ -0,0 +1,32 @@ +function [isne, mess] = ne (obj1, obj2, varargin) +% Return a logical variable stating if two serializable objects are unequal or not +% +% >> [iseq, mess] = ne (obj1, obj2) +% >> [iseq, mess] = ne (obj1, obj2, p1, p2, ...) +% +% Input: +% ------ +% obj1 Object on left-hand side +% +% obj2 Object on right-hand side +% +% Optional: +% p1, p2,... Any set of parameters that the equal_to_tol function accepts +% +% See also equal_to_tol + +% TODO: can be done more efficiently as eq needs to check all +% the fields and ne may return when found first non-equal field + + +names = cell(2,1); +if nargout == 2 + names{1} = inputname(1); + names{2} = inputname(2); + [iseq, mess] = eq_ (obj1, obj2, nargout, names, varargin{:}); +else + iseq = eq_ (obj1, obj2, nargout, names, varargin{:}); +end +isne = ~iseq; + +end diff --git a/herbert_core/utilities/classes/@hashable/private/eq_.m b/herbert_core/utilities/classes/@hashable/private/eq_.m new file mode 100644 index 0000000000..4ec47ea1b2 --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/private/eq_.m @@ -0,0 +1,44 @@ +function [iseq, mess] = eq_ (obj1, obj2, narg_out, names, varargin) +% Check equality of two serializable objects + +[iseq, mess, name_a, name_b, namer, argi] = obj1.process_inputs_for_eq (obj2, ... + narg_out, names, varargin{:}); +if ~iseq + return +end + +iseq = false(numel(obj1),1); +for i=1:numel(obj1) + if nargout == 2 + name_1 = namer(name_a,i); + name_2 = namer(name_b,i); + [iseq(i), mess{i}] = eq_single (obj1(i), obj2(i), ... + 'name_a', name_1, 'name_b', name_2, argi{:}); + else + iseq(i) = eq_single(obj1(i),obj2(i), ... + 'name_a', name_a, 'name_b', name_b, argi{:}); + end +end + +if narg_out > 1 + if any(~iseq) + mess = strjoin(mess,'; '); + else + mess = ''; + end +end + +end + + +%------------------------------------------------------------------------------- +function [iseq, mess] = eq_single (obj1, obj2, ... + name_a, name_a_val, name_b, name_b_val, varargin) +% Compare single pair of serializeble objects + +struc1 = obj1.to_bare_struct(); +struc2 = obj2.to_bare_struct(); +[iseq,mess] = equal_to_tol (struc1, struc2, ... + name_a, name_a_val, name_b, name_b_val, varargin{:}); + +end diff --git a/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m b/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m new file mode 100644 index 0000000000..952390e037 --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m @@ -0,0 +1,33 @@ +function [obj,arr] = to_hashable_array_ (obj) +% Retrieve information specifying +% +% >> S = to_hashable_array_(obj) +% +% Input: +% ------ +% obj Object or array of objects which are hashable +% +% +% Output: +% ------- +% arr array of unique data -- basis for building unique object +% hash + + +% Get saveable fields +field_names = hashingFields (obj(1)); + +% Recursively turn serializable fields into structures +cell_dat = cell (numel(field_names), numel(obj)); +for j = 1:numel(obj) + obj_tmp = obj(j); % get pointer to jth object to save expensive indexing + for i = 1:numel(field_names) + field_name = field_names{i}; + val = obj_tmp.(field_name); + if isa(val,'hashable') + [val,hash] = build_hash(val); + else + end + cell_dat{i,j} = val; + end +end diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m b/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m index 6d8adba10e..8f4692b8c6 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m @@ -30,7 +30,7 @@ % if ix and hash are not specified, call find_in_container to get them if nargin<=2 - [ix,hash] = self.find_in_container(obj); + [ix,hash,obj] = self.find_in_container(obj); end % If the object is not in the container. diff --git a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m index 0c8e5faef4..3dfac7710b 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m +++ b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m @@ -89,12 +89,6 @@ baseclass_ = ''; % if not empty, name of the baseclass suitable for isa calls % (default respecified in constructor inputParser) n_duplicates_ = zeros(1,0); - - % hashify defaults to this undocumented java function handle - % if you try to store objects non-children for serializable and - % the function to convert objects to bytes has not been set - % explicitly - convert_to_stream_f_ = @getByteStreamFromArray; end properties(Access = private) % is set to true if we decide not to use default stream conversion @@ -135,24 +129,6 @@ % Dependent properties set/get functions and subsrefs/subsassign % methods methods - function val = get.conv_func_string(obj) - %GET.CONV_STRING_FUNCTION - report the function handle used to - % generate hashes - val = func2str(obj.convert_to_stream_f_); - end - function obj = set.conv_func_string(obj,val) - %SET.CONV_STRING_FUNCTION - set the function handle used to - % generate hashes. Should only be used with loadobj. After the - % container is created, changing this functionmid-stream may - % invalidate the comparison for unique objects. - - if ~(ischar(val)||isstring(val)) - error('HERBERT:unique_obj_container:invalid_argument',... - 'convert_to_stream_f_string must be a string convertable to function. It is %s',... - class(val)) - end - obj.convert_to_stream_f = str2func(val); - end % function x = expose_unique_objects(self) %EXPOSE_UNIQUE_OBJECTS - return the cell array containing the @@ -199,7 +175,7 @@ if ~isempty(self.stored_hashes_) for ii = 1:numel(self.stored_hashes_) hash = self.stored_hashes_{ii}; - hash2 = Hashing.hashify_obj(self.unique_objects_{ii}); + [self.unique_objects_{ii},hash2] = build_hash(self.unique_objects_{ii}); if hash ~= hash2 error("bad hash"); end @@ -230,8 +206,8 @@ if ~isempty(self.unique_objects_) for ii = 1:numel(self.unique_objects) obj = self.unique_objects_{ii}; - hash = Hashing.hashify_obj(obj); - if hash ~= self.stored_hashes_{ii} + [obj,hash2] = build_hash(obj); + if hash ~= hash2 error("bad hash"); end end @@ -293,28 +269,6 @@ end end % - function x = get.convert_to_stream_f(self) - %GET.CONVERT_TO_STREAM - retrieve the hashing function - x = self.convert_to_stream_f_; - end - function self = set.convert_to_stream_f(self,val) - %SET.CONVERT_TO_STREAM - (re)set the hashing function - % This may invalidate the contents by changing the hashing function - % so should not be used if the container is not empty - if ~(isempty(val)|| isa(val,'function_handle')) - error('HERBERT:unique_objects_container:invalid_argument',... - 'this method accepts function handles for serializing objects only') - end - if isequal(self.convert_to_stream_f_,val) - return; - end - self.convert_to_stream_f_ = val; - self.non_default_f_conversion_set_ = true; - if self.do_check_combo_arg_ - self = self.check_combo_arg(true,false); - end - end - % function x = get.n_duplicates(self) x = self.n_duplicates_; end @@ -446,28 +400,6 @@ end end - function hash = hashify(self,obj,reset_count) - % makes a hash from the argument object - % which will be unique to any identical object - % - % Input: - % - obj : object to be hashed - % - reset_count : if counting hash calcs is exposed in - % hashify_obj then this resets the count - % Output: - % - hash : the resulting has, a row vector of uint8's - % - % this method started out as an instance method but now - % contains no references to self. For simplicity, keeping the - % original method (this one) as a wrapper to the static method - % now used => no other code changes - if nargin<=2 - hash = Hashing.hashify_obj(obj); - else - hash = Hashing.hashify_obj(obj,reset_count); - end - end - function self = rehashify_all(self,with_checks) % recalculate hashes of all objects, stored in the container % @@ -479,7 +411,7 @@ end self.stored_hashes_ =cell(1,self.n_unique); for i=1:self.n_unique - hash = self.hashify(self.unique_objects{i}); + [self.unique_objects{i},hash] = build_hash(self.unique_objects{i}); if with_checks is = ismember(hash,self.stored_hashes_(1:i-1)); if is @@ -493,7 +425,7 @@ end methods - function [ix, hash] = find_in_container(self,obj) + function [ix, hash,obj] = find_in_container(self,obj) %FIND_IN_CONTAINER Finds if obj is contained in self % Input: % - obj : the object which may or may not be uniquely contained @@ -503,7 +435,7 @@ % if it is stored, otherwise empty [] % - hash : the hash of the object from hashify % - hash = self.hashify(obj); + [obj,hash] = build_hash(obj); if isempty(self.stored_hashes_) ix = []; % object not stored as nothing is stored else @@ -528,9 +460,6 @@ % conversion for hashify flds = self.saveableFields(); - flds = [flds(:);'convert_to_stream_f']; % convert_to_stream - % function is not present in saveable properties but may be present - % as input too. % standard serializable constructor self = self.set_positional_and_key_val_arguments(... flds,false,varargin{:}); @@ -648,7 +577,7 @@ % check if you're trying to replace an object with an identical % one. If so silently return. - objhash = self.hashify(obj); + [obj,objhash] = build_hash(obj); curhash = self.stored_hashes_{self.idx_(nuix)}; if isequal(objhash, curhash) return; @@ -665,7 +594,7 @@ % returned as the index to the object in the container. % hash is returned as the hash of the object. If ix is empty % then the object is not in the container. - [ix,hash] = self.find_in_container(obj); + [ix,hash,obj] = self.find_in_container(obj); % If the object is not in the container. % store the hash in the stored hashes @@ -776,9 +705,7 @@ function list(self,field) % This is only used for tests and so its efficiency is not % important. newself = unique_objects_container('baseclass',self.baseclass);%,'convert_to_stream_f',self.convert_to_stream_f'); - % It is unclear why specifying convert_to_stream_f in the - % constructor causes a failure but it works if specified next. - newself.convert_to_stream_f = self.convert_to_stream_f; + for i=1:self.n_objects newself = newself.add(self.get(i)); end @@ -929,7 +856,7 @@ function list(self,field) out_obj = objs{ii}.get(jj); out_hsh = objs{ii}.hash(jj); [~,index] = ismember(out_hsh, out.stored_hashes_); - if index==0, index = []; end; + if index==0, index = []; end out = out.add_single_(out_obj,index,out_hsh); %( objs{ii}.get(jj) ); end end @@ -940,7 +867,7 @@ function list(self,field) out_obj = objs(ii).get(jj); out_hsh = objs(ii).hash(jj); [~,index] = ismember(out_hsh, out.stored_hashes_); - if index==0, index = [], end; + if index==0, index = []; end out = out.add_single_(out_obj,index,out_hsh); %( objs{ii}.get(jj) ); %out = out.add( objs(ii).get(jj) ); diff --git a/herbert_core/utilities/misc/build_hash.m b/herbert_core/utilities/misc/build_hash.m new file mode 100644 index 0000000000..b3df2ac60f --- /dev/null +++ b/herbert_core/utilities/misc/build_hash.m @@ -0,0 +1,53 @@ +function [obj,hash] = build_hash(obj) +% makes a hash from the argument object which will be unique +% when generated from any identical object +% +% Input: +% obj -- object to be hashed +% +% Output: +% obj -- unchanged object. Present here to maintain interface, common +% with hashable and other +% hash -- the resulting hash, a row vector of uint8's +% + +% get config to use use_mex +use_mex = config_store.instance().get_value('hor_config','use_mex'); +persistent Engine; +% In case the java engine is going to be used, initialise it as +% a persistent object +if isa(obj,'serializable') + bytestream = (obj.serialize()); +else + bytestream = serialize(obj); +end + +if use_mex + % mex version to be used, use it + hash = GetMD5(bytestream); +else + + if isempty(Engine) + Engine = java.security.MessageDigest.getInstance('MD5'); + end + + + % mex version not to be used, manually construct from the + % Java engine + Engine.update(bytestream); + hash0 = Engine.digest; + + %using the following typecast to remedy that dec2hex + %does not work with negative numbers before Matlab 2020b. + %the typecast moves negative numbers to twos-complement + %positive representation, as is automatically done by the + %later dec2hex + hash1 = typecast(hash0,'uint8'); + + hash2 = dec2hex(hash1); + hash3 = cellstr(hash2); + hash4 = horzcat(hash3{:}); + hash = lower(hash4); % reduces hash value to at least 64 bits! TODO: + % validate if it is possible to use +end +end From bcfc6b4330661a3d4835b26747b62aa441f802a9 Mon Sep 17 00:00:00 2001 From: Chris Marooney Date: Thu, 12 Dec 2024 10:28:25 +0000 Subject: [PATCH 04/21] add and test hash serialize --- .../store2020_1.mat | Bin 0 -> 2888 bytes .../test_unique_objects.m | 10 ++++ .../@serializable/private/from_bare_struct_.m | 1 + .../private/check_combo_arg_.m | 7 +-- .../unique_objects_container.m | 45 ++++++++++++++++-- 5 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 _test/test_unique_objects_container/store2020_1.mat diff --git a/_test/test_unique_objects_container/store2020_1.mat b/_test/test_unique_objects_container/store2020_1.mat new file mode 100644 index 0000000000000000000000000000000000000000..0986498cef3edff20b9707e1bc6eab2d7dbb4cbd GIT binary patch literal 2888 zcmV-O3%B%5K~zjZLLfCRFd$7qR4ry{Y-KDUP;6mzW^ZzBIv`L(S4mDbG%O%Pa%Ew3 zWn>_4ZaN@WWn>^kWn&;QF(5HGIxsdmH8~(MFfueCARr(hARr(hARr(hARr(hARr(h zARr(hARr(hARr(hARr(hARr*<00000000000ZB~{0001B0001Zoa19)V3+{JY(UHb z#DWYEz{tSNP@10%q*)kzo&AFufnpLsT$-0zSehE2pOlrFTv8mLoS#>cn3Py6$vgf$e$U@~wx7q_1AvRy0owTN zVCh03D+)KTJlrgvQkn`wqrW9_56hdIrG9gYzAi<(lfP@IK3f~_Y->AU*m+v@Z;YQ# z-dEyPEN}3Dy_e89W(grm|6rrkFUKoz&*CBR&6UIbetxCezp2_<_T?!& zDS?oVP%VwJ#Kq?x4Y)`ABpT_uh*XV>eS!uUNtqzvMK}m^cZnN~1`(mj4w0-h5%72_ zXedp7z&$Y}Fl2mZC`=kTgTfkWR)~Y2u=g_-y88qY>hTPTx>>SE3n$Zn3!?1i^)#j~ z50L^3S+avl@sOt2@irGih5_x5BY^?j3GiK<1=vM42MA&eU69=wW50r9niNFEkh{Af zOOcOYqgYzZ8Q}m$2ti;duxAlg^pwgUCisYfKN2AB%D!P11o<>T7IJiyM1$MR6)8qD z6qKVA_6tanI`atnHDn06cZa|Lopb{#*@VQ);()t2mW)*t?%_np$xB7T=!l>{VT7HS zPqP4LpA=nEj*YozQ>AI)RSQ33;b$#;$-vk3 zP7@}5eZjDA;tu?O(lef&-pS4douda>QkHNUdw`^?aH2DV*^RO=UhtA33bzb3btazEx(iuLFluW5g z-^i<&R!bsfI>kPI*`DWdRWA;SqCQTwvR>v|mEtGkK1;l{Upn6Bi+ES2eG&1lqu>7M zi;d5|yIfn}jDx8^Ee>s{#zA7f>(V4v-(AOy--7D+7qNt2efsMC-!80!CA083XYr93 z4?sUkM!EjWNAS^N@|ia7ym35J`uF^GJMQ+pX))(bk1LZ*oqQwL-7NX$u$*zyeDI{j zUt;qscJ4UNkq^H8`@z=W=B0zW`k%+2Jb3uyPp>aRzwSCX>AAD2@O`k(>B=!}u7vWY zAVG0gPzJ{r1_{b5g9hr(P+3cr#Z)&FFInt6(4$&+zy2cS z-`eXYmxGUMkN@F??{EBmSIjm3U;Lli`Gdgsn-aQL}nZg1mqUrjtP+rHQL*MfI}%fAjvJM-@Q|5r2pm{_K-8fnc;9a8WB*yhw3?3&w+X#n_mq> zgZHdQ!^n&qxYYb(i+|womr(LQwbpIRw}ZYr@E`vB+sAMI(ENY*SN#1A{6l5g6N77e z1>qD$kfE4>E5Zd$SWYLBj0dFP$tNv+FW2}<%Pr-Xe$x2&FCfdJ2vR0w2Y$=<`E9SA z-QTRS+qP9AJ}0uE2|0P45fI6s?b`okytiHVVZZltzjw`VKfTv%-#;?= zkD$8m3hb>{Fhha_5fY+6Ge*^>!({f71X2JY6HYK(Ah95dg5qmX1Fhe8Yy7J9c<%W1 zd2J~eBupa82;a@$ci+1FK7AfJ`ePhXIsaM`IG0B7nYnSj+`>L z+5Y~SF@?b*N!9#n zE>1AvNTrJ^d5jp=qc{24f*~al7BIha9Jv~bd!jaRg(8Y!$dee>hh+ji)KuO;wJ?D~ zU1#|)PL$Flc%>3eJWHUu=-|pvU7R=N>fFa&$r~7%UCA37neII}>R08Jwv_VvIkwyV z*@gZGhNmlekIbBFmzzIayGE8>oBU|!oxVAN+wk=&e7)tpaM9Acs`?^s+Lu?+^NYgA*$wQa7(P}D9J=JR%W!EktNW=A%DdMI^ z-9E|51r1&?A}U!{k~q~BcqPet(_xb&2lhM9U9Qo)j4fMljjLUe2!A|ZocJ$P4FccgP=fOjvq&3b+o6Iyl>>KzI{BpuRipi3&8oG533nBS8`MtdN zfAIhSCt^52eH}}i`Am$y7@pquTF(1hxMPb07T5cS`)-dmoL!fz|81-5wBermc~4dF zExDDu;oKIkBeLu{=ctS)(bIi2j2{R6K|gKpmt9v+tl|bA_u)d#eK~*g(bxK_ z<_qA<1yW9lvet_%hh=H}Tfd8z{LU@Eq@E)_Tyiwy48dZ|Xb~|6mnvS^60d8;>zbrW zTr6~w`{Z?D#gSUIEl4Yz^J`23Ml+g4YWxPx>N;nknkJ)|o-QE3h_39AL m(cEX9Ysmlk7pF9E?j}us?(?2HTFZGddYG9w^7{ddp4h^Y(Wox~ literal 0 HcmV?d00001 diff --git a/_test/test_unique_objects_container/test_unique_objects.m b/_test/test_unique_objects_container/test_unique_objects.m index 279f1ca989..b71889adf7 100644 --- a/_test/test_unique_objects_container/test_unique_objects.m +++ b/_test/test_unique_objects_container/test_unique_objects.m @@ -496,5 +496,15 @@ function test_arrays_of_containers(obj) assertTrue(isa(arr{2},'unique_objects_container')); end + function test_hashing_preserved_over_save_and_load(obj) + uoc = unique_objects_container('baseclass','IX_inst'); + uoc{1} = obj.mi1; + uoc{2} = IX_null_inst(); + save('store2020_1.mat','uoc'); + zzz = load('store2020_1.mat'); + assertEqual(uoc.stored_hashes, zzz.uoc.stored_hashes); + + end + end end diff --git a/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m b/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m index 79e870bf68..14b2038237 100644 --- a/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m +++ b/herbert_core/utilities/classes/@serializable/private/from_bare_struct_.m @@ -79,6 +79,7 @@ obj(i).do_check_combo_arg_ = true; % Check interdependent properties. If the object is invalid, an % exception is thrown + obj(i) = obj(i).check_combo_arg(); end if nobj > 1 diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m index 0420901a61..a83e510b0f 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m @@ -45,6 +45,7 @@ obj.baseclass,disp2str(non_type_ind),class(invalid_obj)) end end -if do_rehashify - obj = obj.rehashify_all(with_checks); -end +% not doing rehashify here as hashes are now loaded from a saved object +%if do_rehashify +% obj = obj.rehashify_all(with_checks); +%end diff --git a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m index b3e6299988..0c8e5faef4 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m +++ b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m @@ -195,6 +195,17 @@ error('HERBERT:unique_objects_container:invalid_set', ... 'attempt to set unique objects in container outside of loadobj'); end + %--{ + if ~isempty(self.stored_hashes_) + for ii = 1:numel(self.stored_hashes_) + hash = self.stored_hashes_{ii}; + hash2 = Hashing.hashify_obj(self.unique_objects_{ii}); + if hash ~= hash2 + error("bad hash"); + end + end + end + %--} end % function x = get.stored_hashes(self) @@ -202,6 +213,31 @@ % objects. Only really useful for debugging. x = self.stored_hashes_; end + + function self = set.stored_hashes(self, val) + %GET.STORED_HASHES - list the hashes corresponding to the unique + % objects. Only really useful for debugging. + if ~self.do_check_combo_arg_ + if ~iscell(val) + val = {val}; + end + self.stored_hashes_ = val; + else + error('HERBERT:unique_objects_container:invalid_set', ... + 'attempt to set stored hashes in container outside of loadobj'); + end + %--{ + if ~isempty(self.unique_objects_) + for ii = 1:numel(self.unique_objects) + obj = self.unique_objects_{ii}; + hash = Hashing.hashify_obj(obj); + if hash ~= self.stored_hashes_{ii} + error("bad hash"); + end + end + end + %--} + end % function x = get.idx(self) %GET.IDX - get the indices of each stored object in the container @@ -795,6 +831,7 @@ function list(self,field) fields_to_save_ = { 'baseclass', ... 'unique_objects',... + 'stored_hashes', ... 'idx', ... 'conv_func_string'}; end @@ -847,9 +884,11 @@ function list(self,field) % save-able class obj = unique_objects_container(); obj = loadobj@serializable(S,obj); - if obj.do_check_combo_arg - obj.check_combo_arg(true,true); - end + % not doing a check combo arg here as it is done in the loadobj + % of serializable above + %if obj.do_check_combo_arg + % obj.check_combo_arg(true,true); + %end end function out = concatenate(objs, type) From 12792e4c7746d31b95f08d8f4e17b363e54d6b65 Mon Sep 17 00:00:00 2001 From: Chris Marooney Date: Thu, 12 Dec 2024 17:12:21 +0000 Subject: [PATCH 05/21] fix required hashing --- .../@unique_objects_container/private/check_combo_arg_.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m index a83e510b0f..890704ff25 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m @@ -49,3 +49,6 @@ %if do_rehashify % obj = obj.rehashify_all(with_checks); %end +if isempty(obj.stored_hashes_) && ~isempty(obj.unique_objects) + obj = obj.rehashify_all(with_checks); +end From 6208c9fa7303e079cd35dd234507d8507cc064ca Mon Sep 17 00:00:00 2001 From: abuts Date: Fri, 13 Dec 2024 12:37:25 +0000 Subject: [PATCH 06/21] Re #1788 suggested hashable implementation (incomplete) --- herbert_core/utilities/classes/@hashable/eq.m | 28 ++++ .../utilities/classes/@hashable/hashable.m | 125 ++++++++++++++++++ herbert_core/utilities/classes/@hashable/ne.m | 32 +++++ .../utilities/classes/@hashable/private/eq_.m | 44 ++++++ .../@hashable/private/to_hashable_array_.m | 33 +++++ .../private/add_single_.m | 2 +- .../unique_objects_container.m | 95 ++----------- herbert_core/utilities/misc/build_hash.m | 53 ++++++++ 8 files changed, 327 insertions(+), 85 deletions(-) create mode 100644 herbert_core/utilities/classes/@hashable/eq.m create mode 100644 herbert_core/utilities/classes/@hashable/hashable.m create mode 100644 herbert_core/utilities/classes/@hashable/ne.m create mode 100644 herbert_core/utilities/classes/@hashable/private/eq_.m create mode 100644 herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m create mode 100644 herbert_core/utilities/misc/build_hash.m diff --git a/herbert_core/utilities/classes/@hashable/eq.m b/herbert_core/utilities/classes/@hashable/eq.m new file mode 100644 index 0000000000..90097930f4 --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/eq.m @@ -0,0 +1,28 @@ +function [iseq, mess] = eq (obj1, obj2, varargin) +% Return a logical variable stating if two serializable objects are equal or not +% +% >> [iseq, mess] = eq (obj1, obj2) +% >> [iseq, mess] = eq (obj1, obj2, p1, p2, ...) +% +% Input: +% ------ +% obj1 Object on left-hand side +% +% obj2 Object on right-hand side +% +% Optional: +% p1, p2,... Any set of parameters that the equal_to_tol function accepts +% +% See also equal_to_tol + + +names = cell(2,1); +if nargout == 2 + names{1} = inputname(1); + names{2} = inputname(2); + [iseq, mess] = eq_ (obj1, obj2, nargout, names, varargin{:}); +else + iseq = eq_ (obj1, obj2, nargout, names, varargin{:}); +end + +end diff --git a/herbert_core/utilities/classes/@hashable/hashable.m b/herbert_core/utilities/classes/@hashable/hashable.m new file mode 100644 index 0000000000..b733e1776d --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/hashable.m @@ -0,0 +1,125 @@ +classdef hashable < serializable + % + + properties (Access=protected) + hash_value_ = [] + end + + %--------------------------------------------------------------------------- + % Constructor + %--------------------------------------------------------------------------- + methods + function obj = hashable() + % Class constructor. + % Does nothing except enable methods of the base serializable class + % to be accessed. + end + end + + %--------------------------------------------------------------------------- + % INTERFACE + %--------------------------------------------------------------------------- + % Convert object or array of objects to/from a structure + %--------------------------------------------------------------------------- + methods + function flds = hashingFields (obj) + % function provides set of fields which define hash. By + % default, eqyak ti saveableFields, but different to give + % possibility to overload + flds = obj.saveableFields(); + end + + function S = to_struct (obj) + % overload to_struct to add hash to it if hash was available + S = to_struct@serializable(obj); + if ~isempty(obj.hash_value_) + S.hash_value = obj.hash_value_; + end + end + function [obj,bytestream] = to_hashable_array(obj) + % Function which extracts distignuishable information from the + % object to use as basis for the hash which describes this + % object. + [obj,bytestream] = to_hashable_array_(obj); + end + + + function [obj,hash] = build_hash(obj) + % calculate hash if it not available + if ~isempty(obj.hash_value_) + hash = obj.hash_value_; + return; + end + [obj,bytestream] = to_hashable_array(obj); + use_mex = config_store.instance().get_value('hor_config','use_mex'); + persistent Engine; + % In case the java engine is going to be used, initialise it as + % a persistent object + + if use_mex + % mex version to be used, use it + hash = GetMD5(bytestream); + else + + if isempty(Engine) + Engine = java.security.MessageDigest.getInstance('MD5'); + end + + + % mex version not to be used, manually construct from the + % Java engine + Engine.update(bytestream); + hash0 = Engine.digest; + + %using the following typecast to remedy that dec2hex + %does not work with negative numbers before Matlab 2020b. + %the typecast moves negative numbers to twos-complement + %positive representation, as is automatically done by the + %later dec2hex + hash1 = typecast(hash0,'uint8'); + + hash2 = dec2hex(hash1); + hash3 = cellstr(hash2); + hash4 = horzcat(hash3{:}); + hash = lower(hash4); % reduces hash ! + end + end + end + + methods (Static) + function obj = from_struct (S, varargin) + % overload from_struct to restore hash if available + obj = serializable.from_struct(S,varargin{:}); + if isfield(S,'hash_value') + obj.hash_value_ = S.hash_value; + end + end + end + + + %--------------------------------------------------------------------------- + % Testing equality of serializable objects + %--------------------------------------------------------------------------- + methods + % Return logical variable stating if two serializable objects are equal + % or not + [iseq, mess] = eq (obj1, obj2, varargin) + + % Return logical variable stating if two serializable objects are + % unequal or not + [isne, mess] = ne (obj1, obj2, varargin) + end + + %--------------------------------------------------------------------------- + % Object validation + %--------------------------------------------------------------------------- + methods + function obj = check_combo_arg (obj) + % overload check_combo_arg. Normally arguments have changed + % so existihg hashes should be destroyed. If they are not, + % overload this function for your class appropriately + obj = check_combo_arg@serializable(obj); + obj.hash_value_ = []; + end + end +end diff --git a/herbert_core/utilities/classes/@hashable/ne.m b/herbert_core/utilities/classes/@hashable/ne.m new file mode 100644 index 0000000000..cba336a43d --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/ne.m @@ -0,0 +1,32 @@ +function [isne, mess] = ne (obj1, obj2, varargin) +% Return a logical variable stating if two serializable objects are unequal or not +% +% >> [iseq, mess] = ne (obj1, obj2) +% >> [iseq, mess] = ne (obj1, obj2, p1, p2, ...) +% +% Input: +% ------ +% obj1 Object on left-hand side +% +% obj2 Object on right-hand side +% +% Optional: +% p1, p2,... Any set of parameters that the equal_to_tol function accepts +% +% See also equal_to_tol + +% TODO: can be done more efficiently as eq needs to check all +% the fields and ne may return when found first non-equal field + + +names = cell(2,1); +if nargout == 2 + names{1} = inputname(1); + names{2} = inputname(2); + [iseq, mess] = eq_ (obj1, obj2, nargout, names, varargin{:}); +else + iseq = eq_ (obj1, obj2, nargout, names, varargin{:}); +end +isne = ~iseq; + +end diff --git a/herbert_core/utilities/classes/@hashable/private/eq_.m b/herbert_core/utilities/classes/@hashable/private/eq_.m new file mode 100644 index 0000000000..4ec47ea1b2 --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/private/eq_.m @@ -0,0 +1,44 @@ +function [iseq, mess] = eq_ (obj1, obj2, narg_out, names, varargin) +% Check equality of two serializable objects + +[iseq, mess, name_a, name_b, namer, argi] = obj1.process_inputs_for_eq (obj2, ... + narg_out, names, varargin{:}); +if ~iseq + return +end + +iseq = false(numel(obj1),1); +for i=1:numel(obj1) + if nargout == 2 + name_1 = namer(name_a,i); + name_2 = namer(name_b,i); + [iseq(i), mess{i}] = eq_single (obj1(i), obj2(i), ... + 'name_a', name_1, 'name_b', name_2, argi{:}); + else + iseq(i) = eq_single(obj1(i),obj2(i), ... + 'name_a', name_a, 'name_b', name_b, argi{:}); + end +end + +if narg_out > 1 + if any(~iseq) + mess = strjoin(mess,'; '); + else + mess = ''; + end +end + +end + + +%------------------------------------------------------------------------------- +function [iseq, mess] = eq_single (obj1, obj2, ... + name_a, name_a_val, name_b, name_b_val, varargin) +% Compare single pair of serializeble objects + +struc1 = obj1.to_bare_struct(); +struc2 = obj2.to_bare_struct(); +[iseq,mess] = equal_to_tol (struc1, struc2, ... + name_a, name_a_val, name_b, name_b_val, varargin{:}); + +end diff --git a/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m b/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m new file mode 100644 index 0000000000..952390e037 --- /dev/null +++ b/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m @@ -0,0 +1,33 @@ +function [obj,arr] = to_hashable_array_ (obj) +% Retrieve information specifying +% +% >> S = to_hashable_array_(obj) +% +% Input: +% ------ +% obj Object or array of objects which are hashable +% +% +% Output: +% ------- +% arr array of unique data -- basis for building unique object +% hash + + +% Get saveable fields +field_names = hashingFields (obj(1)); + +% Recursively turn serializable fields into structures +cell_dat = cell (numel(field_names), numel(obj)); +for j = 1:numel(obj) + obj_tmp = obj(j); % get pointer to jth object to save expensive indexing + for i = 1:numel(field_names) + field_name = field_names{i}; + val = obj_tmp.(field_name); + if isa(val,'hashable') + [val,hash] = build_hash(val); + else + end + cell_dat{i,j} = val; + end +end diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m b/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m index 6d8adba10e..8f4692b8c6 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/add_single_.m @@ -30,7 +30,7 @@ % if ix and hash are not specified, call find_in_container to get them if nargin<=2 - [ix,hash] = self.find_in_container(obj); + [ix,hash,obj] = self.find_in_container(obj); end % If the object is not in the container. diff --git a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m index 0c8e5faef4..3dfac7710b 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m +++ b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m @@ -89,12 +89,6 @@ baseclass_ = ''; % if not empty, name of the baseclass suitable for isa calls % (default respecified in constructor inputParser) n_duplicates_ = zeros(1,0); - - % hashify defaults to this undocumented java function handle - % if you try to store objects non-children for serializable and - % the function to convert objects to bytes has not been set - % explicitly - convert_to_stream_f_ = @getByteStreamFromArray; end properties(Access = private) % is set to true if we decide not to use default stream conversion @@ -135,24 +129,6 @@ % Dependent properties set/get functions and subsrefs/subsassign % methods methods - function val = get.conv_func_string(obj) - %GET.CONV_STRING_FUNCTION - report the function handle used to - % generate hashes - val = func2str(obj.convert_to_stream_f_); - end - function obj = set.conv_func_string(obj,val) - %SET.CONV_STRING_FUNCTION - set the function handle used to - % generate hashes. Should only be used with loadobj. After the - % container is created, changing this functionmid-stream may - % invalidate the comparison for unique objects. - - if ~(ischar(val)||isstring(val)) - error('HERBERT:unique_obj_container:invalid_argument',... - 'convert_to_stream_f_string must be a string convertable to function. It is %s',... - class(val)) - end - obj.convert_to_stream_f = str2func(val); - end % function x = expose_unique_objects(self) %EXPOSE_UNIQUE_OBJECTS - return the cell array containing the @@ -199,7 +175,7 @@ if ~isempty(self.stored_hashes_) for ii = 1:numel(self.stored_hashes_) hash = self.stored_hashes_{ii}; - hash2 = Hashing.hashify_obj(self.unique_objects_{ii}); + [self.unique_objects_{ii},hash2] = build_hash(self.unique_objects_{ii}); if hash ~= hash2 error("bad hash"); end @@ -230,8 +206,8 @@ if ~isempty(self.unique_objects_) for ii = 1:numel(self.unique_objects) obj = self.unique_objects_{ii}; - hash = Hashing.hashify_obj(obj); - if hash ~= self.stored_hashes_{ii} + [obj,hash2] = build_hash(obj); + if hash ~= hash2 error("bad hash"); end end @@ -293,28 +269,6 @@ end end % - function x = get.convert_to_stream_f(self) - %GET.CONVERT_TO_STREAM - retrieve the hashing function - x = self.convert_to_stream_f_; - end - function self = set.convert_to_stream_f(self,val) - %SET.CONVERT_TO_STREAM - (re)set the hashing function - % This may invalidate the contents by changing the hashing function - % so should not be used if the container is not empty - if ~(isempty(val)|| isa(val,'function_handle')) - error('HERBERT:unique_objects_container:invalid_argument',... - 'this method accepts function handles for serializing objects only') - end - if isequal(self.convert_to_stream_f_,val) - return; - end - self.convert_to_stream_f_ = val; - self.non_default_f_conversion_set_ = true; - if self.do_check_combo_arg_ - self = self.check_combo_arg(true,false); - end - end - % function x = get.n_duplicates(self) x = self.n_duplicates_; end @@ -446,28 +400,6 @@ end end - function hash = hashify(self,obj,reset_count) - % makes a hash from the argument object - % which will be unique to any identical object - % - % Input: - % - obj : object to be hashed - % - reset_count : if counting hash calcs is exposed in - % hashify_obj then this resets the count - % Output: - % - hash : the resulting has, a row vector of uint8's - % - % this method started out as an instance method but now - % contains no references to self. For simplicity, keeping the - % original method (this one) as a wrapper to the static method - % now used => no other code changes - if nargin<=2 - hash = Hashing.hashify_obj(obj); - else - hash = Hashing.hashify_obj(obj,reset_count); - end - end - function self = rehashify_all(self,with_checks) % recalculate hashes of all objects, stored in the container % @@ -479,7 +411,7 @@ end self.stored_hashes_ =cell(1,self.n_unique); for i=1:self.n_unique - hash = self.hashify(self.unique_objects{i}); + [self.unique_objects{i},hash] = build_hash(self.unique_objects{i}); if with_checks is = ismember(hash,self.stored_hashes_(1:i-1)); if is @@ -493,7 +425,7 @@ end methods - function [ix, hash] = find_in_container(self,obj) + function [ix, hash,obj] = find_in_container(self,obj) %FIND_IN_CONTAINER Finds if obj is contained in self % Input: % - obj : the object which may or may not be uniquely contained @@ -503,7 +435,7 @@ % if it is stored, otherwise empty [] % - hash : the hash of the object from hashify % - hash = self.hashify(obj); + [obj,hash] = build_hash(obj); if isempty(self.stored_hashes_) ix = []; % object not stored as nothing is stored else @@ -528,9 +460,6 @@ % conversion for hashify flds = self.saveableFields(); - flds = [flds(:);'convert_to_stream_f']; % convert_to_stream - % function is not present in saveable properties but may be present - % as input too. % standard serializable constructor self = self.set_positional_and_key_val_arguments(... flds,false,varargin{:}); @@ -648,7 +577,7 @@ % check if you're trying to replace an object with an identical % one. If so silently return. - objhash = self.hashify(obj); + [obj,objhash] = build_hash(obj); curhash = self.stored_hashes_{self.idx_(nuix)}; if isequal(objhash, curhash) return; @@ -665,7 +594,7 @@ % returned as the index to the object in the container. % hash is returned as the hash of the object. If ix is empty % then the object is not in the container. - [ix,hash] = self.find_in_container(obj); + [ix,hash,obj] = self.find_in_container(obj); % If the object is not in the container. % store the hash in the stored hashes @@ -776,9 +705,7 @@ function list(self,field) % This is only used for tests and so its efficiency is not % important. newself = unique_objects_container('baseclass',self.baseclass);%,'convert_to_stream_f',self.convert_to_stream_f'); - % It is unclear why specifying convert_to_stream_f in the - % constructor causes a failure but it works if specified next. - newself.convert_to_stream_f = self.convert_to_stream_f; + for i=1:self.n_objects newself = newself.add(self.get(i)); end @@ -929,7 +856,7 @@ function list(self,field) out_obj = objs{ii}.get(jj); out_hsh = objs{ii}.hash(jj); [~,index] = ismember(out_hsh, out.stored_hashes_); - if index==0, index = []; end; + if index==0, index = []; end out = out.add_single_(out_obj,index,out_hsh); %( objs{ii}.get(jj) ); end end @@ -940,7 +867,7 @@ function list(self,field) out_obj = objs(ii).get(jj); out_hsh = objs(ii).hash(jj); [~,index] = ismember(out_hsh, out.stored_hashes_); - if index==0, index = [], end; + if index==0, index = []; end out = out.add_single_(out_obj,index,out_hsh); %( objs{ii}.get(jj) ); %out = out.add( objs(ii).get(jj) ); diff --git a/herbert_core/utilities/misc/build_hash.m b/herbert_core/utilities/misc/build_hash.m new file mode 100644 index 0000000000..b3df2ac60f --- /dev/null +++ b/herbert_core/utilities/misc/build_hash.m @@ -0,0 +1,53 @@ +function [obj,hash] = build_hash(obj) +% makes a hash from the argument object which will be unique +% when generated from any identical object +% +% Input: +% obj -- object to be hashed +% +% Output: +% obj -- unchanged object. Present here to maintain interface, common +% with hashable and other +% hash -- the resulting hash, a row vector of uint8's +% + +% get config to use use_mex +use_mex = config_store.instance().get_value('hor_config','use_mex'); +persistent Engine; +% In case the java engine is going to be used, initialise it as +% a persistent object +if isa(obj,'serializable') + bytestream = (obj.serialize()); +else + bytestream = serialize(obj); +end + +if use_mex + % mex version to be used, use it + hash = GetMD5(bytestream); +else + + if isempty(Engine) + Engine = java.security.MessageDigest.getInstance('MD5'); + end + + + % mex version not to be used, manually construct from the + % Java engine + Engine.update(bytestream); + hash0 = Engine.digest; + + %using the following typecast to remedy that dec2hex + %does not work with negative numbers before Matlab 2020b. + %the typecast moves negative numbers to twos-complement + %positive representation, as is automatically done by the + %later dec2hex + hash1 = typecast(hash0,'uint8'); + + hash2 = dec2hex(hash1); + hash3 = cellstr(hash2); + hash4 = horzcat(hash3{:}); + hash = lower(hash4); % reduces hash value to at least 64 bits! TODO: + % validate if it is possible to use +end +end From 047f73dde3b4389a3bf518955ac9dcf68a9ea677 Mon Sep 17 00:00:00 2001 From: abuts Date: Fri, 13 Dec 2024 16:34:01 +0000 Subject: [PATCH 07/21] Re #1788 Completed but not tested hashable. Unique_objects_container ready for hashable. --- .../store2020_1.mat | Bin 2888 -> 2531 bytes .../test_unique_objects.m | 46 ++----- herbert_core/utilities/classes/@hashable/eq.m | 20 +-- .../utilities/classes/@hashable/hashable.m | 75 ++++------- .../@hashable/private/to_hashable_array_.m | 29 +++- .../classes/@serializable/serializable.m | 11 -- .../private/check_combo_arg_.m | 7 +- .../private/contains_.m | 4 +- .../unique_objects_container.m | 126 +++++------------- .../unique_references_container.m | 101 +++++++------- herbert_core/utilities/misc/build_hash.m | 14 +- .../@Experiment/private/check_combo_arg_.m | 2 +- .../private/check_lattice_defined_.m | 2 +- .../sqw/@Experiment/private/get_inst_class_.m | 2 +- .../sqw/@Experiment/private/get_mod_pulse_.m | 2 +- .../file_io/@faccess_sqw_v3/faccess_sqw_v3.m | 4 +- 16 files changed, 163 insertions(+), 282 deletions(-) diff --git a/_test/test_unique_objects_container/store2020_1.mat b/_test/test_unique_objects_container/store2020_1.mat index 0986498cef3edff20b9707e1bc6eab2d7dbb4cbd..b8db644eda61c8d310738d0ce1e14b35dd89da39 100644 GIT binary patch delta 2354 zcmV-23C;G%7UL6;G#Ey5X&^*pV<0gzATc&NGB`RgH6SuDGBlA6n^{THoK5p4wNDilo3HeNNGS6NNjR}uo4FmS8$!>o$>5CH#;-TcrLln z1cYcPsE}xoC=o)4Dng>k9{^4M00dM>G@(mD!Sjs05$~XKVA_ zZEa`Au=BL)UzzBGHPJ_u<=VQ4^@% z-lMsbiO&U5cC&gCQHKYANPz_`9->k-q6v1q$%T+%K>L$OU;u|c{uZS^c9G3Kf(S$B zr?)5AE8&R7IZ-j<&ThaGvha&DIGFpk1JnDsc1WpPxMAU@gPqe>+f`kDk;YHIEmsvkHG>#mkJA&fHm~WU5s9M!v*!Sr94G zB=+%&_AEcEdY=6A`Xq(QdXYm_>Yk4KJn`0k>3E;b<6WA6_GQGoihk>(FE&5>?qX$q zGY+QytT?ow90!T{u0!J}*MIAX@f%Pc|2&r9t509P_v^VHSTYZvGZr6-@nG}9c%129 zK7@}Jlh3Si7meeo(!cBV?6}+Wrp25$J+4eLRq~Bock|?%gL1}k^TCrAe~Hbv(}m-> zKtA~P?+06dgBus_SJnSK`sBf%KmPRkGW4sigX5k%>k8ik>zpnf)8lP13@=x+IaK>_RVpJ&}|Sn-m@zS}&mb@$89QvRvDZZf(5 zapmzpJoo+e-|vWp#(&5kRUZGP&FB95^1kWE)!M!waBMaw&mHKXgKm7CU;nyDu|I1(C{af%uY1y9cD|zqftwcE9_%-w_dgr#Bq=y z&S%|%61nLxJ2}OH;DCe|z-~&Pw<~y5?>(@1JXjJ9gNzD9C}x}Sd;P}7yY$(y^ih53 zmt`nc_$veNIq<$QZrgv)T0gS%9clZ2W&7zHbJg#z~rek)3j#{7QU|d)IZ8| z7W+5zQk`{^7jqw6ehNU{Pq+9j-ozgp_&p16`h9$b^o_~ zP>z`>7>0E-BFs_(gE-6O{9-Q6Ld=lJSbcelXsAY~Wvqh{!AZzrdFNYlRh!*^(@PWQ zC?O$?SeC-&VUa<{Y9gjkz8FI;(}{98%%svdIFo@tJ;|U^u8 zD|tgL)6PT2eqCN=OC`@Qdz;4x2+-EK%Tp((How-jW6tTTD#$*x;YjknE%ac7jZ8w~>{^h^j;k!6* zP5k6)(9Tbudrkag{)J|aG8B@6TLOZDog@1);RzAY_O)`5=O`d_G9O;wA+b#8CsS)M zkL5>(4L>D9V7B$=Exw2NwsE0LyiR)W#3VSQnAf7Lz-g{B@LG^{hQlU74)i>5R3{Mh_X(#Nq*R-E>utwWi0(KdFlV&ufF4N<9z^3-Un;rZi9T$!I`+Ulk<+-ANMIY0&3Lc&UCl?S0OIpzcDKeg{C=$bMOEQl{^Baw zxT-a-QjM!l)3sl9Q70IKZtD(rLI3Iw*G8QA3jhHB|6^i+09GL80AfZY3=wBx5P|Y5 zfVea-v#>NZK0hfdHMyiX9;~LgB)=#%B|amuI3pD*#E_X%0c9b7KvH6HYI06uaWRri zUVKVvK~83JVhLO;LrQ*paz<)$c6@SvZc=`HVo`cL10PUa1Bls>+==dP1|*ss^#(}l z*$AlT#G#J`O&|srfGkF+z7Ax4SnP{H(Ffr(FnapJ?Das>&xA!k Ydb(kS@|p6aAu9@hH?Tb1ES^%D3PYp6C2*?pHALa;#Dkf@PNIS&^TrZAxi&Xqtq|QD{#-^A@R+X!~K4KrP{x# z+FJO01qD{}$n4D#+#ypCuy>6v>J#VFYr2Ox$6V^M_I7 zv4o09R%z(T+J2$xa~60hfsl?+Ese6o#pfOkxJUdX8tJ--RE>*$f(95#nIParI0$oh zi5rau5uwQrk*qWk@OUa{C{2FAJuxIOWPE2ROd2_Z!WwE;h=ZT7_cIo{`vel|@eGN& zS+Yk9C)0onqU`4NG^Q?p50L^3S+avl@sOt2@irGih5_x5BY^?j3GiK<1=vM42MA&e zU69=wW50r9niNFEkh{AfOOcOYqgYzZ8Q}m$2ti;duxAlg^pwgUCisYfKN2AB%D!P1 z1o<>T7IJiyM1$MR6)8qD6qKVA_6tanI`atnHDn06cZa|Lopb|#D%pg@%i@5$IF^i6 z6z<_f$jM7Z!sv*gKVgKOm`}3+XP*>ZQjU$eXH%tV;Z+MiW8r5le96L>EqukoFHYfL z?(;58?)%0OrD@@og&hmGEqrbYA1xh=qm(C%IzSn}HdJHTGW1$TyJP4%M!VkrEu*gO zYP~j8Mdv}&2VL)fT6Q|dd)rPECVhRuuy5iH{D0Cjo}J#w&IO&L2Ut>;a2n(r6p`8k zs&APjq>PhUF4Y93cDoW~Y=ZAPlsxr(nmnH}c&;gI=XPo5nqj9s#g6H37b^3xF7=1b z$>Mch;cu*XnbFc2K97`4sY>6-tC&_xB4s+oK7QGr=W$hkFAj;KK2Ei=Uglbr;wR%i zOT4vTI^O4tcvq%<5%I30-~Q-}jnBTjTwC9agQ-6)4sEE$L1Mn^(j->jUB`^yg6jAe zv4mfJ`s)4PF06wkv+y}*@sSu0KtD=Gx&F&X@X=!OnKtgcaXeG{_xyD`?)JQCG3QN> zE0au}d?VL?-7NX$u$*zyeDI{jUt;qscJ4UNkq^H8`@z=W=B0zW`k%+2Jb3uyPp>aR zzwSCX>AAD2@O`k(>B=!}u7vWYAVG0gPzJ{r1_{b5g9hr(P+3cr#Z)&FFInt6(4$&+zy2cS-`eXYmxGUMkN@F+h3{|tepk#j{$Ko`+T*{p@!X$( zT)k3z{CV}YoVNVmZ62-P#9CuPypPH^w*DH{c9h-^je6(KYjF6vV{UKba$ik6Fx$S@ z_t%1Vfy=)RN;~uJ`{Uy0fXTyw{&s(Ro;+WlZ+~K+&icK&b^YD9`fo44Hx`xmO+OyB z{_=Z&abg4|eiphS zpW;RRpSAoS?eWpCn_nL1`^={9B<2I0X?d-bPwu@qmE_x_(D2g7P#EBEM zKyz$wJuZ4-X_2zID^XgKN|PRn{XxIcN!`>fqEXBUkyWp_pC?5$c!7f)cj+Mf8g+!Q1U;u)@{qTgT6cP zAO8E>$8Y}7{D1dX{QV95LuJ_$gKK*Q;S@!Xp_qUx!UauOPA8I#2c+Q1CoO$1*Z4_) z%Pr-Xe$x2&FCfdJ2vR0w2Y$=<`E9SA-QTRS+qP9AJ}0uE2|0P45fI6s z?b`okytiHVVZZltzjw`VKfTv%-#;?=kD$8m3hb>{Fhha_5fY+6Ge*^>!({f71X2JY z6HYK(Ah95dg5qmX1Fhe8Yy7J9c<%Ur^?7Y67$i(0$_U@h-*?}-{62jiIr?Z^#c7_J zHUF)}-}CVIt#Q}>4;=afN8bZuzkHb z#~1hl&&IQha2f<;a(Z!cbbg9Xj`6Im+@s^8@C@Pe@!15Qp(&Y;CwNAtXgoQ8J3U7y z6E`pU;>EUa1XgET$BX%s=amlEF!6Hf3{Aw;vFyTn0iz<1H7}ldV`PqUYB@q@dzjGY9 z8j5?OHgSa_iebo;7}ken0zK4!RNg?fFo8l{XZbKrl+q-4r4mg%OQ5>w;L1;3oHym_ z+{azX8yJ~g$r~D(?mal_SLKzql=AvHw%h&Lh5iSIrz?4n%$#eNn?GE;MwVTh{AlN$ zzBz&0@bxNuz2&`d*zi5)6NCQ>fd9;4DkNggnfsYyxo}Argw7Yz8C@NJ@BCyJr|q7T zwtw|kcl=$Pw9Acs`?^s+Lu?+^ zNYgA*$wQa7(P}D9J=JS}7-iQkAxOjZlquq-M%_Ni$psBwF(N8iR+2c?6?i4ddedQ( zBnS38&t0z3yNoScZ;h*6kqCwv3uN9W*`+>g(}&0y4HF)}le+X>@7KQbZ}R&9*nS^u z&39j3Onc4p;xmKmK8!CfTK(VUl&{M3Ws806%eM9VThPq!<-Y5G`7oA1{<53ht@#?h zKbG;L+rGYd@ilCG-5Ou3#@DCmIfilJSou|#o`5M_~6;{~n$YO<+T z0>)oof%W}%VEvYV^#1_>0RR83l0i$uFcinr>WXD3JBcUh!K*NK^we%J6dVrc!9$^> zHO@$z%rrgh8~73Ya>71}$&#=dx^)fZL90F;hy?=PgU?Oxs|(r;oKIkBeLu{=ctS) z(bIi2j2{R6K|gKpmt9v+tl|bA_u)d#eK~*g(bxK_<_qA<1yW9lvet_%hh=H}Tfd8z z{LU@Eq@E*iK3sA%;|#%K%xDoY2A3*c*b=X6#p{}+N?a^-lKbR!Va1VJwJk_1obzi; z0^`G>@hXBpDwNRl9c8>sGD> [iseq, mess] = eq (obj1, obj2) @@ -10,19 +10,7 @@ % % obj2 Object on right-hand side % -% Optional: -% p1, p2,... Any set of parameters that the equal_to_tol function accepts -% % See also equal_to_tol - - -names = cell(2,1); -if nargout == 2 - names{1} = inputname(1); - names{2} = inputname(2); - [iseq, mess] = eq_ (obj1, obj2, nargout, names, varargin{:}); -else - iseq = eq_ (obj1, obj2, nargout, names, varargin{:}); -end - -end +[~,hash1] = build_hash(obj1); +[~,hash2] = build_hash(obj2); +iseq = strcmp(hash1,hash2); diff --git a/herbert_core/utilities/classes/@hashable/hashable.m b/herbert_core/utilities/classes/@hashable/hashable.m index b733e1776d..3064ecf2e7 100644 --- a/herbert_core/utilities/classes/@hashable/hashable.m +++ b/herbert_core/utilities/classes/@hashable/hashable.m @@ -5,17 +5,6 @@ hash_value_ = [] end - %--------------------------------------------------------------------------- - % Constructor - %--------------------------------------------------------------------------- - methods - function obj = hashable() - % Class constructor. - % Does nothing except enable methods of the base serializable class - % to be accessed. - end - end - %--------------------------------------------------------------------------- % INTERFACE %--------------------------------------------------------------------------- @@ -24,8 +13,13 @@ methods function flds = hashingFields (obj) % function provides set of fields which define hash. By - % default, eqyak ti saveableFields, but different to give - % possibility to overload + % default, equal to saveableFields, but different to give + % possibility to overload. + % + % for example, you may want to store object creation date but + % want to consider two objects with different creation dates + % to be the same for data comparison purpose. + % flds = obj.saveableFields(); end @@ -44,45 +38,26 @@ end - function [obj,hash] = build_hash(obj) + function [obj,hash,is_calculated] = build_hash(obj) % calculate hash if it not available + % Inputs: + % obj -- hashable object + % Returns: + % obj -- hashable object with hash value stored inside + % hash -- value of hash, defining state of the object + % is_calculated + % -- if true, the hash value was calculated for the + % object. If false, object have hash, already attached + % to it so the only thing we did was restoring this + % hash. + is_calculated = false; if ~isempty(obj.hash_value_) hash = obj.hash_value_; return; end + is_calculated = true; [obj,bytestream] = to_hashable_array(obj); - use_mex = config_store.instance().get_value('hor_config','use_mex'); - persistent Engine; - % In case the java engine is going to be used, initialise it as - % a persistent object - - if use_mex - % mex version to be used, use it - hash = GetMD5(bytestream); - else - - if isempty(Engine) - Engine = java.security.MessageDigest.getInstance('MD5'); - end - - - % mex version not to be used, manually construct from the - % Java engine - Engine.update(bytestream); - hash0 = Engine.digest; - - %using the following typecast to remedy that dec2hex - %does not work with negative numbers before Matlab 2020b. - %the typecast moves negative numbers to twos-complement - %positive representation, as is automatically done by the - %later dec2hex - hash1 = typecast(hash0,'uint8'); - - hash2 = dec2hex(hash1); - hash3 = cellstr(hash2); - hash4 = horzcat(hash3{:}); - hash = lower(hash4); % reduces hash ! - end + [~,hash] = build_hash(bytestream); end end @@ -98,16 +73,18 @@ %--------------------------------------------------------------------------- - % Testing equality of serializable objects + % Testing equality of hashable objects %--------------------------------------------------------------------------- methods % Return logical variable stating if two serializable objects are equal % or not - [iseq, mess] = eq (obj1, obj2, varargin) + iseq = eq (obj1, obj2) % Return logical variable stating if two serializable objects are % unequal or not - [isne, mess] = ne (obj1, obj2, varargin) + function isne = ne (obj1, obj2) + isne = ~eq(obj1,obj2); + end end %--------------------------------------------------------------------------- diff --git a/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m b/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m index 952390e037..655707d108 100644 --- a/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m +++ b/herbert_core/utilities/classes/@hashable/private/to_hashable_array_.m @@ -1,5 +1,5 @@ function [obj,arr] = to_hashable_array_ (obj) -% Retrieve information specifying +% Retrieve information specifying % % >> S = to_hashable_array_(obj) % @@ -14,20 +14,39 @@ % hash -% Get saveable fields +% Get hasing fields, used for extracting values, explicitly specifying +% class state field_names = hashingFields (obj(1)); % Recursively turn serializable fields into structures -cell_dat = cell (numel(field_names), numel(obj)); +arr = cell (1,numel(field_names)*numel(obj)); +ic = 0; for j = 1:numel(obj) obj_tmp = obj(j); % get pointer to jth object to save expensive indexing for i = 1:numel(field_names) + ic = ic+1; field_name = field_names{i}; val = obj_tmp.(field_name); if isa(val,'hashable') - [val,hash] = build_hash(val); + [val,hash,new_hash] = build_hash(val); + if new_hash + obj(j).(field_name) = val; + end + arr{ic} = typecast(hash,'uint8'); + elseif isdouble(val) + arr{ic} = typecast(single(round(val,7)),'uint8'); + elseif issingle(val) + arr{ic} = typecast(single(round(val,6)),'uint8'); + elseif isnumeric(val)||islogical(val) + arr{ic} = typecast(val,'uint8'); + elseif istext(val) + arr{ic} = uint8(char(val)); + elseif isstruct(val) + [val,arr{ic}] = build_hash(val); + obj(j).(field_name) = val; else + arr{ic}= serialize(val); end - cell_dat{i,j} = val; end end +arr = [arr(:)]; \ No newline at end of file diff --git a/herbert_core/utilities/classes/@serializable/serializable.m b/herbert_core/utilities/classes/@serializable/serializable.m index 873bc6500e..b417bf3b9c 100644 --- a/herbert_core/utilities/classes/@serializable/serializable.m +++ b/herbert_core/utilities/classes/@serializable/serializable.m @@ -186,17 +186,6 @@ end - %--------------------------------------------------------------------------- - % Constructor - %--------------------------------------------------------------------------- - methods - function obj = serializable() - % Class constructor. - % Does nothing except enable methods of the base serializable class - % to be accessed. - end - end - %--------------------------------------------------------------------------- % ABSTRACT INTERFACE THAT MUST BE DEFINED IN CHILD CLASS %--------------------------------------------------------------------------- diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m index 890704ff25..474686eba1 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/check_combo_arg_.m @@ -1,4 +1,4 @@ -function obj = check_combo_arg_(obj,do_rehashify,with_checks) +function obj = check_combo_arg_(obj,with_checks) % runs after changing property or number of properties to check % the consistency of the changes against all other relevant % properties @@ -45,10 +45,7 @@ obj.baseclass,disp2str(non_type_ind),class(invalid_obj)) end end -% not doing rehashify here as hashes are now loaded from a saved object -%if do_rehashify -% obj = obj.rehashify_all(with_checks); -%end +% if isempty(obj.stored_hashes_) && ~isempty(obj.unique_objects) obj = obj.rehashify_all(with_checks); end diff --git a/herbert_core/utilities/classes/@unique_objects_container/private/contains_.m b/herbert_core/utilities/classes/@unique_objects_container/private/contains_.m index 01e40f087e..4e6f3cf1d8 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/private/contains_.m +++ b/herbert_core/utilities/classes/@unique_objects_container/private/contains_.m @@ -1,4 +1,4 @@ -function [is,unique_ind] = contains_(obj,template,nout) +function [is,unique_ind,template] = contains_(obj,template,nout) % check if the container has the objects of the class "template" % if the template is char, or the the object equal to template, if the % template is the object of the kind, stored in the container @@ -22,7 +22,7 @@ unique_ind = find(belongs); end else % compare against value - the_hash = obj.hashify(template); + [template,the_hash] = build_hash(template); if nout == 1 is = ismember(the_hash,obj.stored_hashes_); unique_ind = []; diff --git a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m index 3dfac7710b..d77aece001 100644 --- a/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m +++ b/herbert_core/utilities/classes/@unique_objects_container/unique_objects_container.m @@ -90,11 +90,6 @@ % (default respecified in constructor inputParser) n_duplicates_ = zeros(1,0); end - properties(Access = private) - % is set to true if we decide not to use default stream conversion - % function - non_default_f_conversion_set_ = false; - end properties (Dependent) n_objects; @@ -129,23 +124,10 @@ % Dependent properties set/get functions and subsrefs/subsassign % methods methods - % - function x = expose_unique_objects(self) - %EXPOSE_UNIQUE_OBJECTS - return the cell array containing the - % unique objects in this container. This provides the interface for - % using this functionality outside of saveobj. It is allowed so - % that users can scan the properties of this container without - % repeating the scan for many duplicates. Note that this does break - % the encapsulation of the class in some sense. - - x = self.unique_objects_; - end function x = get.unique_objects(self) %GET.UNIQUE_OBJECTS Return the cell array containing the unique % objects in this container. This access is solely for saveobj. - % Users wishing to do a scan by unique objects outside of saveobj - % should use expose_unique_objects(). x = self.unique_objects_; end @@ -162,26 +144,12 @@ % NB this set operation should only be done in environments such as % loadobj which disable combo arg checking - if ~self.do_check_combo_arg_ - if ~iscell(val) - val = {val}; - end - self.unique_objects_ = val; - else - error('HERBERT:unique_objects_container:invalid_set', ... - 'attempt to set unique objects in container outside of loadobj'); - end - %--{ - if ~isempty(self.stored_hashes_) - for ii = 1:numel(self.stored_hashes_) - hash = self.stored_hashes_{ii}; - [self.unique_objects_{ii},hash2] = build_hash(self.unique_objects_{ii}); - if hash ~= hash2 - error("bad hash"); - end - end + self.unique_objects_ = val; + check_existing = ~isempty(self.stored_hashes_)||(isempty(self.idx_)); + self.stored_hashes_ = {}; + if self.do_check_combo_arg_ + self = self.check_combo_arg(check_existing); end - %--} end % function x = get.stored_hashes(self) @@ -189,31 +157,7 @@ % objects. Only really useful for debugging. x = self.stored_hashes_; end - - function self = set.stored_hashes(self, val) - %GET.STORED_HASHES - list the hashes corresponding to the unique - % objects. Only really useful for debugging. - if ~self.do_check_combo_arg_ - if ~iscell(val) - val = {val}; - end - self.stored_hashes_ = val; - else - error('HERBERT:unique_objects_container:invalid_set', ... - 'attempt to set stored hashes in container outside of loadobj'); - end - %--{ - if ~isempty(self.unique_objects_) - for ii = 1:numel(self.unique_objects) - obj = self.unique_objects_{ii}; - [obj,hash2] = build_hash(obj); - if hash ~= hash2 - error("bad hash"); - end - end - end - %--} - end + % function x = get.idx(self) %GET.IDX - get the indices of each stored object in the container @@ -321,8 +265,8 @@ elseif nuix == numel(self.idx_)+1 if numel(idxstr)>1 error('HERBERT:unique_objects_container:invalid_subscript', ... - ['when adding to the end of a container, additionally setting ', ... - 'properties is not permitted']); + ['when adding to the end of a container, additionally setting ', ... + 'properties is not permitted']); end self = self.add(val); return; @@ -355,7 +299,7 @@ % OTHER %---------------------------------------------------------------------- methods - function [is,unique_ind] = contains(obj,value) + function [is,unique_ind,obj] = contains(obj,value) % check if the container has the objects of the class "value" % if the value is char, or the the object equal value, if the % value is the object of the kind, stored in container @@ -367,7 +311,8 @@ % unique_ind % -- if requested, the positions of the sample in the % unique objects container - [is,unique_ind] = contains_(obj,value,nargout); + % obj -- obj, which if hashable, contains calculated hash + [is,unique_ind,obj] = contains_(obj,value,nargout); end function obj = replicate_runs(obj,n_objects) @@ -399,7 +344,7 @@ newself = newself.add(item); end end - + function self = rehashify_all(self,with_checks) % recalculate hashes of all objects, stored in the container % @@ -411,7 +356,7 @@ end self.stored_hashes_ =cell(1,self.n_unique); for i=1:self.n_unique - [self.unique_objects{i},hash] = build_hash(self.unique_objects{i}); + [self.unique_objects_{i},hash] = build_hash(self.unique_objects{i}); if with_checks is = ismember(hash,self.stored_hashes_(1:i-1)); if is @@ -486,14 +431,14 @@ function n = get_nruns(self) %GET_NRUNS non-dependent-property form of n_runs % for use with arrayfun in object_lookup - + n = numel(self.idx_); end function n = runs_sz(self) %RUNS_SZ converts n_runs to the form of output from size % to put unique_objects_container on the same footing as % array/cell in object_lookup - + n = size(self.idx_); end @@ -710,14 +655,14 @@ function list(self,field) newself = newself.add(self.get(i)); end end - + function field_vals = get_unique_field(self, field) % determine type of unique_objects_container to make from the % object feld type, and construct the container s1 = self.get(1); v = s1.(field); field_vals = unique_objects_container(class(v)); - + % make container of possible field values from the unique contents of self % this will do the minimal hashing of the field values poss_field_vals = unique_objects_container(class(v)); @@ -727,7 +672,7 @@ function list(self,field) v = sii.(field); poss_field_vals = poss_field_vals.add_single_(v); end - + % make a container for the fields in order of the objects in self using the hash % info previously put into poss_field_vaues for ii=1:self.n_runs @@ -744,9 +689,9 @@ function list(self,field) end end end - + function val = hash(self,index) - % accessor for the stored hashes + % accessor for the stored hashes val = self.stored_hashes_{ self.idx_(index) }; end @@ -758,9 +703,7 @@ function list(self,field) fields_to_save_ = { 'baseclass', ... 'unique_objects',... - 'stored_hashes', ... - 'idx', ... - 'conv_func_string'}; + 'idx'}; end methods @@ -772,36 +715,27 @@ function list(self,field) ver = 1; end - function flds = saveableFields(obj) + function flds = saveableFields(~) % get independent fields, which fully define the state of the % serializable object. - if obj.non_default_f_conversion_set_ - flds = unique_objects_container.fields_to_save_; - else % do not store conversion function - flds = unique_objects_container.fields_to_save_(1:end-1); - end + flds = unique_objects_container.fields_to_save_; end - function obj = check_combo_arg(obj,do_rehashify,with_checks) + function obj = check_combo_arg(obj,with_checks) % runs after changing property or number of properties to check % the consistency of the changes against all other relevant % properties % % Inputs: - % do_rehashify -- if true, run rehashify procedure. If not - % provided, assumed true % with_checks -- if true, each following hash is compared with % the previous hashes and if coincedence found, % throw the error. Necessary when replacing the % unique_objects to check that new objects are % indeed unique. The default is false. if nargin == 1 - do_rehashify = true; - with_checks = false; - elseif nargin == 2 with_checks = false; end - obj = check_combo_arg_(obj,do_rehashify,with_checks); + obj = check_combo_arg_(obj,with_checks); end end @@ -817,7 +751,7 @@ function list(self,field) % obj.check_combo_arg(true,true); %end end - + function out = concatenate(objs, type) %CONCATENATE takes the unique_object and idx (index) arrays from % an array of one or more unique_object_containers and concatenates @@ -834,10 +768,10 @@ function list(self,field) % ------- % - out - single unique_objects_container combining the contents of % the elements of the input array objs - + if isempty(objs) error('HERBERT:unique_objects_container:invalid_input', ... - 'at least one object must be supplied to concatenate'); + 'at least one object must be supplied to concatenate'); end concat_cells = strcmp(type,'{}'); @@ -869,14 +803,14 @@ function list(self,field) [~,index] = ismember(out_hsh, out.stored_hashes_); if index==0, index = []; end out = out.add_single_(out_obj,index,out_hsh); %( objs{ii}.get(jj) ); - + %out = out.add( objs(ii).get(jj) ); end end end end end - + end % static methods end % classdef unique_objects_container diff --git a/herbert_core/utilities/classes/@unique_references_container/unique_references_container.m b/herbert_core/utilities/classes/@unique_references_container/unique_references_container.m index 55a01140b2..d961cdb05a 100644 --- a/herbert_core/utilities/classes/@unique_references_container/unique_references_container.m +++ b/herbert_core/utilities/classes/@unique_references_container/unique_references_container.m @@ -107,11 +107,9 @@ end properties (Dependent) - % saveable fields for save/loadobj stored_baseclass; global_name; % category name for singleton storage - unique_objects; % returns unique_objects_container % other dependent properties n_unique_objects; % number of unique_objects (without creating it) @@ -122,6 +120,9 @@ n_runs; % same as n_objects, provides a domain-specific interface % to the number of objects for SQW-Experiment % instruments and samples + + unique_objects; % returns unique_objects_container. Hidden not to + % expensive operation? end properties (Constant, Access=private) % serializable interface @@ -240,7 +241,7 @@ % obtain a unique_objects_container with the unique objects uoca = self.unique_objects; % convert it to a cell array for external use - uoca = uoca.expose_unique_objects(); + uoca = uoca.unique_objects; end function uoc = get.unique_objects(self) @@ -258,9 +259,9 @@ glc = self.global_container('value', self.global_name_); for ii=1:self.n_objects obj = self.get(ii); - hash = self.hash(ii); - [~,index] = ismember(hash,uoc.stored_hashes_); - if index==0, index=[]; end; + [obj,hash] = build_hash(obj); + [~,index] = ismember(hash,uoc.stored_hashes_); + if index==0, index=[]; end uoc = uoc.add_single_(obj,index,hash); end end @@ -287,17 +288,17 @@ error('HERBERT:unique_references_container:invalid_argument', ... 'set unique objects with wrong stored baseclass'); end - - % get global container + + % get global container glc = self.global_container('value',self.global_name); % reinitialise the indices of this container as empty - % to purge this container of existing contents + % to purge this container of existing contents self.idx_ = zeros(1,0); - + % repopulate container with contents of val for ii = 1:val.n_objects obj = val.get(ii); - hash = val.hash(ii); + [obj,hash] = build_hash(obj); [~,loc] = ismember( hash, glc.stored_hashes_ ); if loc == 0 || isempty(loc) self = self.add_single_( obj, [], hash ); @@ -306,8 +307,8 @@ end end if val.n_objects ~= self.n_objects - error('HORACE:unique_references_container___set.unique_objects:process_error', ... - 'number of objects added does not match'); + error('HORACE:unique_references_container:runtime_error', ... + 'number of objects added does not match'); end end @@ -366,27 +367,27 @@ end methods % overloaded indexers, subsets, find functions - + function field_vals = get_unique_field(self, field) - %GET_UNIQUE_FIELD each of the unique objects referred to by self - % should have a property named 'field'. The code below takes each of the - % referred objects in turn, extracts the object referred to by - % 'field' and stores it in the unique_OBJECTS_container field_vals - % created here. field_vals will then contain unique copies of all - % the values of 'field' within the objects referred to in self, indexed - % in the same way as the original referred objects. + %GET_UNIQUE_FIELD each of the unique objects referred to by self + % should have a property named 'field'. The code below takes each of the + % referred objects in turn, extracts the object referred to by + % 'field' and stores it in the unique_OBJECTS_container field_vals + % created here. field_vals will then contain unique copies of all + % the values of 'field' within the objects referred to in self, indexed + % in the same way as the original referred objects. % determine type of unique_references_container to make from the % object field type - s1 = self.get(1); + s1 = self.get(1); v = s1.(field); - + % initialise the final output container (a unique_references_container) % to hold the unique field value objects from objects in this container % which are of type class(v). global_name = ['GLOBAL_NAME_FIELD_',class(v)]; field_vals = unique_references_container(global_name,class(v)); - + % % get a list without duplicates of indices to the objects in self % these are the indices into the global container @@ -402,7 +403,7 @@ v = sii.(field); poss_field_vals = poss_field_vals.add_single_(v); end - + % now we construct the main unique_references_container `field_vals` of % all the field values within objects in self for ii=1:self.n_objects @@ -410,7 +411,7 @@ v = sii.(field); % get its field index = self.idx_(ii); % get its global index [~,loc] = ismember(index,uix); % find where it is in the list of - % unique indices for objects + % unique indices for objects hash = poss_field_vals.hash(loc); % find the hash at that location % find if that hash is in the hashes already in field % values and if so where it is @@ -420,14 +421,14 @@ % location if loc>0 field_vals = field_vals.add_single_(v,loc,hash); - % if we don't , add it without a location (it will go at - % the end as a new object/hash + % if we don't , add it without a location (it will go at + % the end as a new object/hash else field_vals = field_vals.add_single_(v,[],hash); end end end - + function varargout = subsref(self, idxstr) if numel(self)>1 % input is array or cell of unique_references_containers [varargout{1:nargout}] = builtin('subsref',self,idxstr); @@ -446,7 +447,7 @@ 'subscript must be less than %d',numel(self.idx_)+1); else error('HERBERT:unique_references_container:invalid_subscript',... - 'container is empty and cannot take a subscript'); + 'container is empty and cannot take a subscript'); end end glindex = self.idx_(b); @@ -501,8 +502,8 @@ elseif nuix == self.n_objects+1 if numel(idxstr)>1 error('HERBERT:unique_references_container:invalid_subscript', ... - ['when adding to the end of a container, additionally setting ', ... - 'properties is not permitted']); + ['when adding to the end of a container, additionally setting ', ... + 'properties is not permitted']); end self = self.add(val); return; @@ -548,10 +549,10 @@ glc = self.global_container('value',self.global_name_); val = glc{ self.idx(index) }; end - + function val = hash(self,index) - %HASH - for a given `index` into the container, return the associated hash - % this prevents the calling code from having to recalculate the hash since it is already known + %HASH - for a given `index` into the container, return the associated hash + % this prevents the calling code from having to recalculate the hash since it is already known glc = self.global_container('value',self.global_name_); val = glc.stored_hashes_{ self.idx(index) } ; end @@ -574,14 +575,14 @@ error('HERBERT:unique_references_container:incomplete_setup', ... 'global name unset'); end - + if nargin<=2 % have to recalculate the hash and the position of `inobj` in the global container as % this info is not in the additional arguments [glindex, hash] = self.global_container('value',self.global_name_).find_in_container(inobj); if numel(glindex)>1 error('HORACE:unique_references_container-add-single:invalid_state', ... - 'there should only be one index returned'); + 'there should only be one index returned'); end end if isempty(glindex) @@ -592,7 +593,7 @@ self.idx_ = [ self.idx(:)', glindex ]; nuix = numel(self.idx_); end - + function [self, nuix] = add_copies_(self,obj,n) %ADD_COPIES - add a single object obj at the end of the container % multiple times @@ -608,7 +609,7 @@ % self - the revised container with the additional indices for % the added objects % nuix - the range of added non-unique indices - + if isempty(self.stored_baseclass_) error('HERBERT:unique_references_container:incomplete_setup', ... 'stored baseclass unset'); @@ -748,15 +749,15 @@ if numel(glindex)>1 % unlikely error state but catching just in case error('HORACE:unique_references_container___replace:invalid_state', ... - 'more than one location index found'); + 'more than one location index found'); end if isempty(glindex) [glcont,glindex] = ... self.global_container('value',self.global_name_).add_single_(obj); if numel(glindex)>1 % again, unlikely error, but just in case - error('HORACE:unique_references_container___replace:invalid_state', ... - 'more than one location index found'); + error('HORACE:unique_references_container___replace:invalid_state', ... + 'more than one location index found'); end if glindex == 0 % object was not replaced @@ -766,7 +767,7 @@ end self.idx_(nuix) = glindex; end - + function [self] = replace_all(self,obj) %REPLACE_ALL - substitute object obj at all positions in container % @@ -778,7 +779,7 @@ % The old values are overwritten. [glindex, ~] = self.global_container('value',self.global_name_) ... - .find_in_container(obj); + .find_in_container(obj); if isempty(glindex) [glcont,glindex] = ... self.global_container('value',self.global_name_).add(obj); @@ -853,7 +854,7 @@ % arg3 is the baseclass % 'value' - return the container for the category % arg3 is not used - % 'list' - return struct of the global containers + % 'list' - return struct of the global containers % (internal access only) % 'reset' - change the container for the category as % it has been modified. Used because it is @@ -875,17 +876,17 @@ % If the global container does not exist, initialise it with no % categories - if isempty(glcontainer) + if isempty(glcontainer) glcontainer = struct(); elseif strcmpi(opflag,'CLEAR_ALL') glcontainer = struct(); return; end - + if strcmpi(opflag,'list') disp(glcontainer); return; - end + end % check minimum arguments if nargin<2 @@ -898,7 +899,7 @@ error('HERBERT:unique_references_container:invalid_argument', ... 'global container name is %s not char',glname); end - + % if the category has not yet been created anywhere % create a global container for it if ~isfield(glcontainer,glname) @@ -948,7 +949,7 @@ case 'reset' if nargin < 3 - error('HERBERT:unique_references_container:invalid argument', ... + error('HERBERT:unique_references_container:invalid_argument', ... 'missing arg3 == newcontainer'); end newcontainer = arg3; diff --git a/herbert_core/utilities/misc/build_hash.m b/herbert_core/utilities/misc/build_hash.m index b3df2ac60f..da7a42ed12 100644 --- a/herbert_core/utilities/misc/build_hash.m +++ b/herbert_core/utilities/misc/build_hash.m @@ -1,4 +1,4 @@ -function [obj,hash] = build_hash(obj) +function [obj,hash,is_calculated] = build_hash(obj) % makes a hash from the argument object which will be unique % when generated from any identical object % @@ -7,17 +7,20 @@ % % Output: % obj -- unchanged object. Present here to maintain interface, common -% with hashable and other +% with the same named method in hashable. % hash -- the resulting hash, a row vector of uint8's +% is_calsulated +% -- always true, kept to support common interface to hashable % +is_calculated = true; % get config to use use_mex use_mex = config_store.instance().get_value('hor_config','use_mex'); persistent Engine; % In case the java engine is going to be used, initialise it as % a persistent object -if isa(obj,'serializable') - bytestream = (obj.serialize()); +if isa(obj,'uint8') + bytestream = obj; else bytestream = serialize(obj); end @@ -48,6 +51,5 @@ hash3 = cellstr(hash2); hash4 = horzcat(hash3{:}); hash = lower(hash4); % reduces hash value to at least 64 bits! TODO: - % validate if it is possible to use -end + % validate if it is possible to use end diff --git a/horace_core/sqw/@Experiment/private/check_combo_arg_.m b/horace_core/sqw/@Experiment/private/check_combo_arg_.m index 283321545b..4169382284 100644 --- a/horace_core/sqw/@Experiment/private/check_combo_arg_.m +++ b/horace_core/sqw/@Experiment/private/check_combo_arg_.m @@ -66,7 +66,7 @@ % check if new lattice is defined % NB unique objects used to reduce check time. Should not be used to % replace values -new_uni_obj = obj.samples_.expose_unique_objects(); +new_uni_obj = obj.samples_.unique_objects; new_lat_def = cellfun(@(x)~isempty(x.alatt),new_uni_obj); new_ang_def = cellfun(@(x)~isempty(x.angdeg),new_uni_obj); % if new lattice not defined everywhere diff --git a/horace_core/sqw/@Experiment/private/check_lattice_defined_.m b/horace_core/sqw/@Experiment/private/check_lattice_defined_.m index b29052af6e..5124f1a1d0 100644 --- a/horace_core/sqw/@Experiment/private/check_lattice_defined_.m +++ b/horace_core/sqw/@Experiment/private/check_lattice_defined_.m @@ -7,7 +7,7 @@ end % unique_objects used to save time scanning. Do not use to replace. -old_uni_obj = obj.samples_.expose_unique_objects(); +old_uni_obj = obj.samples_.unique_objects; old_lat_def = cellfun(@(x)~isempty(x.alatt),old_uni_obj); old_ang_def = cellfun(@(x)~isempty(x.angdeg),old_uni_obj); if any(old_lat_def) && any(old_ang_def) diff --git a/horace_core/sqw/@Experiment/private/get_inst_class_.m b/horace_core/sqw/@Experiment/private/get_inst_class_.m index a33e3d340c..33e40d0f28 100644 --- a/horace_core/sqw/@Experiment/private/get_inst_class_.m +++ b/horace_core/sqw/@Experiment/private/get_inst_class_.m @@ -4,7 +4,7 @@ % % unique_objects cell used to minimse scanning time. Should not be used to replace. -inst = obj.instruments.expose_unique_objects(); +inst = obj.instruments.unique_objects; if isempty(inst) all_inst = false; inst = []; diff --git a/horace_core/sqw/@Experiment/private/get_mod_pulse_.m b/horace_core/sqw/@Experiment/private/get_mod_pulse_.m index 7edc487b59..57dedc2608 100644 --- a/horace_core/sqw/@Experiment/private/get_mod_pulse_.m +++ b/horace_core/sqw/@Experiment/private/get_mod_pulse_.m @@ -16,7 +16,7 @@ present = true; % unique_objects cell used to speed access. Should not be used to replace. -uni_inst = inst.expose_unique_objects(); +uni_inst = inst.unique_objects; try first_model = uni_inst{1}.moderator.pulse_model; catch ME diff --git a/horace_core/sqw/file_io/@faccess_sqw_v3/faccess_sqw_v3.m b/horace_core/sqw/file_io/@faccess_sqw_v3/faccess_sqw_v3.m index 4c6e56193c..e74d528653 100644 --- a/horace_core/sqw/file_io/@faccess_sqw_v3/faccess_sqw_v3.m +++ b/horace_core/sqw/file_io/@faccess_sqw_v3/faccess_sqw_v3.m @@ -125,9 +125,9 @@ % get instrument and sample data in the form they would be written % on hdd. % get_unique_instruments as a cell array; - instr = exp_info.instruments.expose_unique_objects(); + instr = exp_info.instruments.unique_objects; % get_unique_samples as a cell array; - sampl = exp_info.samples.expose_unique_objects(); + sampl = exp_info.samples.unique_objects; % convert to structs instr_str = cellfun(@(x)(x.to_struct()),instr,'UniformOutput',false); sampl_str = cellfun(@(x)(x.to_struct()),sampl,'UniformOutput',false); From 684c75b52e6ad5cab3e8a64af2dbb50adead1299 Mon Sep 17 00:00:00 2001 From: abuts Date: Fri, 13 Dec 2024 18:20:47 +0000 Subject: [PATCH 08/21] Re #1788 unique_objects_container ready for hashable --- .../store2020_1.mat | Bin 2531 -> 2531 bytes .../test_unique_objects_2.m | 2 +- .../test_unique_references.m | 96 +++--------------- .../utilities/classes/@hashable/hashable.m | 1 - .../unique_objects_container.m | 8 ++ .../unique_references_container.m | 82 +++++++-------- .../@Experiment/private/check_combo_arg_.m | 2 +- .../private/check_lattice_defined_.m | 2 +- 8 files changed, 63 insertions(+), 130 deletions(-) diff --git a/_test/test_unique_objects_container/store2020_1.mat b/_test/test_unique_objects_container/store2020_1.mat index b8db644eda61c8d310738d0ce1e14b35dd89da39..71d66f3fe5cc51953bea168ca78dcdec63c0ecd0 100644 GIT binary patch delta 29 kcmaDX{8)H`1CNE3p}Cczse+M#k;%kB<%tQb8%y#z0f(Ik!vFvP delta 29 kcmaDX{8)H`1CN=Nk%g6kse+M#k;%kB<%tQb8%y#z0f&bO!T