From e5ad801172faffb4cad3a7b860e5b6efa9b5fb46 Mon Sep 17 00:00:00 2001 From: Lawrence Niu Date: Fri, 26 May 2023 10:07:10 -0400 Subject: [PATCH 1/4] reformat parseDataset --- +io/parseDataset.m | 150 ++++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/+io/parseDataset.m b/+io/parseDataset.m index ba6b703d..ffc90fe6 100644 --- a/+io/parseDataset.m +++ b/+io/parseDataset.m @@ -1,85 +1,85 @@ function parsed = parseDataset(filename, info, fullpath, Blacklist) -%typed and untyped being container maps containing type and untyped datasets -% the maps store information regarding information and stored data -% NOTE, dataset name is in path format so we need to parse that out. -name = info.Name; + %typed and untyped being container maps containing type and untyped datasets + % the maps store information regarding information and stored data + % NOTE, dataset name is in path format so we need to parse that out. + name = info.Name; -%check if typed and parse attributes -[attrargs, Type] = io.parseAttributes(filename, info.Attributes, fullpath, Blacklist); + %check if typed and parse attributes + [attrargs, Type] = io.parseAttributes(filename, info.Attributes, fullpath, Blacklist); -fid = H5F.open(filename, 'H5F_ACC_RDONLY', 'H5P_DEFAULT'); -did = H5D.open(fid, fullpath); -props = attrargs; -datatype = info.Datatype; -dataspace = info.Dataspace; + fid = H5F.open(filename, 'H5F_ACC_RDONLY', 'H5P_DEFAULT'); + did = H5D.open(fid, fullpath); + props = attrargs; + datatype = info.Datatype; + dataspace = info.Dataspace; -parsed = containers.Map; -afields = keys(attrargs); -if ~isempty(afields) - anames = strcat(name, '_', afields); - parsed = [parsed; containers.Map(anames, attrargs.values(afields))]; -end + parsed = containers.Map; + afields = keys(attrargs); + if ~isempty(afields) + anames = strcat(name, '_', afields); + parsed = [parsed; containers.Map(anames, attrargs.values(afields))]; + end -% loading h5t references are required -% unfortunately also a bottleneck -if strcmp(datatype.Class, 'H5T_REFERENCE') - tid = H5D.get_type(did); - data = io.parseReference(did, tid, H5D.read(did)); - H5T.close(tid); -elseif ~strcmp(dataspace.Type, 'simple') - data = H5D.read(did); + % loading h5t references are required + % unfortunately also a bottleneck + if strcmp(datatype.Class, 'H5T_REFERENCE') + tid = H5D.get_type(did); + data = io.parseReference(did, tid, H5D.read(did)); + H5T.close(tid); + elseif ~strcmp(dataspace.Type, 'simple') + data = H5D.read(did); - switch datatype.Class - case 'H5T_STRING' - if datetime(version('-date')) < datetime('25-Feb-2020') - % MATLAB 2020a fixed string support for HDF5, making - % reading strings "consistent" with regular use. - data = data .'; - end - datadim = size(data); - if datadim(1) > 1 - %multidimensional strings should become cellstr - data = strtrim(mat2cell(data, ones(datadim(1), 1), datadim(2))); - end - case 'H5T_ENUM' - if io.isBool(datatype.Type) - data = strcmp('TRUE', data); - else - warning('NWB:Dataset:UnknownEnum', ... - ['Encountered unknown enum under field `%s` with %d members. ' ... - 'Will be saved as cell array of characters.'], ... - info.Name, length(datatype.Type.Member)); - end - end -else - sid = H5D.get_space(did); - pid = H5D.get_create_plist(did); - isChunked = H5P.get_layout(pid) == H5ML.get_constant_value('H5D_CHUNKED'); - - tid = H5D.get_type(did); - class_id = H5T.get_class(tid); - isNumeric = class_id == H5ML.get_constant_value('H5T_INTEGER')... - || class_id == H5ML.get_constant_value('H5T_FLOAT'); - if isChunked && isNumeric - data = types.untyped.DataPipe('filename', filename, 'path', fullpath); - elseif any(dataspace.Size == 0) - data = []; + switch datatype.Class + case 'H5T_STRING' + if datetime(version('-date')) < datetime('25-Feb-2020') + % MATLAB 2020a fixed string support for HDF5, making + % reading strings "consistent" with regular use. + data = data .'; + end + datadim = size(data); + if datadim(1) > 1 + %multidimensional strings should become cellstr + data = strtrim(mat2cell(data, ones(datadim(1), 1), datadim(2))); + end + case 'H5T_ENUM' + if io.isBool(datatype.Type) + data = strcmp('TRUE', data); + else + warning('NWB:Dataset:UnknownEnum', ... + ['Encountered unknown enum under field `%s` with %d members. ' ... + 'Will be saved as cell array of characters.'], ... + info.Name, length(datatype.Type.Member)); + end + end else - data = types.untyped.DataStub(filename, fullpath); + sid = H5D.get_space(did); + pid = H5D.get_create_plist(did); + isChunked = H5P.get_layout(pid) == H5ML.get_constant_value('H5D_CHUNKED'); + + tid = H5D.get_type(did); + class_id = H5T.get_class(tid); + isNumeric = class_id == H5ML.get_constant_value('H5T_INTEGER')... + || class_id == H5ML.get_constant_value('H5T_FLOAT'); + if isChunked && isNumeric + data = types.untyped.DataPipe('filename', filename, 'path', fullpath); + elseif any(dataspace.Size == 0) + data = []; + else + data = types.untyped.DataStub(filename, fullpath); + end + H5T.close(tid); + H5P.close(pid); + H5S.close(sid); end - H5T.close(tid); - H5P.close(pid); - H5S.close(sid); -end -if isempty(Type.typename) - %untyped group - parsed(name) = data; -else - props('data') = data; - kwargs = io.map2kwargs(props); - parsed = eval([Type.typename '(kwargs{:})']); -end -H5D.close(did); -H5F.close(fid); + if isempty(Type.typename) + %untyped group + parsed(name) = data; + else + props('data') = data; + kwargs = io.map2kwargs(props); + parsed = eval([Type.typename '(kwargs{:})']); + end + H5D.close(did); + H5F.close(fid); end \ No newline at end of file From c1026c375200644b0881bb2154e7465c3b6bef87 Mon Sep 17 00:00:00 2001 From: Lawrence Niu Date: Fri, 26 May 2023 10:08:23 -0400 Subject: [PATCH 2/4] Replace datestring with version number compare --- +io/parseDataset.m | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/+io/parseDataset.m b/+io/parseDataset.m index ffc90fe6..582419a3 100644 --- a/+io/parseDataset.m +++ b/+io/parseDataset.m @@ -31,7 +31,9 @@ switch datatype.Class case 'H5T_STRING' - if datetime(version('-date')) < datetime('25-Feb-2020') + versionNumber = getMatlabVersionNumber(); + assert(~isnan(versionNumber)); + if versionNumber < 9.8 % MATLAB 2020a fixed string support for HDF5, making % reading strings "consistent" with regular use. data = data .'; @@ -82,4 +84,9 @@ end H5D.close(did); H5F.close(fid); +end + +function versionNumber = getMatlabVersionNumber() + VersionInfo = ver('MATLAB'); + versionNumber = str2double(VersionInfo.Version); end \ No newline at end of file From ef248f5c8540d1f4ca894d4ef747e3890d879051 Mon Sep 17 00:00:00 2001 From: Lawrence Niu Date: Fri, 26 May 2023 10:27:31 -0400 Subject: [PATCH 3/4] reformat boolTest --- +tests/+unit/boolTest.m | 48 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/+tests/+unit/boolTest.m b/+tests/+unit/boolTest.m index ebae9729..0248ed09 100644 --- a/+tests/+unit/boolTest.m +++ b/+tests/+unit/boolTest.m @@ -1,35 +1,35 @@ function tests = boolTest() -tests = functiontests(localfunctions); + tests = functiontests(localfunctions); end function setupOnce(testCase) -rootPath = fullfile(fileparts(mfilename('fullpath')), '..', '..'); -testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath)); + rootPath = fullfile(fileparts(mfilename('fullpath')), '..', '..'); + testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath)); end function setup(testCase) -testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); -generateCore('savedir', '.'); -schemaPath = fullfile(misc.getMatnwbDir(),... - '+tests', '+unit', 'boolSchema', 'bool.namespace.yaml'); -generateExtension(schemaPath, 'savedir', '.'); -rehash(); + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); + generateCore('savedir', '.'); + schemaPath = fullfile(misc.getMatnwbDir(),... + '+tests', '+unit', 'boolSchema', 'bool.namespace.yaml'); + generateExtension(schemaPath, 'savedir', '.'); + rehash(); end function testIo(testCase) -nwb = NwbFile(... - 'identifier', 'BOOL',... - 'session_description', 'test bool',... - 'session_start_time', datetime()); -boolContainer = types.bool.BoolContainer(... - 'data', logical(randi([0,1], 100, 1)), ... - 'attribute', false); -scalarBoolContainer = types.bool.BoolContainer(... - 'data', false, ... - 'attribute', true); -nwb.acquisition.set('bool', boolContainer); -nwb.acquisition.set('scalarbool', scalarBoolContainer); -nwb.export('test.nwb'); -nwbActual = nwbRead('test.nwb', 'ignorecache'); -tests.util.verifyContainerEqual(testCase, nwbActual, nwb); + nwb = NwbFile(... + 'identifier', 'BOOL',... + 'session_description', 'test bool',... + 'session_start_time', datetime()); + boolContainer = types.bool.BoolContainer(... + 'data', logical(randi([0,1], 100, 1)), ... + 'attribute', false); + scalarBoolContainer = types.bool.BoolContainer(... + 'data', false, ... + 'attribute', true); + nwb.acquisition.set('bool', boolContainer); + nwb.acquisition.set('scalarbool', scalarBoolContainer); + nwb.export('test.nwb'); + nwbActual = nwbRead('test.nwb', 'ignorecache'); + tests.util.verifyContainerEqual(testCase, nwbActual, nwb); end From af0db0af229ae6d0981749749039cd6837c31975 Mon Sep 17 00:00:00 2001 From: Lawrence Niu Date: Fri, 26 May 2023 10:31:59 -0400 Subject: [PATCH 4/4] Vastly simplify ver checks - Also fixes tests. --- +io/parseDataset.m | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/+io/parseDataset.m b/+io/parseDataset.m index 582419a3..7bd8fd90 100644 --- a/+io/parseDataset.m +++ b/+io/parseDataset.m @@ -31,9 +31,7 @@ switch datatype.Class case 'H5T_STRING' - versionNumber = getMatlabVersionNumber(); - assert(~isnan(versionNumber)); - if versionNumber < 9.8 + if verLessThan('MATLAB', '9.8') % MATLAB 2020a fixed string support for HDF5, making % reading strings "consistent" with regular use. data = data .'; @@ -84,9 +82,4 @@ end H5D.close(did); H5F.close(fid); -end - -function versionNumber = getMatlabVersionNumber() - VersionInfo = ver('MATLAB'); - versionNumber = str2double(VersionInfo.Version); end \ No newline at end of file