From 28a5eb79ea5d88a4bf93ef7e5cc16943364563f8 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Fri, 13 Aug 2021 15:35:14 -0400 Subject: [PATCH 01/32] Support objects as a reference view target --- +io/getRefData.m | 15 ++++++++--- +types/+untyped/MetaClass.m | 35 ++++++++++++------------ +types/+untyped/ObjectView.m | 52 +++++++++++++++++++++++++++++------- +types/+untyped/RegionView.m | 29 ++++++++++++-------- 4 files changed, 90 insertions(+), 41 deletions(-) diff --git a/+io/getRefData.m b/+io/getRefData.m index 7618cc09..07af2855 100644 --- a/+io/getRefData.m +++ b/+io/getRefData.m @@ -6,7 +6,12 @@ validPaths = find(~cellfun('isempty', refpaths)); if isa(ref, 'types.untyped.RegionView') for i=validPaths - did = H5D.open(fid, refpaths{i}); + try + did = H5D.open(fid, refpaths{i}); + catch ME + error('MatNWB:getRefData:InvalidPath',... + 'Reference path `%s` was invalid', refpaths{i}); + end sid = H5D.get_space(did); %by default, we use block mode. regionShapes = ref(i).region; @@ -21,8 +26,12 @@ typesize = H5T.get_size(ref(1).type); ref_data = zeros([typesize size(ref)], 'uint8'); for i=validPaths - ref_data(:, i) = H5R.create(fid, ref(i).path, ref(i).reftype, ... - refspace(i)); + try + ref_data(:, i) = H5R.create(fid, ref(i).path, ref(i).reftype, refspace(i)); + catch ME + error('MatNWB:getRefData:InvalidPath',... + 'Reference path `%s` was invalid', ref(i).path); + end if H5I.is_valid(refspace(i)) H5S.close(refspace(i)); end diff --git a/+types/+untyped/MetaClass.m b/+types/+untyped/MetaClass.m index 75ac29c3..fc967152 100644 --- a/+types/+untyped/MetaClass.m +++ b/+types/+untyped/MetaClass.m @@ -1,4 +1,8 @@ classdef MetaClass < handle + properties (Hidden, SetAccess = private) + metaClass_fullPath; + end + methods function obj = MetaClass(varargin) end @@ -22,20 +26,24 @@ io.writeDataset(fid, fullpath, obj.data, 'forceArray'); end catch ME - if strcmp(ME.stack(2).name, 'getRefData') && ... - endsWith(ME.stack(1).file, ... - fullfile({'+H5D','+H5R'}, {'open.m', 'create.m'})) - refs(end+1) = {fullpath}; - return; - else - rethrow(ME); - end + refs = obj.capture_ref_errors(ME, fullpath, refs); + end + end + + function refs = capture_ref_errors(~, ME, fullpath, refs) + isRefDataErr = strcmp(ME.identifier, 'MatNWB:getRefData:InvalidPath'); + isMissingPathErr = strcmp(ME.identifier, 'MatNWB:ObjectView:MissingPath'); + if isRefDataErr || isMissingPathErr + refs(end+1) = {fullpath}; + else + rethrow(ME); end end end - methods + methods function refs = export(obj, fid, fullpath, refs) + obj.metaClass_fullPath = fullpath; %find reference properties propnames = properties(obj); props = cell(size(propnames)); @@ -50,14 +58,7 @@ try io.getRefData(fid, props{i}); catch ME - if strcmp(ME.stack(2).name, 'getRefData') && ... - endsWith(ME.stack(1).file, ... - fullfile({'+H5D','+H5R'}, {'open.m', 'create.m'})) - refs(end+1) = {fullpath}; - return; - else - rethrow(ME); - end + refs = obj.capture_ref_errors(ME, fullpath, refs); end end diff --git a/+types/+untyped/ObjectView.m b/+types/+untyped/ObjectView.m index 7624922d..62c5ad55 100644 --- a/+types/+untyped/ObjectView.m +++ b/+types/+untyped/ObjectView.m @@ -1,28 +1,60 @@ classdef ObjectView < handle - properties(SetAccess=private) - path; + properties (SetAccess = private) + path = ''; + target = []; end - properties(Constant, Hidden) + properties (Constant, Hidden) type = 'H5T_STD_REF_OBJ'; reftype = 'H5R_OBJECT'; end methods - function obj = ObjectView(path) - obj.path = path; + function obj = ObjectView(target) + %OBJECTVIEW a "view" or reference to an object meant to be + %saved in a different location in the NWB file. + % obj = ObjectView(path) + % path = A character or string indicating the full HDF5 path + % to the target object. + % obj = ObjectView(target) + % target = A generated NWB object. + + if ischar(target) || isstring(target) + obj.path = target; + else + validateattributes(target, {'types.untyped.MetaClass'}, {'scalar'}); + obj.target = target; + end end function v = refresh(obj, nwb) - assert(isa(nwb, 'NwbFile'),... - 'NWB:ObjectView:InvalidArgument',... - 'Argument `nwb` must be a valid ''NwbFile'' object.'); + validateattributes(nwb, {'NwbFile'}, {'scalar'}); - v = nwb.resolve({obj.path}); + if isempty(obj.path) + v = obj.target; + else + v = nwb.resolve({obj.path}); + end end function refs = export(obj, fid, fullpath, refs) - io.writeDataset(fid, fullpath, class(obj), obj); + io.writeDataset(fid, fullpath, obj); + end + + function path = get.path(obj) + if isempty(obj.path) + if isempty(obj.target) + path = ''; + elseif isempty(obj.target.metaClass_fullPath) + error('MatNWB:ObjectView:MissingPath',... + ['Target fullpath has not been set yet. '... + 'Is the referenced object assigned in the NWB File?']); + else + path = obj.target.metaClass_fullPath; + end + else + path = obj.path; + end end end end \ No newline at end of file diff --git a/+types/+untyped/RegionView.m b/+types/+untyped/RegionView.m index 85802003..f27b423b 100644 --- a/+types/+untyped/RegionView.m +++ b/+types/+untyped/RegionView.m @@ -1,31 +1,32 @@ classdef RegionView < handle - properties(SetAccess=private) + properties (SetAccess = private) path; + target; view; region; end - properties(Constant,Hidden) + properties (Constant, Hidden) type = 'H5T_STD_REF_DSETREG'; reftype = 'H5R_DATASET_REGION'; end methods - function obj = RegionView(path, varargin) + function obj = RegionView(target, varargin) %REGIONVIEW A region reference to a dataset in the same nwb file. % obj = REGIONVIEW(path, region) % path = char representing the internal path to the dataset. % region = A cell array of indices - obj.view = types.untyped.ObjectView(path); + % obj = REGIONVIEW(target, __) + % target = a generated NWB object. + + if isa(target, 'types.untyped.MetaClass') + validateattributes(target, {'types.untyped.DatasetClass'}, {'scalar'}); + end + obj.view = types.untyped.ObjectView(target); for i = 1:length(varargin) - dimSel = varargin{i}; - validateattributes(dimSel, {'numeric'}, {'positive', 'vector'}); - assert(length(dimSel) == length(unique(dimSel)),... - 'NWB:RegionView:DuplicateIndex',... - ['Due to how HDF5 handles selections, duplicate indices are not ',... - 'supported for RegionView. Ensure indices are unique across any given '... - 'dimension.']); + validateattributes(varargin{i}, {'numeric'}, {'positive', 'vector'}); end obj.region = varargin; end @@ -58,6 +59,8 @@ else data = data.internal.data; end + elseif isa(data, 'types.untyped.DatasetClass') + data = data.data; end v = data(RegionView.region{:}); @@ -71,5 +74,9 @@ function path = get.path(obj) path = obj.view.path; end + + function object = get.target(obj) + object = obj.view.target; + end end end \ No newline at end of file From f7ba7be28a41fe4996e5a601dc119a0668436af9 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Fri, 13 Aug 2021 15:35:41 -0400 Subject: [PATCH 02/32] update region view tests to test new syntax --- +tests/+unit/regionViewTest.m | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/+tests/+unit/regionViewTest.m b/+tests/+unit/regionViewTest.m index 0e4eecca..e5b8bdfb 100644 --- a/+tests/+unit/regionViewTest.m +++ b/+tests/+unit/regionViewTest.m @@ -22,20 +22,20 @@ function testRegionViewIo(testCase) 'session_start_time', datetime()); rcData = types.rrs.RefContainer('data', rand(10, 10, 10, 10, 10)); -rcDataName = 'refdata'; -rcPath = sprintf('/acquisition/%s/data', rcDataName); -nwb.acquisition.set(rcDataName, rcData); +% rcDataName = 'refdata'; +% rcPath = sprintf('/acquisition/%s/data', rcDataName); +nwb.acquisition.set('refdata', rcData); for i = 1:100 rcAttrRef = types.untyped.RegionView(... - rcPath,... + rcData,... getRandInd(10),... getRandInd(10),... getRandInd(10),... getRandInd(10),... getRandInd(10)); rcDataRef = types.untyped.RegionView(... - rcPath,... + rcData,... 1:getRandInd(10),... 1:getRandInd(10),... 1:getRandInd(10),... From 4bd57e8c80012f00ff7f96eb84a0ccee831c8162 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Fri, 13 Aug 2021 18:04:10 -0400 Subject: [PATCH 03/32] Add Support for object targets in Softlinks Fix checks for MetaClass and ObjectView as well. --- +types/+untyped/MetaClass.m | 6 ++-- +types/+untyped/ObjectView.m | 14 ++++++++- +types/+untyped/SoftLink.m | 57 +++++++++++++++++++++++++++++------- NwbFile.m | 9 +++--- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/+types/+untyped/MetaClass.m b/+types/+untyped/MetaClass.m index fc967152..87feabf6 100644 --- a/+types/+untyped/MetaClass.m +++ b/+types/+untyped/MetaClass.m @@ -31,9 +31,9 @@ end function refs = capture_ref_errors(~, ME, fullpath, refs) - isRefDataErr = strcmp(ME.identifier, 'MatNWB:getRefData:InvalidPath'); - isMissingPathErr = strcmp(ME.identifier, 'MatNWB:ObjectView:MissingPath'); - if isRefDataErr || isMissingPathErr + if any(strcmp(ME.identifier, {... + 'MatNWB:getRefData:InvalidPath',... + 'MatNWB:ObjectView:MissingPath'})) refs(end+1) = {fullpath}; else rethrow(ME); diff --git a/+types/+untyped/ObjectView.m b/+types/+untyped/ObjectView.m index 62c5ad55..4dc932f6 100644 --- a/+types/+untyped/ObjectView.m +++ b/+types/+untyped/ObjectView.m @@ -1,7 +1,10 @@ classdef ObjectView < handle + properties (SetAccess = private, Hidden) + target = []; + end + properties (SetAccess = private) path = ''; - target = []; end properties (Constant, Hidden) @@ -20,6 +23,7 @@ % target = A generated NWB object. if ischar(target) || isstring(target) + validateattributes(target, {'char', 'string'}, {'scalartext'}); obj.path = target; else validateattributes(target, {'types.untyped.MetaClass'}, {'scalar'}); @@ -56,5 +60,13 @@ path = obj.path; end end + + function tf = has_path(obj) + try + tf = ~isempty(obj.path); + catch + tf = false; + end + end end end \ No newline at end of file diff --git a/+types/+untyped/SoftLink.m b/+types/+untyped/SoftLink.m index ea24f58f..e0651b14 100644 --- a/+types/+untyped/SoftLink.m +++ b/+types/+untyped/SoftLink.m @@ -1,24 +1,48 @@ classdef SoftLink < handle - properties - path; + + properties (Hidden, SetAccess = private) + target = []; end -% properties(Hidden, SetAccess=immutable) -% type; %type constraint, used by file generation -% end + properties (SetAccess = private) + path = ''; + end methods - function obj = SoftLink(path) - obj.path = path; + function obj = SoftLink(target) + %SOFTLINK HDF5 soft link. + % obj = SOFTLINK(path) make soft link given a HDF5 full path. + % path = HDF5-friendly path e.g. '/acquisition/es1' + % obj = SOFTLINK(target) make soft link from pre-existant + % object. + % target = pre-generated NWB object. + + if ischar(target) || isstring(target) + validateattributes(target, {'char', 'string'}, {'scalartext'}); + obj.path = target; + else + validateattributes(target, {'types.untyped.MetaClass'}, {'scalar'}); + obj.target = target; + end end function set.path(obj, val) - if ~ischar(val) - error('Property `path` should be a char array'); - end + validateattributes(val, {'char', 'string'}, {'scalartext'}); obj.path = val; end + function p = get.path(obj) + if isempty(obj.path) + if isempty(obj.target) + p = ''; + else + p = obj.target.metaClass_fullPath; + end + else + p = obj.path; + end + end + function refobj = deref(obj, nwb) assert(isa(nwb, 'NwbFile'),... 'NWB:SoftLink:Deref:InvalidArgument',... @@ -28,9 +52,20 @@ end function refs = export(obj, fid, fullpath, refs) + if isempty(obj.path) + refs{end+1} = fullpath; + return; + end + + if isempty(obj.path) + target_path = obj.target.metaClass_fullPath; + else + target_path = obj.path; + end + plist = 'H5P_DEFAULT'; try - H5L.create_soft(obj.path, fid, fullpath, plist, plist); + H5L.create_soft(target_path, fid, fullpath, plist, plist); catch ME if contains(ME.message, 'name already exists') previousLink = H5L.get_val(fid, fullpath, plist); diff --git a/NwbFile.m b/NwbFile.m index 94268945..b9071264 100644 --- a/NwbFile.m +++ b/NwbFile.m @@ -112,10 +112,10 @@ function resolveReferences(obj, fid, references) references(resolved) = []; else errorFormat = ['object(s) could not be created:\n%s\n\nThe '... - 'listed object(s) above contain an ObjectView or '... - 'RegionView object that has failed to resolve itself. '... - 'Please check for any invalid reference paths in any '... - 'of the properties of the objects listed above.']; + 'listed object(s) above contain an ObjectView, '... + 'RegionView , or SoftLink object that has failed to resolve itself. '... + 'Please check for any references that were not assigned to the root '... + ' NwbFile or if any of the above paths are incorrect.']; unresolvedRefs = strjoin(references, newline); error('NWB:NwbFile:UnresolvedReferences',... errorFormat, file.addSpaces(unresolvedRefs, 4)); @@ -171,7 +171,6 @@ function embedSpecifications(~, fid) end end end - end end From ce7350c99026ed90a2c2a0a70457085fb8d1dcb5 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Fri, 13 Aug 2021 18:04:55 -0400 Subject: [PATCH 04/32] Update DynamicTable methods to use object refs No longer require `tablepath` as an argument. DynamicTable methods now handle all requirements for Vector Indices --- +types/+util/+dynamictable/addRawData.m | 14 +++++++------- +types/+util/+dynamictable/addRow.m | 5 +---- +types/+util/+dynamictable/addTableRow.m | 9 +-------- +types/+util/+dynamictable/addVarargRow.m | 13 +++---------- +types/+util/+dynamictable/addVecInd.m | 16 ++++++---------- +types/+util/+dynamictable/getIndex.m | 14 ++++++++++---- 6 files changed, 28 insertions(+), 43 deletions(-) diff --git a/+types/+util/+dynamictable/addRawData.m b/+types/+util/+dynamictable/addRawData.m index a77c339f..0f54c78f 100644 --- a/+types/+util/+dynamictable/addRawData.m +++ b/+types/+util/+dynamictable/addRawData.m @@ -1,12 +1,7 @@ -function addRawData(DynamicTable, column, data, index) +function addRawData(DynamicTable, column, data) %ADDRAWDATA Internal method for adding data to DynamicTable given column % name, data and an optional index. validateattributes(column, {'char'}, {'scalartext'}); -if nargin < 4 - % indicates an index column. Note we assume that the index name is correct. - % Validation of this index name must occur upstream. - index = ''; -end % Don't set the data until after indices are updated. if 8 == exist('types.hdmf_common.VectorData', 'class') @@ -29,7 +24,12 @@ function addRawData(DynamicTable, column, data, index) DynamicTable.vectordata.set(column, VecData); end -if ~isempty(index) +index = types.util.dynamictable.getIndex(DynamicTable, column); +if size(data, 1) > 1 || ~isempty(index) + if isempty(index) + index = types.util.dynamictable.addVecInd(DynamicTable, column); + end + if isprop(DynamicTable, index) VecInd = DynamicTable.(index); elseif isprop(DynamicTable, 'vectorindex') % Schema < 2.3.0 diff --git a/+types/+util/+dynamictable/addRow.m b/+types/+util/+dynamictable/addRow.m index 91f9325d..f09a1d4a 100644 --- a/+types/+util/+dynamictable/addRow.m +++ b/+types/+util/+dynamictable/addRow.m @@ -8,7 +8,7 @@ function addRow(DynamicTable, varargin) % ADDROW(DT,col1,val1,col2,val2,...,coln,valn) append a single row % to the DynamicTable % -% ADDROW(DT,___,Name,Value) specify either 'id' or 'tablepath' +% ADDROW(DT,___,Name,Value) optional 'id' % % This function asserts the following: % 1) DynamicTable is a valid dynamic table and has the correct @@ -22,9 +22,6 @@ function addRow(DynamicTable, varargin) % 5) The type of the data cannot be a cell array of numeric values if using % keyword arguments. For table appending mode, this is how ragged arrays % are represented. -% 6) Ragged arrays (that is, rows containing more than one sub-row) require -% an extra parameter called `tablepath` which indicates where in the NWB -% file the table is. validateattributes(DynamicTable,... {'types.core.DynamicTable', 'types.hdmf_common.DynamicTable'},... diff --git a/+types/+util/+dynamictable/addTableRow.m b/+types/+util/+dynamictable/addTableRow.m index 3c7f13f8..7f90bc81 100644 --- a/+types/+util/+dynamictable/addTableRow.m +++ b/+types/+util/+dynamictable/addTableRow.m @@ -2,7 +2,6 @@ function addTableRow(DynamicTable, subTable, varargin) p = inputParser(); p.StructExpand = false; addParameter(p, 'id', [], @(x)isnumeric(x)); % optional as `id` column is supported for tables. -addParameter(p, 'tablepath', '', @(x)ischar(x)); % required for ragged arrays. parse(p, varargin{:}); rowNames = subTable.Properties.VariableNames; @@ -40,19 +39,13 @@ function addTableRow(DynamicTable, subTable, varargin) validateType(TypeMap(rn), rowColumn); end - % instantiate vector index here because it's dependent on the table - % fullpath. - vecIndName = types.util.dynamictable.getIndex(DynamicTable, rn); - if isempty(vecIndName) && (~isempty(p.Results.tablepath) || (~iscellstr(rowColumn) && iscell(rowColumn))) - vecIndName = types.util.dynamictable.addVecInd(DynamicTable, rn, p.Results.tablepath); - end for j = 1:length(rowColumn) if iscell(rowColumn) rv = rowColumn{j}; else rv = rowColumn(j); end - types.util.dynamictable.addRawData(DynamicTable, rn, rv, vecIndName); + types.util.dynamictable.addRawData(DynamicTable, rn, rv); end end diff --git a/+types/+util/+dynamictable/addVarargRow.m b/+types/+util/+dynamictable/addVarargRow.m index 1eb8a471..6f0f66db 100644 --- a/+types/+util/+dynamictable/addVarargRow.m +++ b/+types/+util/+dynamictable/addVarargRow.m @@ -2,7 +2,6 @@ function addVarargRow(DynamicTable, varargin) p = inputParser(); p.KeepUnmatched = true; p.StructExpand = false; -addParameter(p, 'tablepath', '', @(x)ischar(x)); % required for ragged arrays. addParameter(p, 'id', []); % `id` override but doesn't actually show up in `colnames` for i = 1:length(DynamicTable.colnames) @@ -18,9 +17,9 @@ function addVarargRow(DynamicTable, varargin) rowNames = fieldnames(p.Results); % not using setDiff because we want to retain set order. -rowNames(strcmp(rowNames, 'tablepath') | strcmp(rowNames, 'id')) = []; +rowNames(strcmp(rowNames, 'id')) = []; -missingColumns = setdiff(p.UsingDefaults, {'tablepath', 'id'}); +missingColumns = setdiff(p.UsingDefaults, {'id'}); assert(isempty(missingColumns),... 'NWB:DynamicTable:AddRow:MissingColumns',... 'Missing columns { %s }', strjoin(missingColumns, ', ')); @@ -45,13 +44,7 @@ function addVarargRow(DynamicTable, varargin) rv = {rv}; end - % instantiate vector index here because it's dependent on the table - % fullpath. - vecIndName = types.util.dynamictable.getIndex(DynamicTable, rn); - if isempty(vecIndName) && (~isempty(p.Results.tablepath) || size(rv, 1) > 1) - vecIndName = types.util.dynamictable.addVecInd(DynamicTable, rn, p.Results.tablepath); - end - types.util.dynamictable.addRawData(DynamicTable, rn, rv, vecIndName); + types.util.dynamictable.addRawData(DynamicTable, rn, rv); end if specifiesId diff --git a/+types/+util/+dynamictable/addVecInd.m b/+types/+util/+dynamictable/addVecInd.m index 911d580f..e9c08f35 100644 --- a/+types/+util/+dynamictable/addVecInd.m +++ b/+types/+util/+dynamictable/addVecInd.m @@ -1,17 +1,13 @@ -function vecIndName = addVecInd(DynamicTable, colName, tablepath) +function vecIndName = addVecInd(DynamicTable, colName) %ADDVECIND Add VectorIndex object to DynamicTable validateattributes(colName, {'char'}, {'scalartext'}); -validateattributes(tablepath, {'char'}, {'scalartext'}); -assert(~isempty(tablepath),... - 'NWB:DynamicTable:AddRow:MissingTablePath',... - ['addRow cannot create ragged arrays without a full HDF5 path to the Dynamic Table. '... - 'Please either add the full expected HDF5 path under the keyword argument `tablepath` '... - 'or call addRow with row data only.']); vecIndName = [colName '_index']; % arbitrary convention of appending '_index' to data column names -if ~endsWith(tablepath, '/') - tablepath = [tablepath '/']; + +if isprop(DynamicTable, colName) + vecTarget = types.untyped.ObjectView(DynamicTable.(colName)); +else + vecTarget = types.untyped.ObjectView(DynamicTable.vectordata.get(colName)); end -vecTarget = types.untyped.ObjectView([tablepath colName]); oldDataHeight = 0; if isKey(DynamicTable.vectordata, colName) || isprop(DynamicTable, colName) if isprop(DynamicTable, colName) diff --git a/+types/+util/+dynamictable/getIndex.m b/+types/+util/+dynamictable/getIndex.m index 10c157e7..f9f14494 100644 --- a/+types/+util/+dynamictable/getIndex.m +++ b/+types/+util/+dynamictable/getIndex.m @@ -31,7 +31,7 @@ && ~isa(vecData, 'types.core.VectorIndex') continue; end - if isVecIndColumn(vecData, column) + if isVecIndColumn(DynamicTable, vecData, column) indexName = vk; return; end @@ -50,14 +50,20 @@ for i = 1:length(DynamicTableProps) vk = DynamicTableProps{i}; VecInd = DynamicTable.(vk); - if isVecIndColumn(VecInd, column) + if isVecIndColumn(DynamicTable, VecInd, column) indexName = vk; return; end end end -function tf = isVecIndColumn(VectorIndex, column) -tf = endsWith(VectorIndex.target.path, ['/' column]); +function tf = isVecIndColumn(DynamicTable, VectorIndex, column) +if VectorIndex.target.has_path() + tf = endsWith(VectorIndex.target.path, ['/' column]); +elseif isprop(DynamicTable, column) + tf = VectorIndex.target == DynamicTable.(column); +else + tf = VectorIndex.target == DynamicTable.vectordata.get(column); +end end From 2546c3befa6545e77fa9965b8552b83744ff2e5f Mon Sep 17 00:00:00 2001 From: Lawrence Date: Fri, 13 Aug 2021 18:05:52 -0400 Subject: [PATCH 05/32] Fix Tests Update tests to use new View/SoftLink object reference forms Update testCase validation since new temporary properties break verifyEqual. update regionReferenceSchema schema as it was relying on dubious behavior regarding group classes and their region reference properties. --- +tests/+sanity/GenerationTest.m | 3 +-- +tests/+system/DynamicTableTest.m | 3 +-- +tests/+system/ElectricalSeriesIOTest.m | 20 +++++++------------ +tests/+system/ElectrodeGroupIOTest.m | 2 +- +tests/+system/ImagingPlaneIOTest.m | 2 +- +tests/+system/NwbTestInterface.m | 3 +++ +tests/+system/PhotonSeriesIOTest.m | 4 ++-- +tests/+system/UnitTimesIOTest.m | 2 +- +tests/+unit/dataStubTest.m | 2 +- .../regionReferenceSchema/rrs.regref.yaml | 5 ++++- +tests/+unit/regionViewTest.m | 10 ++++------ +tests/+util/verifyContainerEqual.m | 8 ++++++++ 12 files changed, 34 insertions(+), 30 deletions(-) diff --git a/+tests/+sanity/GenerationTest.m b/+tests/+sanity/GenerationTest.m index 5f8264ec..ee45dc82 100644 --- a/+tests/+sanity/GenerationTest.m +++ b/+tests/+sanity/GenerationTest.m @@ -47,8 +47,7 @@ function dynamicTableMethodsTest(testCase) 'stop_time', stop_time,... 'randomvalues', rand_data,... 'stringdata', {'TRUE'},... - 'id', id(i),... - 'tablepath', '/intervals/trials'); + 'id', id(i)); end t = table(id(101:200), (101:200) .', (102:201) .',... mat2cell(rand(500,1), repmat(5, 100, 1)), repmat({'TRUE'}, 100, 1),... diff --git a/+tests/+system/DynamicTableTest.m b/+tests/+system/DynamicTableTest.m index 5c04d721..1803e291 100644 --- a/+tests/+system/DynamicTableTest.m +++ b/+tests/+system/DynamicTableTest.m @@ -16,8 +16,7 @@ function addContainer(~, file) 'stop_time', stop_time,... 'randomvalues', rand_data,... 'stringdata', {'TRUE'},... - 'id', id(i),... - 'tablepath', '/intervals/trials'); + 'id', id(i)); end t = table(id(101:200), (101:200) .', (102:201) .',... mat2cell(rand(500,1), repmat(5, 100, 1)), repmat({'TRUE'}, 100, 1),... diff --git a/+tests/+system/ElectricalSeriesIOTest.m b/+tests/+system/ElectricalSeriesIOTest.m index aa3a6ec1..c69e140a 100644 --- a/+tests/+system/ElectricalSeriesIOTest.m +++ b/+tests/+system/ElectricalSeriesIOTest.m @@ -2,20 +2,14 @@ methods function addContainer(testCase, file) %#ok - devnm = 'dev1'; - egnm = 'tetrode1'; - esnm = 'test_eS'; - devBase = '/general/devices/'; - ephysBase = '/general/extracellular_ephys/'; - devlink = types.untyped.SoftLink([devBase devnm]); - eglink = types.untyped.ObjectView([ephysBase egnm]); - etReg = types.untyped.ObjectView([ephysBase 'electrodes']); dev = types.core.Device(); - file.general_devices.set(devnm, dev); + file.general_devices.set('dev1', dev); + + egnm = 'tetrode1'; eg = types.core.ElectrodeGroup( ... 'description', 'tetrode description', ... 'location', 'tetrode location', ... - 'device', devlink); + 'device', types.untyped.SoftLink(dev)); electrodes = util.createElectrodeTable(); for i = 1:4 @@ -24,7 +18,7 @@ function addContainer(testCase, file) %#ok 'imp', 1,... 'location', {'CA1'},... 'filtering', 0,... - 'group', eglink, 'group_name', {egnm}); + 'group', types.untyped.ObjectView(eg), 'group_name', {egnm}); end file.general_extracellular_ephys_electrodes = electrodes; @@ -35,9 +29,9 @@ function addContainer(testCase, file) %#ok 'electrodes', ... types.hdmf_common.DynamicTableRegion(... 'data', [0;2],... - 'table', etReg,... + 'table', types.untyped.ObjectView(electrodes),... 'description', 'the first and third electrodes')); - file.acquisition.set(esnm, es); + file.acquisition.set('test_eS', es); end function c = getContainer(testCase, file) %#ok diff --git a/+tests/+system/ElectrodeGroupIOTest.m b/+tests/+system/ElectrodeGroupIOTest.m index 2100664b..d4e5b100 100644 --- a/+tests/+system/ElectrodeGroupIOTest.m +++ b/+tests/+system/ElectrodeGroupIOTest.m @@ -7,7 +7,7 @@ function addContainer(testCase, file) %#ok eg = types.core.ElectrodeGroup( ... 'description', 'a test ElectrodeGroup', ... 'location', 'a nonexistent place', ... - 'device', types.untyped.SoftLink('/general/devices/dev1')); + 'device', types.untyped.SoftLink(dev)); file.general_extracellular_ephys.set('elec1', eg); end diff --git a/+tests/+system/ImagingPlaneIOTest.m b/+tests/+system/ImagingPlaneIOTest.m index bfe34fc5..71e55599 100644 --- a/+tests/+system/ImagingPlaneIOTest.m +++ b/+tests/+system/ImagingPlaneIOTest.m @@ -8,7 +8,7 @@ function addContainer(testCase, file) %#ok ip = types.core.ImagingPlane( ... 'description', 'a fake ImagingPlane', ... 'optchan1', oc, ... - 'device', types.untyped.SoftLink('/general/devices/imaging_device_1'), ... + 'device', types.untyped.SoftLink(dev), ... 'excitation_lambda', 6.28, ... 'imaging_rate', 2.718, ... 'indicator', 'GFP', ... diff --git a/+tests/+system/NwbTestInterface.m b/+tests/+system/NwbTestInterface.m index f18cd063..46b03639 100644 --- a/+tests/+system/NwbTestInterface.m +++ b/+tests/+system/NwbTestInterface.m @@ -51,6 +51,9 @@ function verifyContainerEqual(testCase, actual, expected) verifySetEqual(testCase, actualVal, expectedVal, failmsg); elseif isdatetime(actualVal) testCase.verifyEqual(char(actualVal), char(expectedVal), failmsg); + elseif isa(actualVal, 'types.untyped.ObjectView')... + || isa(actualVal, 'types.untyped.RegionView') + testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); else if isa(actualVal, 'types.untyped.DataStub') actualTrue = actualVal.load(); diff --git a/+tests/+system/PhotonSeriesIOTest.m b/+tests/+system/PhotonSeriesIOTest.m index 901853e6..816e27c1 100644 --- a/+tests/+system/PhotonSeriesIOTest.m +++ b/+tests/+system/PhotonSeriesIOTest.m @@ -9,7 +9,7 @@ function addContainer(testCase, file) %#ok ip = types.core.ImagingPlane( ... 'description', 'a fake ImagingPlane', ... 'optchan1', oc, ... - 'device', types.untyped.SoftLink('/general/devices/dev1'), ... + 'device', types.untyped.SoftLink(dev), ... 'excitation_lambda', 6.28, ... 'imaging_rate', 2.718, ... 'indicator', 'GFP', ... @@ -17,7 +17,7 @@ function addContainer(testCase, file) %#ok tps = types.core.TwoPhotonSeries( ... 'data', ones(3,3,3), ... - 'imaging_plane', types.untyped.SoftLink('/general/optophysiology/imgpln1'), ... + 'imaging_plane', types.untyped.SoftLink(ip), ... 'data_unit', 'image_unit', ... 'format', 'raw', ... 'field_of_view', [2, 2, 5] .', ... diff --git a/+tests/+system/UnitTimesIOTest.m b/+tests/+system/UnitTimesIOTest.m index 787b82f3..17366f00 100644 --- a/+tests/+system/UnitTimesIOTest.m +++ b/+tests/+system/UnitTimesIOTest.m @@ -3,7 +3,7 @@ function addContainer(~, file) vdata = rand(10,1); file.units = types.core.Units('description', 'test Units', 'colnames', {'spike_times'}); - file.units.addRow('spike_times', vdata(1), 'tablepath', '/units'); + file.units.addRow('spike_times', vdata(1)); file.units.addRow('spike_times', vdata(2:5)); file.units.addRow('spike_times', vdata(6:end)); end diff --git a/+tests/+unit/dataStubTest.m b/+tests/+unit/dataStubTest.m index 19349b5a..40fe3623 100644 --- a/+tests/+unit/dataStubTest.m +++ b/+tests/+unit/dataStubTest.m @@ -81,7 +81,7 @@ function testObjectCopy(testCase) 'identifier', 'DATASTUB',... 'session_description', 'test datastub object copy',... 'session_start_time', datetime()); -rc = types.rrs.RefContainer('data', rand(100, 100)); +rc = types.rrs.RefContainer('data', types.rrs.RefData('data', rand(100, 100))); rcPath = '/acquisition/rc'; rcDataPath = [rcPath '/data']; rcRef = types.cs.CompoundRefData('data', table(... diff --git a/+tests/+unit/regionReferenceSchema/rrs.regref.yaml b/+tests/+unit/regionReferenceSchema/rrs.regref.yaml index 52378d39..98714d28 100644 --- a/+tests/+unit/regionReferenceSchema/rrs.regref.yaml +++ b/+tests/+unit/regionReferenceSchema/rrs.regref.yaml @@ -1,9 +1,12 @@ +datasets: +- neurodata_type_def: RefData + neurodata_type_inc: NWBData groups: - neurodata_type_def: RefContainer neurodata_type_inc: NWBDataInterface datasets: - name: data - dtype: any + neurodata_type_inc: RefData - neurodata_type_def: ContainerReference neurodata_type_inc: NWBDataInterface attributes: diff --git a/+tests/+unit/regionViewTest.m b/+tests/+unit/regionViewTest.m index e5b8bdfb..b7bbb1e2 100644 --- a/+tests/+unit/regionViewTest.m +++ b/+tests/+unit/regionViewTest.m @@ -21,21 +21,19 @@ function testRegionViewIo(testCase) 'session_description', 'region ref test',... 'session_start_time', datetime()); -rcData = types.rrs.RefContainer('data', rand(10, 10, 10, 10, 10)); -% rcDataName = 'refdata'; -% rcPath = sprintf('/acquisition/%s/data', rcDataName); -nwb.acquisition.set('refdata', rcData); +rcContainer = types.rrs.RefContainer('data', types.rrs.RefData('data', rand(10, 10, 10, 10, 10))); +nwb.acquisition.set('refdata', rcContainer); for i = 1:100 rcAttrRef = types.untyped.RegionView(... - rcData,... + rcContainer.data,... getRandInd(10),... getRandInd(10),... getRandInd(10),... getRandInd(10),... getRandInd(10)); rcDataRef = types.untyped.RegionView(... - rcData,... + rcContainer.data,... 1:getRandInd(10),... 1:getRandInd(10),... 1:getRandInd(10),... diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index b4030d71..5b8b237e 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -19,6 +19,14 @@ function verifyContainerEqual(testCase, actual, expected) testCase.verifyEqual(char(val1), char(val2), failmsg); elseif ischar(val2) testCase.verifyEqual(char(val1), val2, failmsg); + elseif isa(val2, 'types.untyped.ObjectView') + testCase.verifyEqual(val1.path, val2.path, failmsg); + elseif isa(val2, 'types.untyped.RegionView') + testCase.verifyEqual(val1.path, val2.path, failmsg); + testCase.verifyEqual(val1.region, val2.region, failmsg); + elseif isa(val2, 'types.untyped.Anon') + testCase.verifyEqual(val1.name, val2.name, failmsg); + tests.util.verifyContainerEqual(testCase, val1.value, val2.value); else testCase.verifyEqual(val1, val2, failmsg); end From 5c9eaec851cfff461638648a7a2d0304a0a39c5f Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 12:06:06 -0400 Subject: [PATCH 06/32] Minor fix for isodatetimes when read from file. isodatetime types which are read from file are no different from string arrays. However, they must be treated a little differently and must be converted to more useful MATLAB datetime types. This fix is for isodatetime arrays which apparantly weren't turning MxN character arrays into proper cell arrays for datetimes. --- +types/+util/checkDtype.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 6d784b1b..84d82bae 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -125,7 +125,7 @@ errid, errmsg); if ischar(val) || iscellstr(val) if ischar(val) - val = {val}; + val = mat2cell(val, ones(1, size(val,1))); end datevals = cell(size(val)); @@ -148,6 +148,8 @@ if isscalar(val) val = val{1}; + elseif isrow(val) + val = val .'; end elseif strcmp(type, 'char') assert(ischar(val) || iscellstr(val), errid, errmsg); From d3d4e0070089d58535d11ebb70608c4d4c602b21 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 12:08:34 -0400 Subject: [PATCH 07/32] Refactor tests Remove verifyContainerEqual calls in the NwbTestInterface. Use the tests.util variant instead. Tests now use SoftLink object-derived paths instead of full paths. --- +tests/+system/AmendTest.m | 2 +- +tests/+system/NwbTestInterface.m | 77 +---------------------------- +tests/+system/PhotonSeriesIOTest.m | 8 +-- +tests/+system/PyNWBIOTest.m | 2 +- +tests/+system/RoundTripTest.m | 2 +- +tests/+util/verifyContainerEqual.m | 51 +++++++++++-------- 6 files changed, 38 insertions(+), 104 deletions(-) diff --git a/+tests/+system/AmendTest.m b/+tests/+system/AmendTest.m index d2ecc301..a8a84080 100644 --- a/+tests/+system/AmendTest.m +++ b/+tests/+system/AmendTest.m @@ -9,7 +9,7 @@ function testAmend(testCase) writeContainer = testCase.getContainer(testCase.file); readFile = nwbRead(filename); readContainer = testCase.getContainer(readFile); - testCase.verifyContainerEqual(readContainer, writeContainer); + tests.util.verifyContainerEqual(testCase, readContainer, writeContainer); end end diff --git a/+tests/+system/NwbTestInterface.m b/+tests/+system/NwbTestInterface.m index 46b03639..829c0499 100644 --- a/+tests/+system/NwbTestInterface.m +++ b/+tests/+system/NwbTestInterface.m @@ -31,82 +31,7 @@ function setupMethod(testCase) function n = className(testCase) classSplit = strsplit(class(testCase), '.'); n = classSplit{end}; - end - - function verifyContainerEqual(testCase, actual, expected) - testCase.verifyEqual(class(actual), class(expected)); - props = properties(actual); - for i = 1:numel(props) - prop = props{i}; - if strcmp(prop, 'file_create_date') - continue; - end - actualVal = actual.(prop); - expectedVal = expected.(prop); - failmsg = ['Values for property ''' prop ''' are not equal']; - if startsWith(class(actualVal), 'types.')... - && ~startsWith(class(actualVal), 'types.untyped') - verifyContainerEqual(testCase, actualVal, expectedVal); - elseif isa(actualVal, 'types.untyped.Set') - verifySetEqual(testCase, actualVal, expectedVal, failmsg); - elseif isdatetime(actualVal) - testCase.verifyEqual(char(actualVal), char(expectedVal), failmsg); - elseif isa(actualVal, 'types.untyped.ObjectView')... - || isa(actualVal, 'types.untyped.RegionView') - testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); - else - if isa(actualVal, 'types.untyped.DataStub') - actualTrue = actualVal.load(); - else - actualTrue = actualVal; - end - - if isvector(expectedVal) && isvector(actualTrue) && numel(expectedVal) == numel(actualTrue) - actualTrue = reshape(actualTrue, size(expectedVal)); - end - - if isinteger(actualTrue) && isinteger(expectedVal) - actualSize = class(actualTrue); - actualSize = str2double(actualSize(4:end)); - expectedSize = class(expectedVal); - expectedSize = str2double(expectedSize(4:end)); - testCase.verifyGreaterThanOrEqual(actualSize, expectedSize, failmsg); - testCase.verifyEqual(double(actualTrue), double(expectedVal), failmsg); - continue; - end - testCase.verifyEqual(actualTrue, expectedVal, failmsg); - end - end - end - - function verifySetEqual(testCase, actual, expected, failmsg) - testCase.verifyEqual(class(actual), class(expected)); - ak = actual.keys(); - ek = expected.keys(); - verifyTrue(testCase, isempty(setxor(ak, ek)), failmsg); - for i=1:numel(ak) - key = ak{i}; - verifyContainerEqual(testCase, actual.get(key), ... - expected.get(key)); - end - end - - function verifyUntypedEqual(testCase, actual, expected) - testCase.verifyEqual(class(actual), class(expected)); - props = properties(actual); - for i = 1:numel(props) - prop = props{i}; - val1 = actual.(prop); - val2 = expected.(prop); - if isa(val1, 'types.core.NWBContainer') || isa(val1, 'types.core.NWBData') - verifyContainerEqual(testCase, val1, val2); - else - testCase.verifyEqual(val1, val2, ... - ['Values for property ''' prop ''' are not equal']); - end - end - end - + end end methods(Abstract) diff --git a/+tests/+system/PhotonSeriesIOTest.m b/+tests/+system/PhotonSeriesIOTest.m index 816e27c1..96bc4bae 100644 --- a/+tests/+system/PhotonSeriesIOTest.m +++ b/+tests/+system/PhotonSeriesIOTest.m @@ -37,18 +37,18 @@ function addContainer(testCase, file) %#ok function appendContainer(~, file) oldImagingPlane = file.general_optophysiology.get('imgpln1'); - file.general_optophysiology.set('imgpln2',... - types.core.ImagingPlane(... + newImagingPlane = types.core.ImagingPlane(... 'description', 'a different imaging plane',... 'device', oldImagingPlane.device,... 'optchan1', oldImagingPlane.opticalchannel.get('optchan1'),... 'excitation_lambda', 1,... 'imaging_rate', 2,... 'indicator', 'ASL',... - 'location', 'somewhere else in the brain')); + 'location', 'somewhere else in the brain'); + file.general_optophysiology.set('imgpln2', newImagingPlane); hTwoPhotonSeries = file.acquisition.get('test_2ps'); - hTwoPhotonSeries.imaging_plane.path = '/general/optophysiology/imgpln2'; + hTwoPhotonSeries.imaging_plane = types.untyped.SoftLink(newImagingPlane); hTwoPhotonSeries.data = hTwoPhotonSeries.data + rand(); end end diff --git a/+tests/+system/PyNWBIOTest.m b/+tests/+system/PyNWBIOTest.m index 50c2133c..e0451171 100644 --- a/+tests/+system/PyNWBIOTest.m +++ b/+tests/+system/PyNWBIOTest.m @@ -25,7 +25,7 @@ function testInFromPyNWB(testCase) pyfile = nwbRead(filename); pycontainer = testCase.getContainer(pyfile); matcontainer = testCase.getContainer(testCase.file); - testCase.verifyContainerEqual(pycontainer, matcontainer); + tests.util.verifyContainerEqual(testCase, pycontainer, matcontainer); end end diff --git a/+tests/+system/RoundTripTest.m b/+tests/+system/RoundTripTest.m index c9b61353..313ab97d 100644 --- a/+tests/+system/RoundTripTest.m +++ b/+tests/+system/RoundTripTest.m @@ -6,7 +6,7 @@ function testRoundTrip(testCase) writeContainer = testCase.getContainer(testCase.file); readFile = nwbRead(filename); readContainer = testCase.getContainer(readFile); - testCase.verifyContainerEqual(readContainer, writeContainer); + tests.util.verifyContainerEqual(testCase, readContainer, writeContainer); end end end \ No newline at end of file diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 5b8b237e..f0723435 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -3,32 +3,41 @@ function verifyContainerEqual(testCase, actual, expected) props = properties(actual); for i = 1:numel(props) prop = props{i}; - val1 = actual.(prop); - val2 = expected.(prop); + actualVal = actual.(prop); + expectedVal = expected.(prop); failmsg = ['Values for property ''' prop ''' are not equal']; - if isa(val1, 'types.untyped.DataStub') - val1 = val1.load(); + if isa(actualVal, 'types.untyped.DataStub') + actualVal = actualVal.load(); end - if startsWith(class(val2), 'types.') && ~startsWith(class(val2), 'types.untyped') - tests.util.verifyContainerEqual(testCase, val1, val2); - elseif isa(val2, 'types.untyped.Set') - tests.util.verifySetEqual(testCase, val1, val2, failmsg); - elseif isdatetime(val2) - testCase.verifyEqual(char(val1), char(val2), failmsg); - elseif ischar(val2) - testCase.verifyEqual(char(val1), val2, failmsg); - elseif isa(val2, 'types.untyped.ObjectView') - testCase.verifyEqual(val1.path, val2.path, failmsg); - elseif isa(val2, 'types.untyped.RegionView') - testCase.verifyEqual(val1.path, val2.path, failmsg); - testCase.verifyEqual(val1.region, val2.region, failmsg); - elseif isa(val2, 'types.untyped.Anon') - testCase.verifyEqual(val1.name, val2.name, failmsg); - tests.util.verifyContainerEqual(testCase, val1.value, val2.value); + if startsWith(class(expectedVal), 'types.') && ~startsWith(class(expectedVal), 'types.untyped') + tests.util.verifyContainerEqual(testCase, actualVal, expectedVal); + elseif isa(expectedVal, 'types.untyped.Set') + tests.util.verifySetEqual(testCase, actualVal, expectedVal, failmsg); + elseif isdatetime(expectedVal) + testCase.verifyEqual(char(actualVal), char(expectedVal), failmsg); + elseif ischar(expectedVal) + testCase.verifyEqual(char(actualVal), expectedVal, failmsg); + elseif isa(expectedVal, 'types.untyped.ObjectView') + testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); + elseif isa(expectedVal, 'types.untyped.RegionView') + testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); + testCase.verifyEqual(actualVal.region, expectedVal.region, failmsg); + elseif isa(expectedVal, 'types.untyped.Anon') + testCase.verifyEqual(actualVal.name, expectedVal.name, failmsg); + tests.util.verifyContainerEqual(testCase, actualVal.value, expectedVal.value); + elseif isa(expectedVal, 'types.untyped.SoftLink') + testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); else - testCase.verifyEqual(val1, val2, failmsg); + if strcmp(prop, 'file_create_date') + % file_create_date is a very special property in NWBFile which can + % be many array formats and either a datetime or not. + % as such, we rely on the superpower of checkDtype to coerce + % the type for us. + actualVal = types.util.checkDtype('file_create_date', 'isodatetime', actualVal); + end + testCase.verifyEqual(actualVal, expectedVal, failmsg); end end end \ No newline at end of file From bac337673e294e238c017d7067fe6734d4dff136 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 12:25:03 -0400 Subject: [PATCH 08/32] Remove datetime check. --- +tests/+util/verifyContainerEqual.m | 2 -- 1 file changed, 2 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index f0723435..7a1ef298 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -15,8 +15,6 @@ function verifyContainerEqual(testCase, actual, expected) tests.util.verifyContainerEqual(testCase, actualVal, expectedVal); elseif isa(expectedVal, 'types.untyped.Set') tests.util.verifySetEqual(testCase, actualVal, expectedVal, failmsg); - elseif isdatetime(expectedVal) - testCase.verifyEqual(char(actualVal), char(expectedVal), failmsg); elseif ischar(expectedVal) testCase.verifyEqual(char(actualVal), expectedVal, failmsg); elseif isa(expectedVal, 'types.untyped.ObjectView') From 714c77a20c37c58779188dc17c8ae8ac1aba99f5 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 13:05:29 -0400 Subject: [PATCH 09/32] Revert datetime check in tests. testing linux capabilities using datetime. --- +tests/+util/verifyContainerEqual.m | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 7a1ef298..383d8f3b 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -17,7 +17,7 @@ function verifyContainerEqual(testCase, actual, expected) tests.util.verifySetEqual(testCase, actualVal, expectedVal, failmsg); elseif ischar(expectedVal) testCase.verifyEqual(char(actualVal), expectedVal, failmsg); - elseif isa(expectedVal, 'types.untyped.ObjectView') + elseif isa(expectedVal, 'types.untyped.ObjectView') || isa(expectedVal, 'types.untyped.SoftLink') testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); elseif isa(expectedVal, 'types.untyped.RegionView') testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); @@ -25,8 +25,11 @@ function verifyContainerEqual(testCase, actual, expected) elseif isa(expectedVal, 'types.untyped.Anon') testCase.verifyEqual(actualVal.name, expectedVal.name, failmsg); tests.util.verifyContainerEqual(testCase, actualVal.value, expectedVal.value); - elseif isa(expectedVal, 'types.untyped.SoftLink') - testCase.verifyEqual(actualVal.path, expectedVal.path, failmsg); + elseif isdatetime(expectedVal) + % ubuntu MATLAB doesn't appear to propery compare datetimes whereas + % Windows MATLAB does. This is a workaround to get tests to work + % while getting close enough to exact date representation. + testCase.verifyEqual(char(actualVal), char(expectedVal), failmsg); else if strcmp(prop, 'file_create_date') % file_create_date is a very special property in NWBFile which can From 260389f4f876f1e79511541a993e2930dd73a179 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 13:36:36 -0400 Subject: [PATCH 10/32] Further changes to datetime testing Now uses `ntp` format to directly test the precision instead of the potentially rounded char format. --- +tests/+util/verifyContainerEqual.m | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 383d8f3b..86993bed 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -25,19 +25,23 @@ function verifyContainerEqual(testCase, actual, expected) elseif isa(expectedVal, 'types.untyped.Anon') testCase.verifyEqual(actualVal.name, expectedVal.name, failmsg); tests.util.verifyContainerEqual(testCase, actualVal.value, expectedVal.value); - elseif isdatetime(expectedVal) + elseif isdatetime(expectedVal)... + || (iscell(expectedVal) && all(cellfun('isclass', expectedVal, 'datetime'))) % ubuntu MATLAB doesn't appear to propery compare datetimes whereas % Windows MATLAB does. This is a workaround to get tests to work % while getting close enough to exact date representation. - testCase.verifyEqual(char(actualVal), char(expectedVal), failmsg); - else - if strcmp(prop, 'file_create_date') - % file_create_date is a very special property in NWBFile which can - % be many array formats and either a datetime or not. - % as such, we rely on the superpower of checkDtype to coerce - % the type for us. - actualVal = types.util.checkDtype('file_create_date', 'isodatetime', actualVal); + actualVal = types.util.checkDtype(prop, 'isodatetime', actualVal); + if ~iscell(expectedVal) + actualVal = {actualVal}; + expectedVal = {expectedVal}; + end + for iDates = 1:length(expectedVal) + testCase.verifyEqual(... + convertTo(actualVal{iDates}, 'ntp'),... + convertTo(expectedVal{iDates}, 'ntp'),... + failmsg); end + else testCase.verifyEqual(actualVal, expectedVal, failmsg); end end From 26c98967eb81a3c856e9be00e198b2840a0ad193 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 13:56:07 -0400 Subject: [PATCH 11/32] Swap datetime ntp check to ntfs Going from 2e-32 seconds per tick to 100ns --- +tests/+util/verifyContainerEqual.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 86993bed..777ee1fc 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -37,8 +37,8 @@ function verifyContainerEqual(testCase, actual, expected) end for iDates = 1:length(expectedVal) testCase.verifyEqual(... - convertTo(actualVal{iDates}, 'ntp'),... - convertTo(expectedVal{iDates}, 'ntp'),... + convertTo(actualVal{iDates}, 'ntfs'),... + convertTo(expectedVal{iDates}, 'ntfs'),... failmsg); end else From be5ae24e1f5fa530cd6a0fcfdb831e8be4d5d518 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 15:21:29 -0400 Subject: [PATCH 12/32] Fix pynwb IO reference incompatibility The ObjectView path is now lazy so we must use a workaround in exporting a temporary file to guarantee that the MATLAB ObjectView path matches what is in the PyNWB output path. --- +tests/+system/PyNWBIOTest.m | 1 + 1 file changed, 1 insertion(+) diff --git a/+tests/+system/PyNWBIOTest.m b/+tests/+system/PyNWBIOTest.m index e0451171..1417d649 100644 --- a/+tests/+system/PyNWBIOTest.m +++ b/+tests/+system/PyNWBIOTest.m @@ -25,6 +25,7 @@ function testInFromPyNWB(testCase) pyfile = nwbRead(filename); pycontainer = testCase.getContainer(pyfile); matcontainer = testCase.getContainer(testCase.file); + nwbExport(testCase.file, 'temp.nwb'); % hack to fill out ObjectView container paths. tests.util.verifyContainerEqual(testCase, pycontainer, matcontainer); end end From d23cd14c861a717b6355a0939b7c4e1f40f84e1a Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 16:12:55 -0400 Subject: [PATCH 13/32] schema `int` now default 32-bit sized Previously was 64-bit, now 32-bit for pynwb 2.0 compatibility. --- +types/+util/correctType.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/+types/+util/correctType.m b/+types/+util/correctType.m index fa59e6a6..fc0fd792 100644 --- a/+types/+util/correctType.m +++ b/+types/+util/correctType.m @@ -14,9 +14,9 @@ % end elseif startsWith(type, 'int') || startsWith(type, 'uint') if strcmp(type, 'int') - val = int64(val); + val = int32(val); elseif strcmp(type, 'uint') - val = uint64(val); + val = uint32(val); else val = feval(fitIntType(val, type), val); end From 5f55deff4d31888f20f5683d38126b3578e2c29d Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 16:13:50 -0400 Subject: [PATCH 14/32] Fix PyNWB IO tests Due to reference view changes, certain PyNWBIO tests cease to be A2A comparisons. We now add certain testing changes which allow for ignoreLists for certain mutated properties (file_create_timestamp). We also allow a workaround that populates the View/SoftLink `path` changes. --- +tests/+system/NWBFileIOTest.m | 2 +- +tests/+system/PyNWBIOTest.m | 96 +++++++++++++++-------------- +tests/+util/verifyContainerEqual.m | 16 ++++- 3 files changed, 63 insertions(+), 51 deletions(-) diff --git a/+tests/+system/NWBFileIOTest.m b/+tests/+system/NWBFileIOTest.m index 51ac23c3..dd385905 100644 --- a/+tests/+system/NWBFileIOTest.m +++ b/+tests/+system/NWBFileIOTest.m @@ -2,7 +2,7 @@ methods function addContainer(testCase, file) %#ok ts = types.core.TimeSeries(... - 'data', int64(100:10:190) .', ... + 'data', int32(100:10:190) .', ... 'data_unit', 'SIunit', ... 'timestamps', (0:9) .', ... 'data_resolution', 0.1); diff --git a/+tests/+system/PyNWBIOTest.m b/+tests/+system/PyNWBIOTest.m index 1417d649..1f2383e5 100644 --- a/+tests/+system/PyNWBIOTest.m +++ b/+tests/+system/PyNWBIOTest.m @@ -1,52 +1,54 @@ classdef PyNWBIOTest < tests.system.RoundTripTest - % Assumes PyNWB and unittest2 has been installed on the system. - % - % To install PyNWB, execute: - % $ pip install pynwb - % - % To install unittest2, execute: - % $ pip install unittest2 - methods(Test) - function testOutToPyNWB(testCase) - filename = ['MatNWB.' testCase.className() '.testOutToPyNWB.nwb']; - nwbExport(testCase.file, filename); - [status, cmdout] = testCase.runPyTest('testInFromMatNWB'); - if status - testCase.verifyFail(cmdout); - end + % Assumes PyNWB and unittest2 has been installed on the system. + % + % To install PyNWB, execute: + % $ pip install pynwb + % + % To install unittest2, execute: + % $ pip install unittest2 + methods(Test) + function testOutToPyNWB(testCase) + filename = ['MatNWB.' testCase.className() '.testOutToPyNWB.nwb']; + nwbExport(testCase.file, filename); + [status, cmdout] = testCase.runPyTest('testInFromMatNWB'); + if status + testCase.verifyFail(cmdout); + end + end + + function testInFromPyNWB(testCase) + [status, cmdout] = testCase.runPyTest('testOutToMatNWB'); + if status + testCase.assertFail(cmdout); + end + filename = ['PyNWB.' testCase.className() '.testOutToMatNWB.nwb']; + pyfile = nwbRead(filename); + pycontainer = testCase.getContainer(pyfile); + matcontainer = testCase.getContainer(testCase.file); + nwbExport(testCase.file, 'temp.nwb'); % hack to fill out ObjectView container paths. + % ignore file_create_date because nwbExport will actually + % mutate the property every export. + tests.util.verifyContainerEqual(testCase, pycontainer, matcontainer, {'file_create_date'}); + end end - function testInFromPyNWB(testCase) - [status, cmdout] = testCase.runPyTest('testOutToMatNWB'); - if status - testCase.assertFail(cmdout); - end - filename = ['PyNWB.' testCase.className() '.testOutToMatNWB.nwb']; - pyfile = nwbRead(filename); - pycontainer = testCase.getContainer(pyfile); - matcontainer = testCase.getContainer(testCase.file); - nwbExport(testCase.file, 'temp.nwb'); % hack to fill out ObjectView container paths. - tests.util.verifyContainerEqual(testCase, pycontainer, matcontainer); + methods + function [status, cmdout] = runPyTest(testCase, testName) + setenv('PYTHONPATH', fileparts(mfilename('fullpath'))); + + envPath = fullfile('+tests', 'env.mat'); + if 2 == exist(envPath, 'file') + Env = load(envPath, '-mat', 'pythonDir'); + + pythonPath = fullfile(Env.pythonDir, 'python'); + else + pythonPath = 'python'; + end + + cmd = sprintf('"%s" -B -m unittest %s.%s.%s',... + pythonPath,... + 'PyNWBIOTest', testCase.className(), testName); + [status, cmdout] = system(cmd); + end end - end - - methods - function [status, cmdout] = runPyTest(testCase, testName) - setenv('PYTHONPATH', fileparts(mfilename('fullpath'))); - - envPath = fullfile('+tests', 'env.mat'); - if 2 == exist(envPath, 'file') - Env = load(envPath, '-mat', 'pythonDir'); - - pythonPath = fullfile(Env.pythonDir, 'python'); - else - pythonPath = 'python'; - end - - cmd = sprintf('"%s" -B -m unittest %s.%s.%s',... - pythonPath,... - 'PyNWBIOTest', testCase.className(), testName); - [status, cmdout] = system(cmd); - end - end end \ No newline at end of file diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 777ee1fc..1bdf23d8 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -1,8 +1,16 @@ -function verifyContainerEqual(testCase, actual, expected) +function verifyContainerEqual(testCase, actual, expected, ignoreList) +if nargin < 4 + ignoreList = {}; +end +assert(iscellstr(ignoreList),... + 'MatNWB:Test:InvalidIgnoreList',... + ['Ignore List must be a cell array of character arrays indicating props that should be '... + 'ignored.']); testCase.verifyEqual(class(actual), class(expected)); -props = properties(actual); +props = setdiff(properties(actual), ignoreList); for i = 1:numel(props) prop = props{i}; + actualVal = actual.(prop); expectedVal = expected.(prop); failmsg = ['Values for property ''' prop ''' are not equal']; @@ -32,9 +40,11 @@ function verifyContainerEqual(testCase, actual, expected) % while getting close enough to exact date representation. actualVal = types.util.checkDtype(prop, 'isodatetime', actualVal); if ~iscell(expectedVal) - actualVal = {actualVal}; expectedVal = {expectedVal}; end + if ~iscell(actualVal) + actualVal = {actualVal}; + end for iDates = 1:length(expectedVal) testCase.verifyEqual(... convertTo(actualVal{iDates}, 'ntfs'),... From 1c71b4b60b8003b401d9beaaaa8ade714f6e3341 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 16:52:56 -0400 Subject: [PATCH 15/32] Revert 64-bit int size change Apparantly, this is a bug with Anaconda builds for Windows. Reverted default `int` size --- +types/+util/correctType.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/+types/+util/correctType.m b/+types/+util/correctType.m index fc0fd792..fa59e6a6 100644 --- a/+types/+util/correctType.m +++ b/+types/+util/correctType.m @@ -14,9 +14,9 @@ % end elseif startsWith(type, 'int') || startsWith(type, 'uint') if strcmp(type, 'int') - val = int32(val); + val = int64(val); elseif strcmp(type, 'uint') - val = uint32(val); + val = uint64(val); else val = feval(fitIntType(val, type), val); end From 9c2f11b8a3b197d288d95edf13da5ae672691e3b Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 16:58:04 -0400 Subject: [PATCH 16/32] test commit for linux regex error --- +types/+util/checkDtype.m | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 84d82bae..250a06ca 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -239,8 +239,14 @@ % +-hhmm % +-hh % Z + tzre_pattern = '(?:[+-]\d{2}(?::?\d{2})?|Z)$'; tzre_match = regexp(datestr, tzre_pattern, 'once'); + +if endsWith(datestr, 'Z') + error('Test: `%s`, `%s`, `%d`', datestr, tzre_pattern, tzre_match); +end + if isempty(tzre_match) timezone = 'local'; else From c05c0349c67954dec2b67b9488616b2246a711d3 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 17:07:47 -0400 Subject: [PATCH 17/32] linux error alt test --- +types/+util/checkDtype.m | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 250a06ca..472846f5 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -243,16 +243,13 @@ tzre_pattern = '(?:[+-]\d{2}(?::?\d{2})?|Z)$'; tzre_match = regexp(datestr, tzre_pattern, 'once'); -if endsWith(datestr, 'Z') - error('Test: `%s`, `%s`, `%d`', datestr, tzre_pattern, tzre_match); -end - if isempty(tzre_match) timezone = 'local'; else timezone = datestr(tzre_match:end); if strcmp(timezone, 'Z') timezone = 'UTC'; + error('new date_str: %s', datestr(1:(tzre_match - 1))); end datestr = datestr(1:(tzre_match - 1)); end From 85ec804420ab248484acccb7f7b2453603b9561e Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 17:24:35 -0400 Subject: [PATCH 18/32] fix 64-bit tests --- +tests/+system/NWBFileIOTest.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/+tests/+system/NWBFileIOTest.m b/+tests/+system/NWBFileIOTest.m index dd385905..51ac23c3 100644 --- a/+tests/+system/NWBFileIOTest.m +++ b/+tests/+system/NWBFileIOTest.m @@ -2,7 +2,7 @@ methods function addContainer(testCase, file) %#ok ts = types.core.TimeSeries(... - 'data', int32(100:10:190) .', ... + 'data', int64(100:10:190) .', ... 'data_unit', 'SIunit', ... 'timestamps', (0:9) .', ... 'data_resolution', 0.1); From c6690deca6915446e970ce2cca1dd6f014543ef5 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 17:24:48 -0400 Subject: [PATCH 19/32] use stdout instead of errors for test logging --- +types/+util/checkDtype.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 472846f5..73389d50 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -227,7 +227,11 @@ format = sprintf('%s.%s', format, repmat('S', 1, seconds_precision)); end +oldDateStr = datestr; [datestr, timezone] = derive_timezone(datestr); +if endsWith(datestr, 'Z') + fprintf('derive_timezone: %s -> %s\n', oldDateStr, datestr); +end date_time = datetime(datestr,... 'InputFormat', format,... 'TimeZone', timezone); @@ -249,7 +253,6 @@ timezone = datestr(tzre_match:end); if strcmp(timezone, 'Z') timezone = 'UTC'; - error('new date_str: %s', datestr(1:(tzre_match - 1))); end datestr = datestr(1:(tzre_match - 1)); end From 77135ba48b00a785ff4f4aa79607d3a568e2c23d Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 17:36:01 -0400 Subject: [PATCH 20/32] sanity print CI test --- +types/+util/checkDtype.m | 4 +--- +types/+util/correctType.m | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 73389d50..adc21ed8 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -229,9 +229,7 @@ oldDateStr = datestr; [datestr, timezone] = derive_timezone(datestr); -if endsWith(datestr, 'Z') - fprintf('derive_timezone: %s -> %s\n', oldDateStr, datestr); -end +fprintf('derive_timezone: %s -> %s\n', oldDateStr, datestr); date_time = datetime(datestr,... 'InputFormat', format,... 'TimeZone', timezone); diff --git a/+types/+util/correctType.m b/+types/+util/correctType.m index fa59e6a6..ebfeb25e 100644 --- a/+types/+util/correctType.m +++ b/+types/+util/correctType.m @@ -6,12 +6,7 @@ %check different types and correct if startsWith(type, 'float') -% Compatibility with PyNWB -% if strcmp(type, 'float32') -% val = single(val); -% else - val = double(val); -% end + val = double(val); elseif startsWith(type, 'int') || startsWith(type, 'uint') if strcmp(type, 'int') val = int64(val); From 0deb18722dac4b4f6381bb1de874c261baf79bd7 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 17:44:07 -0400 Subject: [PATCH 21/32] trim datestr when read --- +types/+util/checkDtype.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index adc21ed8..506bba71 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -130,7 +130,7 @@ datevals = cell(size(val)); for i = 1:length(val) - datevals{i} = datetime8601(val{i}); + datevals{i} = datetime8601(strtrim(val{i})); end val = datevals; end From d66ae6f5886f74766a1f6a65f33de91948535670 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Mon, 16 Aug 2021 18:13:28 -0400 Subject: [PATCH 22/32] Allow differing integer sizes in tests Given how the schema defines types, it is entirely possible that the pynwb implementation will use larger integer sizes than MatNWB (which uses whatever's listed in the schema). --- +tests/+util/verifyContainerEqual.m | 10 ++++++++++ +types/+util/checkDtype.m | 2 -- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 1bdf23d8..f6e05bbe 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -51,6 +51,16 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) convertTo(expectedVal{iDates}, 'ntfs'),... failmsg); end + elseif startsWith(class(expectedVal), 'int') || startsWith(class(expectedVal), 'uint') + actualTypeSize = regexp(class(actualVal), 'int(\d+)', 'once', 'tokens'); + expectedTypeSize = regexp(class(expectedVal), 'int(\d+)', 'once', 'tokens'); + testCase.verifyGreaterThanOrEqual(actualTypeSize{1}, expectedTypeSize{1}); + + if startsWith(class(expectedVal), 'int') + testCase.verifyEqual(int64(actualVal), int64(expectedVal), failmsg); + else + testCase.verifyEqual(uint64(actualVal), uint64(expectedVal), failmsg); + end else testCase.verifyEqual(actualVal, expectedVal, failmsg); end diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 506bba71..1492a0c8 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -227,9 +227,7 @@ format = sprintf('%s.%s', format, repmat('S', 1, seconds_precision)); end -oldDateStr = datestr; [datestr, timezone] = derive_timezone(datestr); -fprintf('derive_timezone: %s -> %s\n', oldDateStr, datestr); date_time = datetime(datestr,... 'InputFormat', format,... 'TimeZone', timezone); From 4bdffc292117a15e09dc4886fe0dff1666431b5e Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 10:55:52 -0400 Subject: [PATCH 23/32] export -> nwbExport --- +tests/+unit/anonTest.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/+tests/+unit/anonTest.m b/+tests/+unit/anonTest.m index 6e470803..ec9d9338 100644 --- a/+tests/+unit/anonTest.m +++ b/+tests/+unit/anonTest.m @@ -23,7 +23,7 @@ function testAnonDataset(testCase) 'session_description', 'anonymous class schema testing',... 'session_start_time', datetime()); nwbExpected.acquisition.set('ag', ag); -nwbExpected.export('testanon.nwb'); +nwbExport(nwbExpected, 'testanon.nwb'); tests.util.verifyContainerEqual(testCase, nwbRead('testanon.nwb'), nwbExpected); end From f78e97ef85b9e357f8da354a1bd55153bddb0180 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 10:56:23 -0400 Subject: [PATCH 24/32] linux debug printing --- +types/+util/checkDtype.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 1492a0c8..03ee7ef6 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -230,7 +230,9 @@ [datestr, timezone] = derive_timezone(datestr); date_time = datetime(datestr,... 'InputFormat', format,... - 'TimeZone', timezone); + 'TimeZone', timezone,... + 'Format', 'yyyy-MM-dd''T''HH:mm:ss.SSSSSSZZZZZ'); +fprintf('%s -> %s\n', datestr, char(datetime)); end function [datestr, timezone] = derive_timezone(datestr) From ba7865fe1fbe165c74f9518c1eda7f36f81c6baf Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 11:00:43 -0400 Subject: [PATCH 25/32] Clarify datetime print for linux testing --- +types/+util/checkDtype.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 03ee7ef6..1a34f1ec 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -230,8 +230,8 @@ [datestr, timezone] = derive_timezone(datestr); date_time = datetime(datestr,... 'InputFormat', format,... - 'TimeZone', timezone,... - 'Format', 'yyyy-MM-dd''T''HH:mm:ss.SSSSSSZZZZZ'); + 'TimeZone', timezone); +date_time.Format = 'yyyy-MM-dd''T''HH:mm:ss.SSSSSSZZZZZ'; fprintf('%s -> %s\n', datestr, char(datetime)); end From db8b72cab465c7eec74e58241b264d7ee2156ef9 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 11:08:14 -0400 Subject: [PATCH 26/32] Fix clarification of previous debug print --- +types/+util/checkDtype.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 1a34f1ec..73701e48 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -182,7 +182,7 @@ end end -function date_time = datetime8601(datestr) +function dt = datetime8601(datestr) addpath(fullfile(fileparts(which('NwbFile')), 'external_packages', 'datenum8601')); [~, ~, format] = datenum8601(datestr); format = format{1}; @@ -228,11 +228,11 @@ end [datestr, timezone] = derive_timezone(datestr); -date_time = datetime(datestr,... +dt = datetime(datestr,... 'InputFormat', format,... 'TimeZone', timezone); -date_time.Format = 'yyyy-MM-dd''T''HH:mm:ss.SSSSSSZZZZZ'; -fprintf('%s -> %s\n', datestr, char(datetime)); +dt.Format = 'yyyy-MM-dd''T''HH:mm:ss.SSSSSSZZZZZ'; +fprintf('%s -> %s\n', datestr, char(dt)); end function [datestr, timezone] = derive_timezone(datestr) From b47d63a6ec31dbeffc03e4b9b2c20175f82395bf Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 11:20:06 -0400 Subject: [PATCH 27/32] Revert debug changes. revert to format comparison. --- +tests/+util/verifyContainerEqual.m | 6 +++--- +types/+util/checkDtype.m | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index f6e05bbe..3e1c9726 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -35,7 +35,7 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) tests.util.verifyContainerEqual(testCase, actualVal.value, expectedVal.value); elseif isdatetime(expectedVal)... || (iscell(expectedVal) && all(cellfun('isclass', expectedVal, 'datetime'))) - % ubuntu MATLAB doesn't appear to propery compare datetimes whereas + % linux MATLAB doesn't appear to propery compare datetimes whereas % Windows MATLAB does. This is a workaround to get tests to work % while getting close enough to exact date representation. actualVal = types.util.checkDtype(prop, 'isodatetime', actualVal); @@ -47,8 +47,8 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) end for iDates = 1:length(expectedVal) testCase.verifyEqual(... - convertTo(actualVal{iDates}, 'ntfs'),... - convertTo(expectedVal{iDates}, 'ntfs'),... + char(actualVal{iDates}),... + char(expectedVal{iDates}),... failmsg); end elseif startsWith(class(expectedVal), 'int') || startsWith(class(expectedVal), 'uint') diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index 73701e48..aca72339 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -231,8 +231,6 @@ dt = datetime(datestr,... 'InputFormat', format,... 'TimeZone', timezone); -dt.Format = 'yyyy-MM-dd''T''HH:mm:ss.SSSSSSZZZZZ'; -fprintf('%s -> %s\n', datestr, char(dt)); end function [datestr, timezone] = derive_timezone(datestr) From 0a80ecead74412ef95a9b413a5e6f094444a1cea Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 12:08:25 -0400 Subject: [PATCH 28/32] More minor refactor --- +tests/+sanity/GenerationTest.m | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/+tests/+sanity/GenerationTest.m b/+tests/+sanity/GenerationTest.m index ee45dc82..cd53df57 100644 --- a/+tests/+sanity/GenerationTest.m +++ b/+tests/+sanity/GenerationTest.m @@ -23,10 +23,8 @@ function roundtripTest(testCase) expected = NwbFile('identifier', 'TEST',... 'session_description', 'test nwbfile',... 'session_start_time', datetime()); - expected.export('empty.nwb'); - - actual = nwbRead('empty.nwb'); - tests.util.verifyContainerEqual(testCase, actual, expected); + nwbExport(expected, 'empty.nwb'); + tests.util.verifyContainerEqual(testCase, nwbRead('empty.nwb'), expected); end function dynamicTableMethodsTest(testCase) From 5499998e9f82e78607d884a8e8d37363dac95099 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 12:20:16 -0400 Subject: [PATCH 29/32] Add testing comparison error for datetimes. --- +tests/+util/verifyContainerEqual.m | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 3e1c9726..90ab2813 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -18,7 +18,7 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) if isa(actualVal, 'types.untyped.DataStub') actualVal = actualVal.load(); end - + if startsWith(class(expectedVal), 'types.') && ~startsWith(class(expectedVal), 'types.untyped') tests.util.verifyContainerEqual(testCase, actualVal, expectedVal); elseif isa(expectedVal, 'types.untyped.Set') @@ -46,10 +46,12 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) actualVal = {actualVal}; end for iDates = 1:length(expectedVal) - testCase.verifyEqual(... - char(actualVal{iDates}),... - char(expectedVal{iDates}),... - failmsg); + % ignore microseconds as linux datetime has some strange error + % even when datetime doesn't change in Windows. + testCase.verifyEqual(... + convertTo(actualVal{iDates}, 'ntfs') / 100,... + convertTo(expectedVal{iDates}, 'ntfs') / 100,... + failmsg); end elseif startsWith(class(expectedVal), 'int') || startsWith(class(expectedVal), 'uint') actualTypeSize = regexp(class(actualVal), 'int(\d+)', 'once', 'tokens'); From 1d807ffd196710baa250d470e1690aae12855096 Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 12:24:49 -0400 Subject: [PATCH 30/32] Increase error to 10 microseconds --- +tests/+util/verifyContainerEqual.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index 90ab2813..f13ec987 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -49,8 +49,8 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) % ignore microseconds as linux datetime has some strange error % even when datetime doesn't change in Windows. testCase.verifyEqual(... - convertTo(actualVal{iDates}, 'ntfs') / 100,... - convertTo(expectedVal{iDates}, 'ntfs') / 100,... + convertTo(actualVal{iDates}, 'ntfs') / 1000,... + convertTo(expectedVal{iDates}, 'ntfs') / 1000,... failmsg); end elseif startsWith(class(expectedVal), 'int') || startsWith(class(expectedVal), 'uint') From 6361b507a2172c15af3afe0a8a9ffc39c99efe0c Mon Sep 17 00:00:00 2001 From: Lawrence Date: Tue, 17 Aug 2021 12:30:16 -0400 Subject: [PATCH 31/32] Make more precise datetime error comparison --- +tests/+util/verifyContainerEqual.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/+tests/+util/verifyContainerEqual.m b/+tests/+util/verifyContainerEqual.m index f13ec987..ce012809 100644 --- a/+tests/+util/verifyContainerEqual.m +++ b/+tests/+util/verifyContainerEqual.m @@ -48,10 +48,10 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList) for iDates = 1:length(expectedVal) % ignore microseconds as linux datetime has some strange error % even when datetime doesn't change in Windows. - testCase.verifyEqual(... - convertTo(actualVal{iDates}, 'ntfs') / 1000,... - convertTo(expectedVal{iDates}, 'ntfs') / 1000,... - failmsg); + actualNtfs = convertTo(actualVal{iDates}, 'ntfs'); + expectedNtfs = convertTo(expectedVal{iDates}, 'ntfs'); + testCase.verifyGreaterThanOrEqual(actualNtfs, expectedNtfs - 10, failmsg); + testCase.verifyLessThanOrEqual(actualNtfs, expectedNtfs + 10, failmsg); end elseif startsWith(class(expectedVal), 'int') || startsWith(class(expectedVal), 'uint') actualTypeSize = regexp(class(actualVal), 'int(\d+)', 'once', 'tokens'); From 33533fd04dee3a9d4738b56e0958b21c8f92cd9e Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 17 Aug 2021 13:52:35 -0400 Subject: [PATCH 32/32] update utils and tutorials * create_indexed_column no longer needs the path * update ecephys tutorial to use objects instead of paths * generate new html * update ophys tutorial to use objects instead of paths * generate new html --- +util/createElectrodeTable.m | 20 +- +util/createUnitTimes.m | 18 -- +util/create_indexed_column.m | 51 +++-- +util/create_spike_times.m | 16 -- +util/read_indexed_column.m | 10 +- +util/table2nwb.m | 2 +- tutorials/ecephys.m | 350 ---------------------------- tutorials/ecephys.mlx | Bin 498396 -> 498361 bytes tutorials/html/ecephys.html | 299 ++++++++++++------------ tutorials/html/ophys.html | 415 +++++++++++++++++----------------- tutorials/ophys.mlx | Bin 309921 -> 309905 bytes 11 files changed, 409 insertions(+), 772 deletions(-) delete mode 100644 +util/createUnitTimes.m delete mode 100644 +util/create_spike_times.m delete mode 100644 tutorials/ecephys.m diff --git a/+util/createElectrodeTable.m b/+util/createElectrodeTable.m index 7dea5069..e2568f51 100644 --- a/+util/createElectrodeTable.m +++ b/+util/createElectrodeTable.m @@ -1,24 +1,24 @@ function table = createElectrodeTable() table = types.hdmf_common.DynamicTable(... + 'description', 'A table of all electrodes (i.e. channels) used for recording.', ... 'colnames', {'x', 'y', 'z', 'imp', 'location', 'filtering', 'group', 'group_name'},... 'id', types.hdmf_common.ElementIdentifiers('data', []),... 'x', types.hdmf_common.VectorData('data', [],... - 'description', 'the x coordinate of the channel location'),... + 'description', 'the x coordinate of the channel location'),... 'y', types.hdmf_common.VectorData('data', [],... - 'description', 'the y coordinate of the channel location'),... + 'description', 'the y coordinate of the channel location'),... 'z', types.hdmf_common.VectorData('data', [],... - 'description', 'the z coordinate of the channel location'),... + 'description', 'the z coordinate of the channel location'),... 'imp', types.hdmf_common.VectorData('data', [],... - 'description', 'the impedance of the channel'),... + 'description', 'the impedance of the channel'),... 'location', types.hdmf_common.VectorData('data', {},... - 'description', 'the location of channel within the subject e.g. brain region'),... + 'description', 'the location of channel within the subject e.g. brain region'),... 'filtering', types.hdmf_common.VectorData('data', [],... - 'description', 'description of hardware filtering'),... + 'description', 'description of hardware filtering'),... 'group', types.hdmf_common.VectorData('data', types.untyped.ObjectView.empty,... - 'description', 'a reference to the ElectrodeGroup this electrodes is a part of'),... + 'description', 'a reference to the ElectrodeGroup this electrodes is a part of'),... 'group_name', types.hdmf_common.VectorData('data', {},... - 'description', 'name of the ElectrodeGroup this electrode is a part of'),... - 'description', 'A table of all electrodes (i.e. channels) used for recording.'... - ); + 'description', 'name of the ElectrodeGroup this electrode is a part of')... +); end diff --git a/+util/createUnitTimes.m b/+util/createUnitTimes.m deleted file mode 100644 index d42f88e3..00000000 --- a/+util/createUnitTimes.m +++ /dev/null @@ -1,18 +0,0 @@ -function UnitTimes = createUnitTimes(cluster_ids, spike_times, spike_loc) - -[sorted_cluster_ids, order] = sort(cluster_ids); -uids = unique(cluster_ids); -vdata = spike_times(order); -bounds = [0,find(diff(sorted_cluster_ids)),length(cluster_ids)]; - -vd = types.core.VectorData('data', vdata); - -vd_ref = types.untyped.RegionView(spike_loc, 1:bounds(2), size(vdata)); -for i = 2:length(bounds)-1 - vd_ref(end+1) = types.untyped.RegionView(spike_loc, bounds(i)+1:bounds(i+1)); -end - -vi = types.core.VectorIndex('data', vd_ref); -ei = types.core.ElementIdentifiers('data', int64(uids)); -UnitTimes = types.core.UnitTimes('spike_times', vd, ... - 'spike_times_index', vi, 'unit_ids', ei); diff --git a/+util/create_indexed_column.m b/+util/create_indexed_column.m index 5556f11d..2afc7b9b 100644 --- a/+util/create_indexed_column.m +++ b/+util/create_indexed_column.m @@ -1,20 +1,16 @@ -function [data_vector, data_index] = create_indexed_column(data, path, ids, description, table) +function [data_vector, data_index] = create_indexed_column(data, description, table) %CREATE_INDEXED_COLUMN creates the index and vector NWB objects for storing %a vector column in an NWB DynamicTable % -% [DATA_VECTOR, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA, PATH) +% [DATA_VECTOR, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA) % expects DATA as a cell array where each cell is all of the data % for a row and PATH is the path to the indexed data in the NWB file -% EXAMPLE: [data_vector, data_index] = util.create_indexed_colum({[1,2,3], [1,2,3,4]}, '/units/spike_times') -% -% [DATA_VECTOR, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA, PATH, IDS) -% expects DATA as a single array 1D of doubles and IDS as a single 1D array of ints. -% EXAMPLE: [data_vector, data_index] = util.create_indexed_colum([1,2,3,1,2,3,4], '/units/spike_times', [0,0,0,1,1,1,1]) +% EXAMPLE: [data_vector, data_index] = util.create_indexed_colum({[1,2,3], [1,2,3,4]}) % -% [DATA_VECTOR, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA, PATH, IDS, DESCRIPTION) +% [DATA_VECTOR, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA, DESCRIPTION) % adds the string DESCRIPTION in the description field of the data vector % -% [DYNAMICTABLEREGION, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA, PATH, IDS, DESCRIPTION, TABLE) +% [DYNAMICTABLEREGION, DATA_INDEX] = CREATE_INDEXED_COLUMN(DATA, DESCRIPTION, TABLE) % If TABLE is supplied as on ObjectView of an NWB DynamicTable, a % DynamicTableRegion is instead output which references this table. % DynamicTableRegions can be indexed just like DataVectors @@ -23,27 +19,32 @@ description = 'no description'; end -if ~exist('ids', 'var') || isempty(ids) - bounds = NaN(length(data),1); - for i = 1:length(data) - bounds(i) = length(data{i}); - end - bounds = int64(cumsum(bounds)); - data = cell2mat(data)'; -else - [sorted_ids, order] = sort(ids); - data = data(order); - bounds = int64([find(diff(sorted_ids)), length(ids)]); +bounds = NaN(length(data), 1); +for i = 1:length(data) + bounds(i) = length(data{i}); end +bounds = int64(cumsum(bounds)); + +data = cell2mat(data)'; if exist('table', 'var') - data_vector = types.hdmf_common.DynamicTableRegion('table', table, ... - 'description', description, 'data', data); + data_vector = types.hdmf_common.DynamicTableRegion( ... + 'table', table, ... + 'description', description, ... + 'data', data ... + ); else - data_vector = types.hdmf_common.VectorData('data', data, 'description', description); + data_vector = types.hdmf_common.VectorData( ... + 'data', data, ... + 'description', description ... + ); end -ov = types.untyped.ObjectView(path); -data_index = types.hdmf_common.VectorIndex('data', bounds, 'target', ov); +ov = types.untyped.ObjectView(data_vector); +data_index = types.hdmf_common.VectorIndex( ... + 'data', bounds, ... + 'target', ov, ... + 'description', 'indexes data' ... +); diff --git a/+util/create_spike_times.m b/+util/create_spike_times.m deleted file mode 100644 index 85d1186b..00000000 --- a/+util/create_spike_times.m +++ /dev/null @@ -1,16 +0,0 @@ -function [spike_times_vector, spike_times_index] = create_spike_times(cluster_ids, spike_times) - -[sorted_cluster_ids, order] = sort(cluster_ids); -bounds = [0,find(diff(sorted_cluster_ids)),length(cluster_ids)]; - -spike_times_vector = types.core.VectorData('data', spike_times(order),... - 'description','spike times for all units in seconds'); - -vd_ref = types.untyped.RegionView('/units/spike_times', 1:bounds(2), size(spike_times)); -for i = 2:length(bounds)-1 - vd_ref(end+1) = types.untyped.RegionView('/units/spike_times', bounds(i)+1:bounds(i+1)); -end - -ov = types.untyped.ObjectView('/units/spike_times'); - -spike_times_index = types.core.VectorIndex('data', vd_ref, 'target', ov); diff --git a/+util/read_indexed_column.m b/+util/read_indexed_column.m index 19ca6e73..8632decd 100644 --- a/+util/read_indexed_column.m +++ b/+util/read_indexed_column.m @@ -1,7 +1,7 @@ function data = read_indexed_column(vector_index, vector_data, row) %READ_INDEXED_COLUMN returns the data for a specific row of an indexed vector % -% DATA = READ_INDEXED_COLUMN(VECTOR_INDEX, ROW) takes a VectorIndex from a +% DATA = READ_INDEXED_COLUMN(VECTOR_INDEX, VECTRO_DATA, ROW) takes a VectorIndex from a % DynamicTable and a ROW number and outputs the DATA for that row (1-indexed). try @@ -18,12 +18,8 @@ lower_bound = vector_index.data(row - 1) + 1; end end -%% -% Then select the corresponding spike_times_index element -if isa(vector_data.data,'types.untyped.DataStub') - data = vector_data.data.load(lower_bound,upper_bound); -else - data = vector_data.data(lower_bound:upper_bound); + +data = vector_data.data(lower_bound:upper_bound); end diff --git a/+util/table2nwb.m b/+util/table2nwb.m index c97e1a72..b87a2d95 100644 --- a/+util/table2nwb.m +++ b/+util/table2nwb.m @@ -30,6 +30,6 @@ if ~strcmp(col.Properties.VariableNames{1},'id') nwbtable.vectordata.set(col.Properties.VariableNames{1}, ... types.hdmf_common.VectorData('data', col.Variables',... - 'description','my description')); + 'description', 'my description')); end end \ No newline at end of file diff --git a/tutorials/ecephys.m b/tutorials/ecephys.m deleted file mode 100644 index 96b8a300..00000000 --- a/tutorials/ecephys.m +++ /dev/null @@ -1,350 +0,0 @@ -%% Neurodata Without Borders: Neurophysiology (NWB:N), Extracellular Electrophysiology Tutorial -% How to write ecephys data to an NWB file using matnwb. -% -% author: Ben Dichter -% contact: ben.dichter@gmail.com -% last edited: Sept 27, 2019 - -%% NWB file -% All contents get added to the NWB file, which is created with the -% following command - -session_start_time = datetime(2018, 3, 1, 12, 0, 0, 'TimeZone', 'local'); - -nwb = NwbFile( ... - 'session_description', 'a test NWB File', ... - 'identifier', 'mouse004_day4', ... - 'session_start_time', session_start_time); - -%% -% You can check the contents by displaying the NwbFile object -disp(nwb); - -%% Subject -% Subject-specific information goes in type |Subject| in location -% |general_subject|. - -nwb.general_subject = types.core.Subject( ... - 'description', 'mouse 5', 'age', '9 months', ... - 'sex', 'M', 'species', 'Mus musculus'); - -%% Data dependencies -% The data needs to be added to nwb in a specific order, which is specified -% by the data dependencies in the schema. The data dependencies for LFP are -% illustrated in the following diagram. In order to write LFP, you need to -% specify what electrodes it came from. To do that, you first need to -% construct an electrode table. -%% -% -% <> -% -%% Electrode Table -% Electrode tables hold the position and group information about each -% electrode and the brain region and filtering. Groups organize electrodes -% within a single device. Devices can have 1 or more groups. In this example, -% we have 2 devices that each only have a single group. - -shank_channels = [3, 4]; -variables = {'x', 'y', 'z', 'imp', 'location', 'filtering', 'group', 'label'}; -device_name = 'implant'; -nwb.general_devices.set(device_name, types.core.Device()); -device_link = types.untyped.SoftLink(['/general/devices/' device_name]); -for ishank = 1:length(shank_channels) - nelecs = shank_channels(ishank); - group_name = ['shank' num2str(ishank)]; - nwb.general_extracellular_ephys.set(group_name, ... - types.core.ElectrodeGroup( ... - 'description', ['electrode group for shank' num2str(ishank)], ... - 'location', 'brain area', ... - 'device', device_link)); - group_object_view = types.untyped.ObjectView( ... - ['/general/extracellular_ephys/' group_name]); - for ielec = 1:length(nelecs) - if ishank == 1 && ielec == 1 - tbl = table(NaN, NaN, NaN, NaN, {'unknown'}, {'unknown'}, ... - group_object_view, {[group_name 'elec' num2str(ielec)]}, ... - 'VariableNames', variables); - else - tbl = [tbl; {NaN, NaN, NaN, NaN, 'unknown', 'unknown', ... - group_object_view, [group_name 'elec' num2str(ielec)]}]; - end - end -end - -% add the |DynamicTable| object to the NWB file in -% /general/extracellular_ephys/electrodes - -electrode_table = util.table2nwb(tbl, 'all electrodes'); -nwb.general_extracellular_ephys_electrodes = electrode_table; - -%% Multielectrode recording -% In order to write a multielectrode recording, you need to construct a -% region view of the electrode table to link the signal to the electrodes -% that generated them. You must do this even if the signal is from all of -% the electrodes. Here we will create a reference that includes all -% electrodes. Then we will generate a signal 1000 timepoints long from 10 -% channels. - -electrodes_object_view = types.untyped.ObjectView( ... - '/general/extracellular_ephys/electrodes'); - -electrode_table_region = types.hdmf_common.DynamicTableRegion( ... - 'table', electrodes_object_view, ... - 'description', 'all electrodes', ... - 'data', [0 height(tbl)-1]'); - -%% -% once you have the |ElectrodeTableRegion| object, you can create an -% |ElectricalSeries| object to hold your multielectrode data. An -% |ElectricalSeries| is an example of a |TimeSeries| object. For all -% |TimeSeries| objects, you have 2 options for storing time information. -% The first is to use |starting_time| and |rate|: - -% generate data for demonstration -data = reshape(1:10000, 10, 1000); - -electrical_series = types.core.ElectricalSeries( ... - 'starting_time', 0.0, ... % seconds - 'starting_time_rate', 200., ... % Hz - 'data', data, ... - 'electrodes', electrode_table_region, ... - 'data_unit', 'V'); - -nwb.acquisition.set('multielectrode_recording', electrical_series); -%% -% You can also specify time using |timestamps|. This is particularly useful if -% the sample times are not evenly sampled. In this case, the electrical series -% constructor will look like this - -electrical_series = types.core.ElectricalSeries(... - 'timestamps', (1:1000)/200, ... - 'data', data,... - 'electrodes', electrode_table_region,... - 'data_unit', 'V'); - -%% LFP -% Store LFP (generally downsampled and/or filtered data) as an -% |ElectricalSeries| in a processing module called |'ecephys'|. - -ecephys_module = types.core.ProcessingModule(... - 'description', 'holds extracellular electrophysiology data'); -ecephys_module.nwbdatainterface.set('LFP', ... - types.core.LFP('lfp', electrical_series)); -nwb.processing.set('ecephys', ecephys_module); - -%% Trials -% You can store trial information in the trials table - -trials = types.core.TimeIntervals( ... - 'colnames', {'correct','start_time','stop_time'}, ... - 'description', 'trial data and properties', ... - 'id', types.hdmf_common.ElementIdentifiers('data', 0:2), ... - 'start_time', types.hdmf_common.VectorData('data', [.1, 1.5, 2.5], ... - 'description','start time of trial'), ... - 'stop_time', types.hdmf_common.VectorData('data', [1., 2., 3.], ... - 'description','end of each trial'), ... - 'correct', types.hdmf_common.VectorData('data', [false, true, false], ... - 'description','my description')); - -nwb.intervals_trials = trials; - -%% -% |colnames| is flexible - it can store any column names and the entries can -% be any data type, which allows you to store any information you need about -% trials. -% -%% DynamicTables -% NWB makes use of the |DynamicTable| neurodata_type to deal with data -% organized in a table. These tables are powerful in that they can contain -% user-defined columns, can hold columns where each row is itself an array, -% and can reference other tables. We have already seen one example of -% a |DynamicTable| with the |electrodes| table. We will contruct a similar -% table for the |units| table, but this time we will show off the -% flexibility and customization of |DynamicTable|s. When adding a column to -% a |DynamicTable|, you need to ask yourself 3 questions: -% 1) Is this a default name value or a user-defined value -% 2) Is each element of this column itself an array (e.g. spike times) -% 3) Is this column referencing another column (e.g. electrodes)? -% -% Standard columns should be |VectorData| objects. -% -% 1) If the column is a default optional column, it can be added to the -% table by assigning the |VectorData| to a table attribute of the same name. -% This is illustrated by |waveform_mean| in the |units| table below. -% On the other hand, if the column is a custom name, it must be added by setting -% |DynamicTable.vectordata.set('name', VectorData)| (e.g. |'quality'|) -% below. You can see what the default column names for |units| are by -% typing |nwb.units|. |colnames|, |description|, |id|, |vectordata| -% and |help| are all |DynamicTable| properties, but the -% others are default columns. -% -% 2) Each row of a column is an array of varying length, you must add two objects, a -% |VectorData| and a |VectorIndex|. The |VectorData| object stores all of -% the data. For instance for spike times, it stores all of the spike times -% for the first cell, then the second, then the third, all the way up in a -% single array. The |VectorIndex| object indicates where to slice the -% |VectorData| object in order to get just the data for a specific row. -% A convenience function |util.create_indexed_column| is supplied to make -% the creation of these objects easier. Its usage is demonstrated for -% 'spike_times' and 'obs_invervals'. -% -% 3) If a column is a reference to another table, it should not be a -% |VectorData| but instead a |DynamicTableRegion| object, which can be -% indexed with a |VectorIndex| just like |VectorData| objects to create an -% array of references per row. You can create a |DynamicTableRegion| -% |VectorIndex| pair by using |util.create_indexed_column| and including -% the |ObjectView| of the table as the 5th argument. - -% First, instantiate the table, listing all of the columns that will be -% added and us the |'id'| argument to indicate the number of rows. Ifa -% value is indexed, only the column name is included, not the index. For -% instance, |'spike_times_index'| is not added to the array. -nwb.units = types.core.Units( ... - 'colnames', {'spike_times', 'waveform_mean', 'quality', 'electrodes'}, ... - 'description', 'units table', ... - 'id', types.hdmf_common.ElementIdentifiers('data', int64(0:2))); - -% Then you can add the data column-by-column: -waveform_mean = types.hdmf_common.VectorData('data', ones(30, 3), ... - 'description', 'mean of waveform'); -nwb.units.waveform_mean = waveform_mean; - -quality = types.hdmf_common.VectorData('data', [.9, .1, .2],... - 'description', 'sorting quality score out of 1'); -nwb.units.vectordata.set('quality', quality); - -spike_times_cells = {[0.1, 0.21, 0.5, 0.61], [0.34, 0.36, 0.66, 0.69], [0.4, 0.43]}; -[spike_times_vector, spike_times_index] = util.create_indexed_column( ... - spike_times_cells, '/units/spike_times'); -nwb.units.spike_times = spike_times_vector; -nwb.units.spike_times_index = spike_times_index; - -[electrodes, electrodes_index] = util.create_indexed_column( ... - {[0,1], [1,2], [2,3]}, '/units/electrodes', [], [], ... - electrodes_object_view); -nwb.units.electrodes = electrodes; -nwb.units.electrodes_index = electrodes_index; - -%% Side note about electrodes -% In the above example, we assigned multiple electrodes to each unit. This -% is useful in some recording setups, where electrodes are close together -% and multiple electrodes pick up signal from a single cell. In other -% instances, it makes more sense to only assign a single electrode per -% cell. In this case you do not need to define an |electrodes_index|. -% Instead, you can add electrodes like this: - -% clear electrodes_index (not generally necessary) -nwb.units.electrodes_index = []; - -% assign DynamicTableRegion object with n elements -nwb.units.electrodes = types.hdmf_common.DynamicTableRegion( ... - 'table', electrodes_object_view, ... - 'description', 'single electrodes', ... - 'data', int64([0, 0, 1])); - -%% Processing Modules -% Measurements go in |acquisition| and subject or session data goes in -% |general|, but if you have the intermediate processing results, you -% should put them in a processing module. - -behavior_mod = types.core.ProcessingModule('description', 'contains behavioral data'); - -%% -% Position data is stored first in a |SpatialSeries| object, which is a -% |TimeSeries|. This is then stored in |Position| -% (a |MultiContainerInterface|), which is stored in a processing module - -position_data = [linspace(0,10,100); linspace(1,8,100)]'; -position_ts = types.core.SpatialSeries( ... - 'data', position_data, ... - 'reference_frame', 'unknown', ... - 'data_conversion', 1.0, 'data_resolution', NaN, ... - 'timestamps', linspace(0, 100)/200); - -Position = types.core.Position('Position', position_ts); -behavior_mod.nwbdatainterface.set('Position', Position); - -%% -% I am going to call this processing module "behavior." As a convention, -% try to use the names of the NWB core namespace modules as the names of -% processing modules. However this is not a rule and you may use any name. - -nwb.processing.set('behavior', behavior_mod); - -%% Writing the file -% Once you have added all of the data types you want to a file, you can save -% it with the following command - -nwbExport(nwb, 'ecephys_tutorial.nwb') - -%% Reading the file -% load an NWB file object with - -nwb2 = nwbRead('ecephys_tutorial.nwb'); - -%% Reading data -% Note that |nwbRead| does *not* load all of the dataset contained -% within the file. matnwb automatically supports "lazy read" which means -% you only read data to memory when you need it, and only read the data you -% need. Notice the command - -disp(nwb2.acquisition.get('multielectrode_recording').data) - -%% -% returns a DataStub object and does not output the values contained in -% |data|. To get these values, run - -data = nwb2.acquisition.get('multielectrode_recording').data.load; -disp(data(:, 1:10)); - -%% -% Loading all of the data can be a problem when dealing with real data that can be -% several GBs or even TBs per session. In these cases you can load a specific section of -% data. For instance, here is how you would load data starting at the index -% (1,1) and read 10 rows and 20 columns of data - -nwb2.acquisition.get('multielectrode_recording').data.load([1,1], [10,20]) - -%% -% run |doc('types.untyped.DataStub')| for more details on manual partial -% loading. There are several convenience functions that make common data -% loading patterns easier. The following convenience function loads data -% for all trials - -% data from .05 seconds before and half a second after start of each trial -window = [-.05, 0.5]; % seconds - -% only trials where the attribute 'correct' == 0 and 'start_time' is > 0.5 -conditions = containers.Map({'correct', 'start_time'}, {0, @(x)x>0.5}); - -% get multielectode data -timeseries = nwb2.acquisition.get('multielectrode_recording'); - -[trial_data, tt] = util.loadTrialAlignedTimeSeriesData(nwb2, ... - timeseries, window, 'start_time', conditions); - -% plot data from the first electrode for that one -plot(tt, squeeze(trial_data(:, 1, :))) -xlabel('time (seconds)') -ylabel(['data (' timeseries.data_unit ')']) - -%% Reading indexed column (e.g. spike times) -data = util.read_indexed_column(nwb.units.spike_times_index, nwb.units.spike_times, 2); - - -%% External Links -% NWB allows you to link to datasets within another file through HDF5 -% |ExternalLink| s. This is useful for separating out large datasets that are -% not always needed. It also allows you to store data once, and access it -% across many NWB files, so it is useful for storing subject-related -% data that is the same for all sessions. Here is an example of creating a -% link from the Subject object from the |ecephys_tutorial.nwb| file we just -% created in a new file. - -nwb3 = NwbFile('session_description', 'a test NWB File', ... - 'identifier', 'mouse004_day4', ... - 'session_start_time', session_start_time); -nwb3.general_subject = types.untyped.ExternalLink('ecephys_tutorial.nwb',... - '/general/subject'); - -nwbExport(nwb3, 'link_test.nwb') diff --git a/tutorials/ecephys.mlx b/tutorials/ecephys.mlx index d8935808273d503dfdf1b9ebcb78e9b717cecd10..3344db9c51abd33b0b6a7a1382c28e448a281f30 100644 GIT binary patch delta 12197 zcmZv?WmH{D&?bCvcXzko?(XjH9yGYa!9#!`K@RTj5+Jx+aJPfITX37)d%t;S&8+Xo zu70|@t9R|{>fWoX3*1E7+C*B{0SR5_h@sio14#{m@()`3>@3Db0syEcT|-de&}ofilcE<+X4-_sVg`MtZXbam2xkxM_!7dGpig>7G41c^{kR>Vdddy;=VZy$J zJ-WFC3Mu`#wl2mLloV;OWF)=?0=iDW&WxyRS#p&r^^K6o=J)U7M-n}y_%->N5UiNE z#7f>vct>{+|Jk*a>K38V%xxUqPdYaxqchpL-9-$#t}G-B+)%BnE+9LJ*xnt9BNe6b zN)~JBA-z7`^%nIeYg<^ftM+n*ZO<_BySS^Lk?5bj9reESC5))aUOux0vJynv^{v)< z1tYe<{E%wy4-~ySk6M2M%NMu^WkGhly0BTJ*Z;y3{)+Tlno8{PYWO`eotq59`{4FPC-#u+HCw$2@nmX~>Q4pb^JX=~43DC~zE& z%(?53rvUuih%36+8uMkZbg#YjZXdO{Ahph(r0I?kUu9V6rgICUZ8?M!x$xAIHBa*o z>vzo!20xMsXX)C$w8-9W0wJT;<;(VSo6?of-p=ELBFLR-G%wk}hZ;>$htBexjN1)3 zSI9?Ly&gbsnEV%2_gq~=?}_J3-UD#1Pv0!NVq{k5N9)9M&eQ{!e|6xJV|GsNzKyDX zcV90jEBXLRm5*v;?mEw9#$mRfv)3L;^T#(?()e@~V=D+ERzQOMZ4@S z7bA-IFb~h`#_fT?#gjc+Y2^3^F7w-;o@=dwV~ILBt4~Z#mv%IG-&dc`l?CCxjM^DZ zVp?Uq2pro~dwR`{Q@j`+b+V@EZi2IOmnvIs1@DL>wqFD;8oer4o%iEjoDO$$Y#g|z zrkpP_T6_iWNw@hrXK?+sKANSf> z8)OTxztt-5`RasIbx!{-S@@8|4W#r-*Hy)W{pLajzx;|@k%BT-V-#Ajel5f^VRi8g zEcaG(2GqHsvn&Kr1jdIAWB4j&^F)k6`?~YqU_IaL;;?@SG8$$K6GFP`SO)UBl%r_K zwYal%)`}{y`H)Jm@je3XJ8$*#lRhfiMic2q^QwXQYA#>vFFTJ4{)`ts7;BD^5)<}D zwID{{><-F_;OJ(Y5yzuo+{MQ1;f3FBecenN6N7b}3f?rl_SPXQjea%TC5ycpiqmSI z#lki}(^Ab;UW^^)_u=%vnA7June9?D_w)6!K=^|Ax~0C%4Rb;mz;FSIeY*t_%k;} z|C4GwIy3y$un|(U!R9azYQH|Rhnf?;q?e^lX8=ht7v3(rw^$8GA35(Tl%o-@W1K06 zkLDhjfeWToYpmC*F`b(w#chZO8O^BX=l1Z2I=c60z$yhQ$M_Rb9^VOHw7X=DbO8dY z;7?4BxtZtG;9HCqSstr-uDL7cnQ6PT8=rZ%S`Wko4_ZZsByL@%xlUX~v(PvHK-!-} z)CTU!$J$5H;ZAO>AhaSExD4#DeJ6Guy1lftcb+8NXf|R&u^SMXNFt!AFOZbSy%j}2|x4kht-^V$5 z2!Uu-m6#0ct(_%4dxWTvo|$t>niiRu|5Fm2A_^fEaUu}LK&e8L2kCL|NejmRWV~M| zQIp;SOQRJa9Fpat9UMvhEUp*pV?lLqP!*<--77oW(<_oU6%3oI;00|OY96MFB8ewH zIKv@EdNWdrZv%1XNWQ~qxfXM^O(7d2*NpfQMJhohY!juIr~2Hpl?S1RO>FUd3G1cV z19^V9W(D{nj~1eBm1<3rf3B(L5*@+87%Ks0cQHivBIkHCw~ArCD_qgv#k#!uCiAeE zlhZfcdW`_l76rIs24wB#N+X9Qujr>Q5M2S&vvsyVeq8_BLJs1Ul;vMb?rgDRI{Mue zVI6%tE?P|cU=$z|Sh1iwwI~$DF$lFQDX}};QwSU};uxg*+Is1}Sxd;UBjUJt9ygbR zX7QB`@oUGM3sAq(pvnHcX#TkuHy1Cqg22eQQ0DcR4p*VympZ-|s-xW~n}v!26Y41) zna}~rsC6q-k0zK7j0p>aE{cNMJf!sibn4TG&- zdi7J=#QgNNmwM{&cT|iey_V=Ip+6F4$Pn}}jE82Lw+w2Nm{7?0SSwm=ZYAdO5;bpN zND?fZ&CEAYA0%BC8fL90Vw4F8h7ck{ev9%|zgI`q z_j_9$vhz%&No?VnWI17VcB01xw)Cxzs00Cv;8s!a(FeMQ%#@cE)h@}wO3w&v0=?^j z%tZ-?^5z8PFfUMpgh50KArjPzT^NG-cjLwO0fX}bNbkl+C5r1qQSC2+!K84lrx7!l z%iU^4vfu7l>IWR|38HcqO{-EJ9!;A(24kCPx!>G8J*aWQoR#nwVBq(FO=?)1$i~=* z5ZP?JwfXEa9cbYMd>w1?;gv0bMjb_Z7(a7Jp->9>Frg?&H^fsqhS>~2zYNN}9(zBG zr9qG#7R@aIG0c1k#lx_|ujO~H`*7OFy}|Nx|L$_?is^U)LU_ngtZ`p{>QeR{v9gBr z43Z#Rc?N4qIHA3rtt~O20TS~qn~?2;IUcuD4yBfZYa||Mg_t3Qha)N{q%xW3`0m@H z*$K4dmQcD2Y28Rpa*He0GV1ylSAI8XnnmHMSP9~wvju(%)w9KP?#n~0?(&fUU8T#T1-6isd9Va! z?y$pgnW8Z<93u&BN7(q!*m==>d4%iO>`HRxt=Y3=Tzg*+nC?fRRQTo)qb0;9)jA0G zG@=mmqTuW;h1g*=k%h10qTKK^NfOUul(!z!3TDv)W7k2AauLX;6Dn2MwxKX*_@4Il z#so@>Yv72k4s37W1AlWI%0#d7iS3!*=(S}7HM-fdeAA~GgOVzt%&=-ykFmOY=aSVZ z{R;AU!0!wF!xlx12f2`6NV&Frr?El0Mh8O{FmUANVX^7LbrQ!%M}{bh6FvJSG2u7; z=;yMpB9~_v*$S;N2E9dJcQsI5$?}{Ag&0n(%Mr|!Po`#pDXpQtuSvlm_{2HT=yzlz znto-s7HiAty>wUun*$AAEFM-IE1i~*sx!o)A>Do3)`y~%(ybK zar=^C0*YtI0|s83%@y8;8Y7Ekf~*ZtQw_~6w2iA)Mh)&e8;#;>T%QN_<^e?=2Io8C z@w;QIUbk>NpQwWuM{Aj(QtM}qQyTS>rWKT=(E-A?H*xF|%iqo}{qkF1n~8MuovNtv zF~ZK`6aVhRB$I6?izi8e_k$sMrtgqg>c>z z6g-SlHiB@K!;uxSTk4@n*olLXt*yVVb;>k{ z8JKe(%Iu7X-fLJLDo7lvP-KdIN!9zL1`WXvslBY`Egmd8AfGHC-}V-`x|A*v@Lck= zQ?|e-WKSY?_j0KjsSR z;l8zVysK!Hx}CR!I&AbQ%)KIP8yU7-5*U;sk^W{fvaTGex~&IPyLz?s6`@wn+{Xxk z;g$KZ+xkcLcMJ9@VOwF4E;YNpcPIn+G*in}Jz}sDJ2&uxzNyuYpsaD z>ces2ca_aHGh^j-4CzDn0k76b%}L@Mg17Fpts+RMJ$1dVoO2PiGr z@u|*&`=qGLo;i>!b9;t`k}p)tuAw${_Q&B z?Yzx7b+OwcrulhiYd4DZYQ-A3b4ZS#ep&slYMK>ZoO_zCJr?r4`avP28>63*g(R4HaxWjzq6pvi0QwKlLwe$IP%dbg}k782d0+l(8(b&edOyzTUi8i^<)Q^SajF*DbXnYT3T2XD+DG6kijkr=-?YgCI_p5R&>}$rYkEy^)*DTw0tff?^;#q@XnV_ zE>c?97~w3WPxWTb-)_Tt(}bk;72(w7kZHiso{^>LqHYzvBspQm9+=@lB`Ej8^Aul3 zJvdAc9pn*4(-X$fUZzvorxxNnLG;A}rGBJeJuF)0=$lcWW?q%Cg3nwyr@cfFo&m=P zQg>#Po15XY(KHo1JEnE*wiFC*QK4U!lyrThh7HWO#Bt42^F&@P#&Zc@FA?zf=7CuN zuzj(#F4t^kr2y?1P zo4Q`3d(b|n@crvWV<4zWPo}x9fzQxIOdhImNRyghY zCc3e9xgD@KF(+{`#qW}Ai}xn$?9)nF*BgHYOnaw4Wvxnf0Hz)eOCLeITQFoRePbC{ z>kYjfurI!htXsTZPNCkeM`N`;QZ$k9&hGaU9EIYXKd%d8TKG_ z$L23K-`v>3mb7EelmUX0naXf?KFvk)Ey^_sQ^g|8C?K{`+*Qr@k&!_5?WSM)(Ol#5 z?)A^apbb6Ag^VjX?FSfZy>=b2TH1tgq0-4lUqngg3lz?bsZi3f5^e`k9&32v!CSq> z`-(V*%Du2GF}4j9u?5vwa~@(_r-ma%3&sgaI7jAu_IfW(B}k^q8X#UZ>w~T7s}>LCVE)BQj%nFeg2ttMNr@Loi<6ttTrJsq*D}b zATkbC3Od^RVLxjP9@xTs=An@Rr*=?wji98ND+66=L!jBjtR1zgD=LhTN|4i%u0w;2 zV_EU)SXf^hwLUI^V;F{3aL#H~3(~wvj_zD-Y4PLJ&Ujd=i393iVjonS&^WpxtjsGa z45%z>zA})!Yh}dSB`c=mc>jWW9AI?qtu!u?^VaPqb|vtWU~C*P!#gn^?sSQ#r0Ck86ns~flCF(ou)J966>(c@P{)90{gG$3tF zSOSA+thY0XUz!^WEtpPi!0IK_JklVq1Sm$<)#_go4DN)0%RX`XXe;Dv7^)27_vd2g z0qb~@3$Oml2B1v1f%j7STH@vPXVKIf)Y?)ere|Kw$k(4b{;tMtPoTwO2cq)XCSbP1 ztgLJhkk$e4_)+4wJCoSf&BTaM#nRp|?q-@qX_%J#+4U(Il8Gdy{cf0}PRMXu3ISqS z$@!PA@s3otvd(E++n`nws8iLkzReNU+;Lah#CKEB1g?9|6JT@~H*T9XX7gppC`F=j zV=!~Ss{;4-%&qotee2J-xR>J@KJYVFCJBosUM{l$Z|ltJ@MqkcAIw?VhRi#%9b;`q z1bY=WBS_<{?x<=6>uY~fJO*E9(_Y`3=g=1rmOmoPCwo)8-K|f0$9fNSJLtu6PS~1v z{jd$dOKyvPo8mH;4i{GH5PghZKqKUTDn3rCk9rc|fJGjWDWD zB%bOSg(q5-CM0*l9vn#}P|Tj6^R)*#fQ@>LoNptcjHKCTk$ z2G!T^bud(t23c%qm1>v-%fLs8!I2lrjo?Dfh)9nKecf9ry@%lMyXXAo{8l)%KXoM^ zwY1o@^U$mpd_u;$j~rer066SP{orliJP_nQ2qR|4c9gnEBGWuL6LT;p=jcanm8zW; zv~|;SgTbY~H(`)GkN`qkNkfEUOnuBp0LK?%`hn6}bcoFz}_3`j@ngZ)*f78{KB zkSC;s5q6?b=Lxf%>Z+R^L1EJKlkIx{R4r&K1()!&w*M3l+?7Y;?~1Q@&&kf%z_D8+ zHpjua22T{EUfzbJ{?~g+3C%vMA_zkRC+fyUZTg)e)06KaA{q)0+Xy&msozsg~aFtNGq0;G|t+9t& z|HDE?;h9vvo~E|jZNU##=%Y-p^Mhs`-XFc5ij&CLn(EqhFbkDxpmWAGQTxYPEK_X#_cFv7hviDKMRb3YLW%v)Jn}S%sfRg!0fFPloOPwkP z<-O!7$v&z$h|R?Y=eiaGQ>V%hlE6SWm;v}l1nX_b9GpAtkdP^3R2{gs!%KwW8mPT5 zDL$~*QaPm!6w-=o|52QNwe$Py5d@cjwTNOPU?5cU zYteq0L@kFugm7QxBWQfaYh3%{CH#+^ShA3!e&*1P6a%zuI-y`IA3`BeVbClm zMKb~l2cUGpeqvcCz5 zAmTDSa_E=E2GA?|5Uq*bRsOpziU^pC)&AV8-TZTQ1q~kGj;St}u4$UYI;Bz)sqqeA zCp_M6>PCVu0Xm@-QdJkerP`e1WT7Vnm320OK0V@oZ zES~sa&|wTQV)!R(IvS?|j}5{*>jQ@sWs%qW z8z&=05zN0Ji8-VHK}7%U%unvD3yi#EMHKzQ`RI$Z^#Ez){?a1(U?Xoek}VtgDsAOp z#=qSXr-JO4Kh;$`|lY0dCzJHCA_J6d|;h)V@QC*RkCWp$@rv7;%+0!KNT1|S zL+iw}Uh&}k^xs3L#=5}j_~@*&uyc!$9yuvwNMzK}*s;w-wBPYgQxKTcaL{7n$c>a_ z)Oj=Ooa=K-N@2u-55As%cKW;8Rc?NlwGyG}Qhnw1 zZ60a#5Xwjz54^Bp5o+`72TugQesbsJ$Z1!SQ8yL_mTyRRov{LSswZ%tT4Qr7{5jZhYo)DiQHl^9Z`e7;eDmJ4qr4Fw;c=z!;IyfB2x)e`@NGMeevo2|%Awcfqy2puDh3!Gc=z^}tA7#zB6S$mN7w+RKF(>u+ zN6klLCEl=ZcS$8bTFW0r++(>i;x8ZOOKRgRwIkHI^m9&70yinPBv_}$@ak16)tk)+ zDHD7kh9#@G6Ur6ch`XVtHWJQfN$RF^i|B& zS3d;KnUZ4>D`bHain7I1M!tjBKiirT=awjqmH^vb3JhUFecUE^|3iY*5QmWh!!uyXHWW?pg$%{S$jJaFAZ9I4G^ z)Umu`NYKjLq0GuZb?;;(K84CG`Y$ta){keies1WdMR2tJrq)BT-(9e$p5WR@boUhL zoJ`|8Q20J}BtRN}0scb*hw-7k^+6Xk@<{N9^J zF2yd(`q~I#)U%{VD(?sbc2om3YBKa9Tt-!=W45}2?OG#>W%fl)yi!~--g+Kb9FI3u zeo#&&eSPB0D+}qFn*7_}b#w13EN7*M6M0l_4qG%bG`9+4@Ea<&qdKWoW|PIWqv!8P z9x`!AnA<~3(*nBoe%#q9grm=j6%#(teINi)Vo3-7>9JFdI;AkNAuE)SWaf=%D1Ml- zzEA=TDHD>=a!}a_I}PUpPgD|cROlWJh470m&2{DS)G2CiOMT}4k}_7Y3M2x`PGnja z5eJniQDrx@K4dMQ>5A^Vw6~_hSmA55jI<MoxgnnBMH=w(OaHD2#--edej`u_!*82uCpfWb2m(Q9K2@&@w;q)XmqS zQ13c9)|hB>Ak(s#lMkEjWQjFV+SBbR!d(~N$s6*5O_tg>9%TB6>}m64@H04DmlUI(3wK8cZIK+v0D}TgvB0Mm(5k2Cn=f|6K&$daY_zU)lBG+BB&u=W@}lBb zqUmg5l}mnm_9yGtB%-M?ZSJg8x_~h)ZITSqF^blF#n%ZoD>41u;RNm=Fz*93~O=a8Dz`JD4vRw58huGPJtAU&7H8oN_i1E&JCNZ9r8hF2_EWJh)WWEtPeZ zS0kAc)#&Tf`T;gr2AhnSS=LZln^t_8^t}P<@z_J)<0Y-aHJJx1JfJs6x*XewR@Tm~+OHp~hBwRsIb{<$Dj)ed@*gX&n+^}$Zx8#F@gmaK z#va$&Zj578UzWFLrXX5hMi^$(i2@~l?gmR?3ath|E<>Hj5~GtD7C$b38XVRs4{}U7 z`L7`STev!=5Jo604Z4?$ZNaFXY+&Xt2wrY>13wBlggndo{%|R4=2nP5D+D0nNAD1Li9 znPU;8S`Z%!&VT4nzH3;<+dz4Kl*rVDtSK@rrnL9Y=f+AP+`K7o8&_!?mQdT(Y} zAk)UmSX+?ZV@$zrDkT%?2Xi;Ta&_0W=2kn=ftKdOjVu4QK8&bqNn&fU%7C1&>H=B* zQ}g<%%#m+FCle!JW&c0tf<8VZbnd)iYkMW4-bt!YJF_u|e~b(4&|y~^;<+a@2rkKN zT(>axWp5lvsi^}?-lMqpepPsE31e$&%SoLVBouZ_*e0`r*|OR;=tA^?Lk6NJio<{J z@J;1%BS_p*YN{I)@El@Mz+s;LQ;J{O*nn#%jHNMKSn}Qm zV;*POi#L=0$QeaV<0;I?OhAZ|W!%lqx340bBeZVFIstsnXBDJdrmwg_kShKI$h#h@h_nj;f(+Nk`T4(+~}Lg|Rz7$@T#> z4&2m++~Co@O$~O0jTTJ>3FcPq_?ex(TF3$g&^PUx@YVM#SmzG`D(Fq>$Bsjiyy~8| z>rhU#!V;#6jPC`XbR)TH?1f2AdAetn6pbj>BwSp-`DHkfqjGWqt)e3fxTpEY=MfFy z2zGwYawpBLM($4Q>=iHe9{}7ftd8Y+9K#w~2iT&{UD6`zdTY6caeYPx^n?~j!WzAh zsY@cbu|zT4x$BwbkQ)=A(HuqUBDmRsvBXKc$MApmDbi&N^ojYwlw)Kr74xH2G2{Xw zB}{>=xB(-$HYa!>j(x^a8>LYltdOjlUIuOQx5Pp7R7(VTv1+J%mbvC-BiAW0s6w^x zO&ULF&6=H~8P{zZORC`Frn|SX6=qRv$&;PQEh`NT^}c>yinb^Wu8EG9mcUixt-Eag z=Fz;-;|wk7tVuRsJ5I%@R#v}|r^X15p*(AhMBxT^(t`_t4!_hiYsK?L)WRgP3stC3 zc#yvi)sR%T#OXwySu0qLY?MgN;z`X65f#YBcrQl=w~$;Ou|{ikqps0!i2F`K!nSj% zyOyJ5?O-C8{F)~T^t7vDVRo;hY^>VBJY7A(jP!A#D-50ICxF-c3K_bTA1raACZ^s* zFT;#T-|W=}Ox6AhiKHGwskM|CBM&rl*bO}V)N%|8@*#1|0*mQ4KjLccp(V`z2)ry8~l^pDWEMy=S1MjLSdW+ups zy9^Z1ISdwy9#LWFlKsTSTQ>xw>MWJv>0WqZ2UwWcF8-G27NSgy2rS(zGtp&@)n<(i zFkgPqSzV?O2_;z^6D&&s1x?7%ei`2GMlwJ1GyjA;W!LKSEYFmq$yiH+ma#a4SIa|L z@GLL@{P7i|a+hsZnnJ_$8!}fp8+L%bbHs^5VhDJW{#S}AL1{vkdLTPT0^FcDXZNST zrYmfFRxQ)-%Wl5eHVY#Lkf+p(EnOC_#H}3@Hv{UD%7KHST&-NlL2kVzU6kT-q=E#m zdwb=q3h2ss8on6|$-^n7T}NVaIVkwA^LxE_Ku=_v~D_lpyUChcMTsrUzE; z&Vg0->}I9QuX1}4aI5^nAPX!6;|VVgXtP|u>zEL(&-*B(iJiaH7A^U{-N zAtrgmIsdq;-h#agqnc9HJJkbby>u&E%z(f1B zQ-|*z=(99+no(etaX)1VtaJfo9fp0%@8W}A(5lkS}; zGB@r6?*Bd{FUM3aE0O{Li+>;Ve-6pkR(9sBc8&3DCZzwR9$%|?YW;`o3nwrBMJ+-< zAfy8RgW;%#b;W)*^xdz{<|RT4Oq+$$Fc4_kX8=gKksdxXAwh zLLGl65&-cJ0f>TRKyc*$jbIOgBZQCvMS$SQjQ^1era~W4)B*rt4gZ__e+B*}-75d% z2~Os=)*LJ@&bDfb5Rd@C{|T==LDPLIx&Q#$5Fq~boc|-u2ZBh0Bl}meRKVYKyZ``z zC-VP&1tLiFe|L(Y^|&lG2LK}N|KCm;puoSK{?k4$?XUK9p!WY^Z2ZGe0X_W>gE;*! zBMZcv{=Y>G{$W6aJk#N*NdEPL{pRWIzC^Hcb1Wtq_1`%h#p@X*n%0Qm} n0KKKd$wQoiUeo_JSZ2WSLY^l7!y^K+D delta 12271 zcmZvCV{|56)9xLc6Wf?cGD#**Cbn(cw(ewN+qP}nwr$%w^SsZ|_nq^jcU}A1Rjayc z_3G7KRV@t!>kS0e?V#YLTC(zW-JoP32>&4KfC8{Yd;lON?goqmsM3{Nj=)1iV2+6I zm!H7Oi)tY>j{d9mLt9c3S;_^(ih0A5kbn8?>yxoAvhvUvT=X%mnp{qCDJH?i*yw97 zOGY5;_bt_9@Ahf6u@VwCQu_46pU;IInWu}q!5tpp^U!lILWbk8pFiIhQ74uTK4Pq) zqg+Y5T0w zl#geunX0Lg{v%7z8`C$Hppc~J_v$}7@2ADWSx=~zWye9Sde8Lrc|lVl_~g^H)j{v4 z*Bd`G-lQAcG|Muwx+@1UGd-M4>!+*lJ#YIwGKF(2C|TYHfD({;snr; z%~wU8_%#|?oMLE^kl%cg2p=iWxutrC>%26j`NKwOPgrb^CN*=ckTSvwv#zPCzj@J(p-#Fk6CppHti> z`4FdHiTG>=0WYfti-0SgQ+XkO-a9q|>R>!1E145H8L}#{40;J`k8TY*zidB9n@|pv zvl^a>r?fmtdB}Oj%YV91p*Q#&ofTJT_vkY2##})lO%EWZ^I>V>yMtup0`<~wm1oX$ z-G8{F4>G}jO6}~-4ty?~F|S6Xy`-A`Wj>#H=6d-+th6-GTD0 zcZa$#y^ToiHQFs!9vO>l>b+&T?cXG?bXRi+)-?4A&8Jp4-@WIj2yhU99YwYqs@PU- zZ8oY?0wK)MZU2_rmou;kT)ww?yI~Qf<(R->4dCi(DYfP-$by)$|XDo{%>f+AU2Bk71 zJD}L4ZJK)s7P=5nPNm-NpPM)6O}-4D-qwwrfoLmnT0PH&EB<%Ct5WVh|8V(XJt3w% zI+ZzlDU=;uKgUh0ifU(@ZlP;4r&X+}lTuQrMJkq<>0{h*)1RU9>f|(}H}WDhmmKoG zC9cVJ2kP~AtwHCo_*?C@FPb+<-|ed6nceN-D+EG?2qpTD7a2uBIg>PIbw3j|kxZV8 z0N3tfGSu37{pMIFURT5rkv5sHwpcXO^;R?3*t@u`R2-O|>_A5iqoq0wjdR|WZ#mqA zQbEwIenhWKL(-O8a3YN*9QgB0C`=4gD~P+0vzi-4uGD!}_Ovec@Jmd~Iu-mju1`+X zm44GQn?Hb3v(lr=uF%^fAoMr-Ia0az61r5Ls>Z!7ZB>Hc7cgP&p}6o?CaJ+^U;kmOdut!3$z&sc1g2uF zlgZY_C|2ss&0ZeWMkQ&^Oyy*Cv;VPl=$40+^pXnmz$H0(;JR$HO&{(Ac$Pz-YM(IE z&dEZvYA&)oSFqeLm(9~sbY;}ME^W6Q@@gEm@C@+Z*-kT^+VkLj+qwrezW}ZB+7=5p z52nPJ++2ZafzP!a-ldPgvo2}0gT(sXU(Tr)hXjKGY;%S`b)Y&#q2meLF)!RE_8w+# z;&UCWrljoTzzgKVisHq&m-bgx2JM_96d+H+eTzW!P6~eL&%$$DCmG@2LTWv_URh^+@K9P$#_^#vigt+@G%2=n4GSFJZolz#dS?`Ma(Q&EWm8@>&KhQGNV=!) zji~$Oj^{K=%1!Y81aE$!0$u0NZ@|W(pT@|e26IJn55mWK4IQwyG9k*uEwt-}d?Xzj z68wrBGoZW`rV<|;cZh)j6Nwz%?~(T#1Q#&``eMYL#!rB9dvh!GWUkg#EA6AfZul$G z@yZZUYNBTohj=4n2xKk?cVJlL$(vlqFxgfk9skPqEr`YMNzz^3R`8n^lC`Ucj}GFx zh6z;T{e2epJU;wL;_2EwLlRrj%bgoEEgFSQf^i(jP0wZuz<>x!zFp|=q{at(sQK{- z^j;UzMNctOgEDdq0V8#KkJpul`qV7E4uqI;RoQj#f$LSN+kUtdf1@$ z7h4y(GW7=Yi}KqtbfRwvG=QNLr812L^$OWmrXl}5j{zFFw^?SwIa>Bawli%VKk`(8 zi^1x;tf_$g#Y6|<=nIDBx^-|sT^m*)|7PqC+jW5|JoKO<>^LSC1h&FzKj?j2HXJ07 zOtqq#%1+l9`IlOJH;@nKw?`b=6uKGh2K_mjOFd?__2`cyy0wO zHwL|1lw8Jb7wBc$0eu=SVy#HR-%aDKyd3hK_K+MrDJ24I-&HJcl-R{(dt6zB0Iw58 zkn#%mk9>N31%`+ysh3i$j?@0qR0o15k6hJ*n|`djl_p{5D9?&5Z|^yCY-NkUPwzNb zA)?8Ea2}xUGp)iFW6!oS2mjZ45b?@QAQXT04Ys^qcdIt?F6As(*!ro4Bjd|1=g5F)@$SZ5a@t#ioE`pb(HG1 zT;_-eK5IHqNv5uCZGQ=Vw?tkd8)65a^O?A5`=79Gk*NfCYu0KWyQ7l+sc(uLmnNGGQ) zO=FR6g7w1VZJr4AE@=3um66ATZqS1e^hD?L5eR~?2Ry`~nMwKRY0QcR8Icz_Vo5QjrQe2UoK;#L_ zkFU)EyL7O#wIu>p1SGZ)X4ZXFho!Q}pp-VS4TOzZ2F8n9U<*!lD~YGwy9-*-JN+TH z%?xKpQZSf#g(MAVX9!xqP@Ya z{{8T6HdrM1T^xsg?J6iH2|nS=ZYEBB|OEM zm9>D5c5oZ84Nqez(0H5T;g@sq{xh)}Dm49hzIuWXt%7pqL{B1AtC6Y)r$WU5jZ(A- z!04^~?kynLBb#^KcS<4MNxbU^oxMTLAF$|ZzClSs)f`9L>yE$z<8AvnA^sP>;73yL zESJY%>4J683f=VqJL;fjsOd)iOtgpEdGKnA2a^-vtY#0*r&zay|M1D!zA$8E><&fC zS}lX=tt1%jFNcydn6xBV#)#juehD?$LZ z`<2w#1UMC5iLO&TApe{IosR!KBte+Y^tUp<(e}-7WO|RFn7+PQwMdL;r>x_}dZ1Bi zo4%s6%qpjaFHXM)i>d2@7c9R_zMU0VxjGJz#uj0^=F&`K?RPH%e3uo71HQ=8;{%4F z0RFaDe?TSHdjRWvXc*SM@5&b1{6~5=-9#zMJRhm#NNi&sJ^nzG8+7?Vco$zQj|SbM zjr8P{G|q)2-6@6wUFx_yU%Z)-bDt|$Bcyc|VZcdnE$+H4-gC~yAiAmd-|t&}3YK@E z>)a>yYGDq`(|2rl2T8HGm9$fL68syS+ST=sCSPC^abVo)sefVIU_1940z+{a%#^t4 zYT#w*&khrja&`OGv-5{62xxt-4yaelrmcD$lL_TU&ke>lPkYMfr*3rZH5bz44(1b&Ua)Kl5fwV6{{hDNvyKIu`1 zy=BbD_PsNtbR9m@$3+6;St@GdYH%&f!O%);9F{YJtxLlgQB$QqR(5Yz#_9%uaaPp> z6QCrnWlFX|Ua}$gVY^;rY=ogiL?YR+xTkc`*THNd@|0q%T{3SR3c*oZOxk!2(oa$9 z9>^SL!+7M@-O45uq3V)Hn6hTzE{CXAEZcWd1eD<J-QtcVXJQ#BD3E%@U$goglE+IuW;!Y#mUXQ$g!e z7eM(Hw86-^^@mKG?NO`y*xtU7sj769SYDn&54O+)jdJ(13|?Y~T&;pjVuL?hU8Ti( z;%bJc?2^5W{|h;0qh)9Zc#9ugZ+Q+8p28x>5>9nrZ>hK2p(^vkg%!XU#jpbq)yAbJ zLBId=yN4lZ#YUF`L1cZPK9BLP;Hfw~64!1*C7I5>H1e9%D!6_v+eu48`dfkZ$c%eu z`#Mg&72+KZ^G+P|tcJ{1dPg&bXQPXhYJM=X2{F_#!EoZd`t3+7(7|Rw;x?BpO_@Us zEbsyU)j8=cXcjCxch$d$a{_1N-pRJVLNT!BX}~+dOdJ+(36KSN=t+P_Mx4frvB~y2KN&X z65JKf)Zf*>)sI95RsuEauoE@rmw(pBTRb;^G9C6zNwNLPbxi4(n^_624%{($Iq6^; zuC@7zr1=?~Q0Kt1fzW`CxoV(F2pz{eJkAtRor9nMJB^DQs3^E=fCkFbI+(lBZo0k4)0Yht2%5yXVe+*T_GgJ3g>HBV|&DY0^8ve5}0Q{$SG8OiuFeJS?&kc-~*Oi*(OCs1(L za+>%aF=S6DQqNzMx4xZyygJqrz+Z@_h|}BEjL}{RaWif^l>>GYsG5+9jIp>nq49V^8i|h zfm_Ws=nP$QRT4^th8SJtZRH_)v&!XpnyQubMWMI`>v9cA^t4l|svOlcB%7`30{?w$ zXq-QUfqC|c3iUrQbu!Uu5^71J>&pVlO%`ASC?ZaTFDnycZ(<;R0iZ~eRQy@F`Gt4+ zPCNE}fT1FK)e>LVkU_>Zhj6_xGGa&Ad1wB;Vf!H?jH^~4yrW!0r;fhfZD6$znj{&D z7fa23Z~WuJDw0d@{7vJlcWNjP%+HbRlZh{p-x0s3ZxyzgH$k`CevKW9dCRUJE74hV znCPj5X}XQL``UdE1$XVT)5GGy5wT&ItNR}E3n-bV0&`?rs>#IcT?W&= z9Z~y!iZDlD3OeW}sGu`GZmP;686mEDI&fTKcFo38G{MDN+0s-CcF1bvZQqUOlxY#I zs8$L9rnoDzP+H2^q)X@g2}PBErm4m-j{rs0XPKG0C_v`c+LzcNUS>{Dcw zW24+jZi?{`1(UaEO51O}JJH7XNO@Lu9=E;L^V>q~U0VZBrQSzslr=VfV!IbV73OU+ zhyfmrGK42<+g;EIv=&XR&y$Q|$M@_A*kxss)?9Ju>4vW1GY>4)jGYy%vOE0@IOzdl zZ?FZdWvoJ4F|NMTT@+=d4;NURDIjDwsG4V7-(^dZ7^7e$L z{muf(B0xLbTTXE?Lw8e*sr~cW(f|spKx6Y7^VPy9Hj4SefT^`e=TP28GH^?yeT=b* ze_L_lc2E6c!0BXAM?I;sqCkA9u~Li>@3}+HJFNR_X#uzAI4LQ^XvOz{Ssg>|@<*)$ zT-~>css?$V0t52qO;O>pqOlw*VnEsm!c*y4&aujrxt2;(viWUpQq+V>(gbn1)wr7+ znFr1*pC^N3|Hf&UxKKOI4`AbU0>gDdF@SK6+#;Ooew1AlgTHzVmh zPIMIF@fAh~e(tv>MAx1YK=YA*!X$mny=q1?v7=0TJ*{A)6yxsP{_0k1?|z)G(efP2 zlqn;bu8ty$mSd^qbF~i{`!*JVs~DGdm$Lo2RTX=`w0jtC{O31pSs?bt%6)?4$op*a z`%R(OcX90MF9d~*&mTW;M?C|R{HCg0Mf^rr^ty*1sI*(k4$2)*{K7XE)<@dSL2y)@ z66?G(bny_`sS%$42$CF~&#JSDd%cYErTBJ$i70ZMXETW`lqh#_9lISHQb<9mpHB0a zg&Lij)+igl#;~VEp8|i#iV2cE8QmsmK#L)U&AwgRI zs!Xs&IiZ&h68dE7r8Gx3==k0sdoUG3?0oK>ZrpR9$Zfw2twGP2KY(&qZAc#D8)2gp z-cEVbikAUo%I2JDy{GJLg|o!7Ie-}>{t;Esfzw-*OHvQ&MFmtEg46uWzj`k-SL&C1 zbnrUP+$4(E5vKWGK)x4UY99z!* z-7?ii-~~U)hz+Ra<)Yn%@B~4-a`&NHs}rmilYs34o|>4lBD{R9bnrDP6f@qAo{kjG zoWrgd?mNsa;2=@$n~or=@44e;X%J41Sv64oo#q`urNV@yN0%it1Fa16HucZ4MhJe5 zI<*V=s;UlwBW(IzD=ad`VfU+2S_j9Mx^z>52>Vs5;b$;4eV!9MnVXnKr!>RTZSi|2Jy;Cf1%uC24^$D4xpM!ruIbjQ| z-}n6L<%VXN#vd_>1uW`eKmPo}+^f>9Gk)L~o9CH@gA9m_@4#b6UW}x^HL}AlFqaih ztr&GwXeI(?RTTzs30#TB*N zV+&rc-cv)45t%rQ5GCszcf*V3T>nhsfV`w)yW{~^EH@T$hJ>-S8j|1hPl{1%7!yD? z`p($W#%CHwl}JD_CV@+&`y%|gOj@Hz%!$|X(buJFN~w@ zZgoNL1#Dc%GK0}*Dz6e}q(h;0u-u?*q3A2n|Q}-Lc=3&@%eL6VMzQu5MW$TqyvFBbem~K zu}N|f4Tu;K3QR7vOlB6szBE%`s4Ta%4^6<1{-ddFuBVK-MYA$5g@x~VGqDoeJI-j; z#K~Y=6^5)Y#Nkryv%`WPQ#dKj*6B;?6^kV~0i}tWiXlLa7cDO#IHw4Dsa8iT@ZfLU zg<4~9JG`j8hp=2)#OsZaBh#!Bw=6uHzKBQB){Jk6FQdJ??c;LI2 znADYHn$AITJsM)CcbY9uNlT9LHYA&nzzI=lnc-twN7lLjY;!ZadbAb3YX8HgkRyA8 z_A1JrB`x-WM0$Hy4WU&1y=r%2ffFpAQt;e8yA^J+ilC8ty-@-D;{CN=iKj0=8l@(A&vLC{YT)eb!M>kUiUDXfeUN>k>pMoLe4C;BbD~3g=T@Su8)}K<-%R_|oZ?i!m_Sew60aZbnT#^(FP(9)?@;HB*@3 za_hPJPN=QfQI^RxWlw2E70L;cLu*|`nMXe)&m zz2%fxUR_(?EI7TD=UnMP5({Nih2O+Vdl-{kR`eeF%>FP~inRpN6OjsF^=fbTh10q; z%~xD5<|XEQZ3L<1Z_F^k#XW~aa*8gQa!LQ8@AS(3dVd|VQ2DZ+E~-O80L5vixpj*% z@%s!v{8Zq1+L_59ME6EKXKl%v>lu%N<2iLMsE zVi7GNqp_S54o(fz>g23!adBMN`-Rq#(B*P-rT27F4{VXAa9CBd;7%^d?oTh*LC^} zciOkm>)XNB>nc-09Odiy6e(~%ZPxVjdkUt*#lXk$5PdGNHSr(}nj?$M&f80-Z_R8I zms~Cz=#U1~KxN?5%3?Ns(&Ju84n$+hSG$@amAdiuMHnl&G*ihJ;8;wE9U=Tu_E$@G zC}cgJ;-PPnTcp;1y$u5Hk%b;N*LPoPCwbn};ms|!DqG%zjT;YZB9a;IF{PKeZye@ROJ3duJc4rB zdbq3HyPZavp`piQA)a`)$|5>LokcT(6^L~!S+mHIlZ4lCjl<=vFHG4DZ!qeZjtKhY z@M!$DUL*>CTGYk7Lgs?Eg!1i}*Tr;lmO1fEu5$!t+lPved$cyo%<47;u%xJ#%Fe0G z+r?gG`i9b{9Kp0r_T6lPs3rASqEN`$w=MIO|6so>$(nq(BQHpxCB8aGYqbAFJDSL6>s@sW+4&WpoW5O-lfTimZw9!{**0a zV^*5JhBKs<+=zwrsQi#$oVheq6NbH$kfmrSiT~0O=Fx%UWR z%@Y6EbEMUw)n&b1mrQ-EaQVSA=TLWBshcmN&YG&D1hv=(d8_D-duL+Qh}uu>iZLDo zx87DPW(56?^sck$3;kf&P%d1Iq85+4rqtGW09GAIH}iyGu$#whQkQigmo6_~-22D+ z7<9lxzxDf+%m*t;uH$YdepAbW2i*hGvB0%ljI1< z)G*03^p)h7=h$4P1l3z)bschvj4ljdJ-K%fzFOW7i+i#N@DZFFKlV>N_1zXuhMI9BwQ^sKr)@?suAq;DA9%hK~qG%d(#_QaP0MnNbUuP%DAh$dpTruwc8f;m%h z*|JJaEG26diDz`9y`xA5!w3i0AHke2qiTK-$EcBIQg3v_7+X+P-~xI=#SH`K!mvaa`g(Ap*I|}F?U|ChqAYBrevu=n%>yvJGkx|G9#P{ zc3H^c4|?exug?pLb6qK5GzqA4YTc)g+jkj*F{m;MYACqZU^P58ZtaTt_F1A1MKBXq z-F-A+w}+Av#G#_7pt8duzKKXi%bsb^IEGK^^rID2CF2K>EjtC~qO1pU9a)u7;6MS_ zT(p0Y*(i?Df!W_i0?R+b`O#=}vXf?IC5ik8yE;;gL*YD%b$@1}UI!v?u}aq|PO-KuD&K@cY;DbwaO=lrqtjg<)J)o> z;`ljGAqXW|BY=jUQH^FneL4QOVY7-^LAeQ@k49VBm_N!E7?yd~YnFLb-aU4@bicT@ z(Qn?^TB54B#jK4&zQ}(Dy`#>)x%?qbd%f`(hb?H-|gW_Jo2gyAGBVxeYl5F06x zGEZO4AN*Gw{uP<4`Nl@G=$;iI;2^GyUy+b_z^it*yTiQ6llx4>J^|>Hs0Vrv?Z4AP z>M@@u8+$nD;hhi(YoV}?Dls${woLriqX{=GBR7TXuc5?o2=kbvjjSXsST}bOrkdf; zYQkhmw%oBOKX*LANws@l?!rwqdtXF_+x_oQ!d+pGrhetR!5ood-=I#4(CZ7Zb)C7W z0CCMeaj`(UUVSl-vM50!v}vQ-Acbg1XM`%7dkkq4-7s z^{BhdD_O{HF)bqNc#bRUsddAWtKb*USvBkG_70hti8o@?iIvM?(N);X16xWBrl+tT z+F=+S-|%9~Ibop~Tj8qAeG`eK*`wfB2+{dVYyBQR9%;vX@QYA^h>O@HJMcaaXu2P@ zl@#w(x{y~{zqKd!w`5*-E1)-dl z?YhRMGI_Y;k!pocTi4kXnb@>R#v7Ywq-QxYaw~!?-hRvJo|Q?+9;a)PkXx0`Ei*ZOG4$QhRm{E8V47y%R;72N^E1G}Gj^i+mLCVhjUdYnz$>RjFyRO4+Oa zXJxKlOtx%TK=2>`0YpkY7|XGHk0Kg@*FNIF`y>JQ6PX~a%_LFxe!udjl>6YOvkFD0 zSwA0M=p9C8;$4md@%oPwx)mko;^C%%VZDpY0ko^Yj+iM1O0vKO zgt*C{)jb2D!Z{Ly{PSioK>~s9_t<^dX zF(>*BvqlAgRp*h|9e)KAss*F2Phz>GpT=*+VLTceq~nz($|?j-o(bGl#dUv8u53je zcXTe;%;rlHY8FO^DFcCChJ-w1l-B|cossf7W6@_G`}aV2Bh3azgt(&%cDF$_$;_hp zr9?`pw|!t-(Ri?Ggn*?RPEAz0a3qM}yl6k4ypif%O8~mE0gA|_Nx559Gvkz=-Jm)B z?r>c099^aP)=w^|^MD1@{)Dr$3(l@bwec#P8drqZ%$-b;2vz!6i@vofx|L!S7-3^S zZpy8}hG`K5ar)S+eb={@lRQX#tPxD}O>gU0WM<%Hx2QIAcQL*tR|Pk1D3NO9?|*d& zlY3}OxGx?Jixxx>=;#yks5iZxUyFOd1`*DX*oQo`A)%6)_Q7gxXiL43?+zMc@*(_* z5QKE?_uc0iz8BFVgL6v6mOAW*dxQIkr~{z$dWfU6Xt_e08UV<~eV|5}#JbGtfM8%C zD3!Gu(?8Auf&x;=Sq5^`rXWP+fzLHlwmp9}pxgBP46}A1NB*)n-+{~4P1~8uSr~v` zYgO;_I2)??vXU84Jf?ZZo0_ncAvP0B5XAJri)M$wHM=|r4qKeuA$Eaht9o82X@N%P zNqXF5+G*c8pNfrNno<)O$K60HRj+mDvMKS922AU&<;P)55TbR+B*!Tm55vZ8By9oy zCuy(w!o@-e_iG$wjjMMSk3t--=s4sIz+@5r%WrO-BE1vA zzk|pV(oTy1BH_Hr%m1JztXL31&wrsVBIu0XzmVA$lyvVuFi2AYtVvS=T-@T{ZD}WX zQ)wrJ;lHO%sJW0yR}eL&?MN6&l`&A1z?m8IJ|Z;8Z55bJO7qwN5;6ChnKA^(vE}8A zV^rmTt7`tks+(XT9|hxoIO(>$AMX~b53CSod3+Cb&gO8%+|3hkU>w@Qso zB%JS#fS_C_E!e((-JKOg$nvD^T!7h-f64TuM<2bjK2&HM8IvP)lb_OhP~Jr~aA!Bp z#Ycf3w!Hf;(IYZS*AfFIV_2Cc|E2Z=ydG2l3AWxPnF1}VP5jl*H|?8)>&&BWLYrD< zY(a~Qy;;Owa`?(T+*8r>$H|S6yE)CnO&NH0E7y#q>NdeD72y0y-{CfKgNF%Mv51VE zL2Qr723n8ZXyr7oA9y&G4coSYIr=5&j%GG9{pY}ih8%MjexAHjUH=2D=`0p19PB?S zO!7#C68*39;7QYoP)xx8Us2+3T>=2|&jQFH{{NIHWFUyC{YUvncvGrgZ?R(d8z40N{ECjZqMUsm3S6nR)e5deVH`0wU>N!dwIgoOXx z;tM$C_xwu$pcMw>zxmKi+Wf2ZZ;&9#P&goTNqEWsi{SZ(a86SH--OpcghW#E|0df0 zA?%YD{}NjNID%kmZsphm0T`2k{x?V6o#6j>Fl_&CFtYMv(w$>wR{%iQ{eO>zDoH*C ziWK)h5AEOFBLB@G?*B~Fzp3s@icWzN1x1WYT1|lx14WGgN8%>&rb4lTA}0L1iBE+h MghWe(`fK<<01cm@W&i*H diff --git a/tutorials/html/ecephys.html b/tutorials/html/ecephys.html index 8d674711..df4902e8 100644 --- a/tutorials/html/ecephys.html +++ b/tutorials/html/ecephys.html @@ -92,53 +92,58 @@ Learn more! See the API documentation to learn what data types are available. MATLAB tutorials - Python tutorials

This tutorial

Create fake data for a hypothetical extracellular electrophysiology experiment with a freely moving animal. The types of data we will convert are:
  • Subject (species, strain, age, .etc)
  • Animal position
  • Trials
  • Voltage recording
  • Local field potential (LFP)
  • Spike times

Installing matnwb

Use the code below within the brackets to install MatNWB from source. MatNWB works by automatically creating API classes based on the schema. Use generateCore() to generate these classes.
%{
!git clone https://github.com/NeurodataWithoutBorders/matnwb.git
cd matnwb
addpath(genpath(pwd));
generateCore();
%}

Set up the NWB file

An NWB file represents a single session of an experiment. Each file must have a session_description, identifier, and session start time. Create a new NWBFile object with those and additional metadata. For all MatNWB functions, we use the Matlab method of entering keyword argument pairs, where arguments are entered as name followed by value.
nwb = NwbFile( ...
'session_description', 'mouse in open exploration',...
'identifier', 'Mouse5_Day3', ...
'session_start_time', datetime(2018, 4, 25, 2, 30, 3), ...
'general_experimenter', 'My Name', ... % optional
'general_session_id', 'session_1234', ... % optional
'general_institution', 'University of My Institution', ... % optional
'general_related_publications', 'DOI:10.1016/j.neuron.2016.12.011'); % optional
nwb
nwb =
NwbFile with properties: - - nwb_version: '2.2.5' - acquisition: [0×1 types.untyped.Set] - analysis: [0×1 types.untyped.Set] - file_create_date: [] - general: [0×1 types.untyped.Set] - general_data_collection: [] - general_devices: [0×1 types.untyped.Set] - general_experiment_description: [] - general_experimenter: 'My Name' - general_extracellular_ephys: [0×1 types.untyped.Set] - general_extracellular_ephys_electrodes: [] - general_institution: 'University of My Institution' - general_intracellular_ephys: [0×1 types.untyped.Set] - general_intracellular_ephys_filtering: [] - general_intracellular_ephys_sweep_table: [] - general_keywords: [] - general_lab: [] - general_notes: [] - general_optogenetics: [0×1 types.untyped.Set] - general_optophysiology: [0×1 types.untyped.Set] - general_pharmacology: [] - general_protocol: [] - general_related_publications: 'DOI:10.1016/j.neuron.2016.12.011' - general_session_id: 'session_1234' - general_slices: [] - general_source_script: [] - general_source_script_file_name: [] - general_stimulus: [] - general_subject: [] - general_surgery: [] - general_virus: [] - identifier: 'Mouse5_Day3' - intervals: [0×1 types.untyped.Set] - intervals_epochs: [] - intervals_invalid_times: [] - intervals_trials: [] - processing: [0×1 types.untyped.Set] - scratch: [0×1 types.untyped.Set] - session_description: 'mouse in open exploration' - session_start_time: 2018-04-25T02:30:03.000000-04:00 - stimulus_presentation: [0×1 types.untyped.Set] - stimulus_templates: [0×1 types.untyped.Set] - timestamps_reference_time: [] - units: [] -

Subject information

Create a Subject object to store information about the experimental subject, such as age, species, genotype, sex, and a freeform description. Then set nwb.general_subject to the Subject object.
Each of these fields is free-form, so any values will be valid, but here are our recommendations:
  • For age, we recommend using the ISO 8601 Duration format
  • For species, we recommend using the formal latin binomal name (e.g. mouse -> Mus musculus, human -> Homo sapiens)
  • For sex, we recommend using F (female), M (male), U (unknown), and O (other)
subject = types.core.Subject( ...
'subject_id', '001', ...
'age', 'P90D', ...
'description', 'mouse 5', ...
'species', 'Mus musculus', ...
'sex', 'M')
subject =
Subject with properties: + Python tutorials

This tutorial

Create fake data for a hypothetical extracellular electrophysiology experiment with a freely moving animal. The types of data we will convert are:
  • Subject (species, strain, age, .etc)
  • Animal position
  • Trials
  • Voltage recording
  • Local field potential (LFP)
  • Spike times

Installing matnwb

Use the code below within the brackets to install MatNWB from source. MatNWB works by automatically creating API classes based on the schema. Use generateCore() to generate these classes.
%{
!git clone https://github.com/NeurodataWithoutBorders/matnwb.git
cd matnwb
addpath(genpath(pwd));
generateCore();
%}

Set up the NWB file

An NWB file represents a single session of an experiment. Each file must have a session_description, identifier, and session start time. Create a new NWBFile object with those and additional metadata. For all MatNWB functions, we use the Matlab method of entering keyword argument pairs, where arguments are entered as name followed by value.
nwb = NwbFile( ...
'session_description', 'mouse in open exploration',...
'identifier', 'Mouse5_Day3', ...
'session_start_time', datetime(2018, 4, 25, 2, 30, 3), ...
'general_experimenter', 'My Name', ... % optional
'general_session_id', 'session_1234', ... % optional
'general_institution', 'University of My Institution', ... % optional
'general_related_publications', 'DOI:10.1016/j.neuron.2016.12.011'); % optional
nwb
nwb =
NwbFile with properties: + + nwb_version: '2.4.0' + acquisition: [0×1 types.untyped.Set] + analysis: [0×1 types.untyped.Set] + file_create_date: [] + general: [0×1 types.untyped.Set] + general_data_collection: [] + general_devices: [0×1 types.untyped.Set] + general_experiment_description: [] + general_experimenter: 'My Name' + general_extracellular_ephys: [0×1 types.untyped.Set] + general_extracellular_ephys_electrodes: [] + general_institution: 'University of My Institution' + general_intracellular_ephys: [0×1 types.untyped.Set] + general_intracellular_ephys_experimental_conditions: [] + general_intracellular_ephys_filtering: [] + general_intracellular_ephys_intracellular_recordings: [] + general_intracellular_ephys_repetitions: [] + general_intracellular_ephys_sequential_recordings: [] + general_intracellular_ephys_simultaneous_recordings: [] + general_intracellular_ephys_sweep_table: [] + general_keywords: [] + general_lab: [] + general_notes: [] + general_optogenetics: [0×1 types.untyped.Set] + general_optophysiology: [0×1 types.untyped.Set] + general_pharmacology: [] + general_protocol: [] + general_related_publications: 'DOI:10.1016/j.neuron.2016.12.011' + general_session_id: 'session_1234' + general_slices: [] + general_source_script: [] + general_source_script_file_name: [] + general_stimulus: [] + general_subject: [] + general_surgery: [] + general_virus: [] + identifier: 'Mouse5_Day3' + intervals: [0×1 types.untyped.Set] + intervals_epochs: [] + intervals_invalid_times: [] + intervals_trials: [] + processing: [0×1 types.untyped.Set] + scratch: [0×1 types.untyped.Set] + session_description: 'mouse in open exploration' + session_start_time: 2018-04-25T02:30:03.000000-04:00 + stimulus_presentation: [0×1 types.untyped.Set] + stimulus_templates: [0×1 types.untyped.Set] + timestamps_reference_time: [] + units: [] +

Subject information

Create a Subject object to store information about the experimental subject, such as age, species, genotype, sex, and a freeform description. Then set nwb.general_subject to the Subject object.
Each of these fields is free-form, so any values will be valid, but here are our recommendations:
  • For age, we recommend using the ISO 8601 Duration format
  • For species, we recommend using the formal latin binomal name (e.g. mouse -> Mus musculus, human -> Homo sapiens)
  • For sex, we recommend using F (female), M (male), U (unknown), and O (other)
subject = types.core.Subject( ...
'subject_id', '001', ...
'age', 'P90D', ...
'description', 'mouse 5', ...
'species', 'Mus musculus', ...
'sex', 'M')
subject =
Subject with properties: age: 'P90D' date_of_birth: [] @@ -146,9 +151,10 @@ genotype: [] sex: 'M' species: 'Mus musculus' + strain: [] subject_id: '001' weight: [] -
nwb.general_subject = subject;

SpatialSeries and Position

Many types of data have special data types in NWB. To store the spatial position of a subject, we will use the SpatialSeries and Position classes.
SpatialSeries is a subclass of TimeSeries. TimeSeries is a common base class for measurements sampled over time, and provides fields for data and time (regularly or irregularly sampled).
Here, we put a SpatialSeries object called 'SpatialSeries' in a Position object, and put that in a ProcessingModule named 'behavior'.
position_data = [linspace(0,10,100); linspace(0,8,100)];
spatial_series_ts = types.core.SpatialSeries( ...
'data', position_data, ...
'reference_frame', '(0,0) is bottom left corner', ...
'timestamps', linspace(0, 100)/200)
spatial_series_ts =
SpatialSeries with properties: +
nwb.general_subject = subject;

SpatialSeries and Position

Many types of data have special data types in NWB. To store the spatial position of a subject, we will use the SpatialSeries and Position classes.
SpatialSeries is a subclass of TimeSeries. TimeSeries is a common base class for measurements sampled over time, and provides fields for data and time (regularly or irregularly sampled).
Here, we put a SpatialSeries object called 'SpatialSeries' in a Position object, and put that in a ProcessingModule named 'behavior'.
position_data = [linspace(0,10,100); linspace(0,8,100)];
spatial_series_ts = types.core.SpatialSeries( ...
'data', position_data, ...
'reference_frame', '(0,0) is bottom left corner', ...
'timestamps', linspace(0, 100)/200)
spatial_series_ts =
SpatialSeries with properties: reference_frame: '(0,0) is bottom left corner' starting_time_unit: 'seconds' @@ -158,6 +164,7 @@ control: [] control_description: [] data: [2×100 double] + data_continuity: [] data_conversion: 1 data_resolution: -1 data_unit: 'meters' @@ -165,58 +172,63 @@ starting_time: [] starting_time_rate: [] timestamps: [1×100 double] -
To help data analysis and visualization tools know that this SpatialSeries object represents the position of the animal, store the SpatialSeries object inside of a Position object.
Position = types.core.Position('SpatialSeries', spatial_series_ts);
NWB differentiates between raw, acquired data, which should never change, and processed data, which are the results of preprocessing algorithms and could change. Let's assume that the animal's position was computed from a video tracking algorithm, so it would be classified as processed data. Since processed data can be very diverse, NWB allows us to create processing modules, which are like folders, to store related processed data or data that comes from a single algorithm.
Create a processing module called "behavior" for storing behavioral data in the NWBFile and add the Position object to the module.
% create processing module
behavior_mod = types.core.ProcessingModule( ...
'description', 'contains behavioral data')
behavior_mod =
ProcessingModule with properties: +
To help data analysis and visualization tools know that this SpatialSeries object represents the position of the animal, store the SpatialSeries object inside of a Position object.
Position = types.core.Position('SpatialSeries', spatial_series_ts);
NWB differentiates between raw, acquired data, which should never change, and processed data, which are the results of preprocessing algorithms and could change. Let's assume that the animal's position was computed from a video tracking algorithm, so it would be classified as processed data. Since processed data can be very diverse, NWB allows us to create processing modules, which are like folders, to store related processed data or data that comes from a single algorithm.
Create a processing module called "behavior" for storing behavioral data in the NWBFile and add the Position object to the module.
% create processing module
behavior_mod = types.core.ProcessingModule( ...
'description', 'contains behavioral data')
behavior_mod =
ProcessingModule with properties: description: 'contains behavioral data' dynamictable: [0×1 types.untyped.Set] nwbdatainterface: [0×1 types.untyped.Set] -
% add the Position object (that holds the SpatialSeries object)
behavior_mod.nwbdatainterface.set(...
'Position', Position);
% add the processing module to the NWBFile object, and name it "behavior"
nwb.processing.set('behavior', behavior_mod);

Test write

Now, write the NWB file that we have built so far.
nwbExport(nwb, 'ecephys_tutorial1.nwb')
We can then read the file and print it to inspect its contents. We can also print the SpatialSeries data that we created by referencing the names of the objects in the hierarchy that contain it. The processing module called 'behavior' contains our Position object. By default, the Position object is named 'Position'. The Position object contains our SpatialSeries object named 'SpatialSeries'.
read_nwbfile = nwbRead('ecephys_tutorial1.nwb')
read_nwbfile =
NwbFile with properties: - - nwb_version: '2.2.5' - acquisition: [0×1 types.untyped.Set] - analysis: [0×1 types.untyped.Set] - file_create_date: [1×1 types.untyped.DataStub] - general: [0×1 types.untyped.Set] - general_data_collection: [] - general_devices: [0×1 types.untyped.Set] - general_experiment_description: [] - general_experimenter: [1×1 types.untyped.DataStub] - general_extracellular_ephys: [0×1 types.untyped.Set] - general_extracellular_ephys_electrodes: [] - general_institution: 'University of My Institution' - general_intracellular_ephys: [0×1 types.untyped.Set] - general_intracellular_ephys_filtering: [] - general_intracellular_ephys_sweep_table: [] - general_keywords: [] - general_lab: [] - general_notes: [] - general_optogenetics: [0×1 types.untyped.Set] - general_optophysiology: [0×1 types.untyped.Set] - general_pharmacology: [] - general_protocol: [] - general_related_publications: [1×1 types.untyped.DataStub] - general_session_id: 'session_1234' - general_slices: [] - general_source_script: [] - general_source_script_file_name: [] - general_stimulus: [] - general_subject: [1×1 types.core.Subject] - general_surgery: [] - general_virus: [] - identifier: 'Mouse5_Day3' - intervals: [0×1 types.untyped.Set] - intervals_epochs: [] - intervals_invalid_times: [] - intervals_trials: [] - processing: [1×1 types.untyped.Set] - scratch: [0×1 types.untyped.Set] - session_description: 'mouse in open exploration' - session_start_time: 2018-04-25T02:30:03.000000-04:00 - stimulus_presentation: [0×1 types.untyped.Set] - stimulus_templates: [0×1 types.untyped.Set] - timestamps_reference_time: 2018-04-25T02:30:03.000000-04:00 - units: [] -
read_nwbfile.processing.get('behavior').nwbdatainterface.get('Position').spatialseries.get('SpatialSeries')
ans =
SpatialSeries with properties: +
% add the Position object (that holds the SpatialSeries object)
behavior_mod.nwbdatainterface.set(...
'Position', Position);
% add the processing module to the NWBFile object, and name it "behavior"
nwb.processing.set('behavior', behavior_mod);

Test write

Now, write the NWB file that we have built so far.
nwbExport(nwb, 'ecephys_tutorial1.nwb')
We can then read the file and print it to inspect its contents. We can also print the SpatialSeries data that we created by referencing the names of the objects in the hierarchy that contain it. The processing module called 'behavior' contains our Position object. By default, the Position object is named 'Position'. The Position object contains our SpatialSeries object named 'SpatialSeries'.
read_nwbfile = nwbRead('ecephys_tutorial1.nwb')
read_nwbfile =
NwbFile with properties: + + nwb_version: '2.4.0' + acquisition: [0×1 types.untyped.Set] + analysis: [0×1 types.untyped.Set] + file_create_date: [1×1 types.untyped.DataStub] + general: [0×1 types.untyped.Set] + general_data_collection: [] + general_devices: [0×1 types.untyped.Set] + general_experiment_description: [] + general_experimenter: [1×1 types.untyped.DataStub] + general_extracellular_ephys: [0×1 types.untyped.Set] + general_extracellular_ephys_electrodes: [] + general_institution: 'University of My Institution' + general_intracellular_ephys: [0×1 types.untyped.Set] + general_intracellular_ephys_experimental_conditions: [] + general_intracellular_ephys_filtering: [] + general_intracellular_ephys_intracellular_recordings: [] + general_intracellular_ephys_repetitions: [] + general_intracellular_ephys_sequential_recordings: [] + general_intracellular_ephys_simultaneous_recordings: [] + general_intracellular_ephys_sweep_table: [] + general_keywords: [] + general_lab: [] + general_notes: [] + general_optogenetics: [0×1 types.untyped.Set] + general_optophysiology: [0×1 types.untyped.Set] + general_pharmacology: [] + general_protocol: [] + general_related_publications: [1×1 types.untyped.DataStub] + general_session_id: 'session_1234' + general_slices: [] + general_source_script: [] + general_source_script_file_name: [] + general_stimulus: [] + general_subject: [1×1 types.core.Subject] + general_surgery: [] + general_virus: [] + identifier: 'Mouse5_Day3' + intervals: [0×1 types.untyped.Set] + intervals_epochs: [] + intervals_invalid_times: [] + intervals_trials: [] + processing: [1×1 types.untyped.Set] + scratch: [0×1 types.untyped.Set] + session_description: 'mouse in open exploration' + session_start_time: 2018-04-25T02:30:03.000000-04:00 + stimulus_presentation: [0×1 types.untyped.Set] + stimulus_templates: [0×1 types.untyped.Set] + timestamps_reference_time: 2018-04-25T02:30:03.000000-04:00 + units: [] +
read_nwbfile.processing.get('behavior').nwbdatainterface.get('Position').spatialseries.get('SpatialSeries')
ans =
SpatialSeries with properties: reference_frame: '(0,0) is bottom left corner' starting_time_unit: 'seconds' @@ -226,6 +238,7 @@ control: [] control_description: [] data: [1×1 types.untyped.DataStub] + data_continuity: [] data_conversion: 1 data_resolution: -1 data_unit: 'meters' @@ -233,7 +246,7 @@ starting_time: [] starting_time_rate: [] timestamps: [1×1 types.untyped.DataStub] -
We can also use the HDFView tool to inspect the resulting NWB file.

Trials

Trials are stored in a TimeIntervals object which is a subclass of DynamicTable. DynamicTable objects are used to store tabular metadata throughout NWB, including for trials, electrodes, and sorted units. They offer flexibility for tabular data by allowing required columns, optional columns, and custom columns.
The trials DynamicTable can be thought of as a table with this structure:
Trials are stored in a TimeInterval object which subclasses DynamicTable. Here, we are adding 'correct', which will be a boolean array.
trials = types.core.TimeIntervals( ...
'colnames', {'start_time', 'stop_time', 'correct'}, ...
'description', 'trial data and properties', ...
'id', types.hdmf_common.ElementIdentifiers('data', 0:2), ...
'start_time', types.hdmf_common.VectorData('data', [.1, 1.5, 2.5], ...
'description','start time of trial'), ...
'stop_time', types.hdmf_common.VectorData('data', [1., 2., 3.], ...
'description','end of each trial'), ...
'correct', types.hdmf_common.VectorData('data', [false, true, false], ...
'description', 'whether the trial was correct'))
trials =
TimeIntervals with properties: +
We can also use the HDFView tool to inspect the resulting NWB file.

Trials

Trials are stored in a TimeIntervals object which is a subclass of DynamicTable. DynamicTable objects are used to store tabular metadata throughout NWB, including for trials, electrodes, and sorted units. They offer flexibility for tabular data by allowing required columns, optional columns, and custom columns.
The trials DynamicTable can be thought of as a table with this structure:
Trials are stored in a TimeInterval object which subclasses DynamicTable. Here, we are adding 'correct', which will be a boolean array.
trials = types.core.TimeIntervals( ...
'colnames', {'start_time', 'stop_time', 'correct'}, ...
'description', 'trial data and properties', ...
'id', types.hdmf_common.ElementIdentifiers('data', 0:2), ...
'start_time', types.hdmf_common.VectorData( ...
'data', [.1, 1.5, 2.5], ...
'description','start time of trial' ...
), ...
'stop_time', types.hdmf_common.VectorData( ...
'data', [1., 2., 3.], ...
'description','end of each trial' ...
), ...
'correct', types.hdmf_common.VectorData( ...
'data', [false, true, false], ...
'description', 'whether the trial was correct') ...
)
trials =
TimeIntervals with properties: start_time: [1×1 types.hdmf_common.VectorData] stop_time: [1×1 types.hdmf_common.VectorData] @@ -245,22 +258,21 @@ description: 'trial data and properties' id: [1×1 types.hdmf_common.ElementIdentifiers] vectordata: [1×1 types.untyped.Set] - vectorindex: [0×1 types.untyped.Set] -
nwb.intervals_trials = trials;

Extracellular electrophysiology

In order to store extracellular electrophysiology data, you first must create an electrodes table describing the electrodes that generated this data. Extracellular electrodes are stored in a electrodes table, which is also a DynamicTable. electrodes has several required fields: x, y, z, impedence, location, filtering, and electrode_group.

Electrode table

Since this is a DynamicTable, we can add additional metadata fields. We will be adding a "label" column to the table.
Here, we also demonstate another method for creating DynamicTables, by first creating a MATLAB native Table object and then calling util.table2nwb to convert this Table object into a DynamicTable.
nshanks = 4;
nchannels_per_shank = 3;
variables = {'x', 'y', 'z', 'imp', 'location', 'filtering', 'group', 'label'};
tbl = cell2table(cell(0, length(variables)), 'VariableNames', variables);
device = types.core.Device(...
'description', 'the best array', ...
'manufacturer', 'Probe Company 9000');
device_name = 'array';
nwb.general_devices.set(device_name, device);
device_link = types.untyped.SoftLink(['/general/devices/' device_name]);
for ishank = 1:nshanks
group_name = ['shank' num2str(ishank)];
nwb.general_extracellular_ephys.set(group_name, ...
types.core.ElectrodeGroup( ...
'description', ['electrode group for shank' num2str(ishank)], ...
'location', 'brain area', ...
'device', device_link));
group_object_view = types.untyped.ObjectView( ...
['/general/extracellular_ephys/' group_name]);
for ielec = 1:nchannels_per_shank
tbl = [tbl; {5.3, 1.5, 8.5, NaN, 'unknown', 'unknown', ...
group_object_view, [group_name 'elec' num2str(ielec)]}];
end
end
tbl
tbl = 12×8 table
 xyzimplocationfilteringgrouplabel
15.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank1elec1'
25.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank1elec2'
35.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank1elec3'
45.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank2elec1'
55.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank2elec2'
65.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank2elec3'
75.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank3elec1'
85.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank3elec2'
95.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank3elec3'
105.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank4elec1'
115.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank4elec2'
125.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank4elec3'
electrode_table = util.table2nwb(tbl, 'all electrodes');
nwb.general_extracellular_ephys_electrodes = electrode_table;

Links

In the above loop, we create ElectrodeGroup objects. The electrodes table then uses an ObjectView in each row to link to the corresponding ElectrodeGroup object. An ObjectView is an object that allow you to create a link from one neurodata type referencing another.

ElectricalSeries

Voltage data are stored in ElectricalSeries objects. ElectricalSeries is a subclass of TimeSeries specialized for voltage data. In order to create our ElectricalSeries object, we will need to reference a set of rows in the electrodes table to indicate which electrodes were recorded. We will do this by creating a DynamicTableRegion, which is a type of link that allows you to reference specific rows of a DynamicTable, such as the electrodes table, by row indices.
Create a DynamicTableRegion that references all rows of the electrodes table.
electrodes_object_view = types.untyped.ObjectView( ...
'/general/extracellular_ephys/electrodes');
electrode_table_region = types.hdmf_common.DynamicTableRegion( ...
'table', electrodes_object_view, ...
'description', 'all electrodes', ...
'data', (0:height(tbl)-1)');
Now create an ElectricalSeries object to hold acquisition data collected during the experiment.
electrical_series = types.core.ElectricalSeries( ...
'starting_time', 0.0, ... % seconds
'starting_time_rate', 30000., ... % Hz
'data', randn(12, 3000), ...
'electrodes', electrode_table_region, ...
'data_unit', 'volts');
This is the voltage data recorded directly from our electrodes, so it goes in the acquisition group.
nwb.acquisition.set('ElectricalSeries', electrical_series);

LFP

Local field potential (LFP) refers in this case to data that has been downsampled and/or filtered from the original acquisition data and is used to analyze signals in the lower frequency range. Filtered and downsampled LFP data would also be stored in an ElectricalSeries. To help data analysis and visualization tools know that this ElectricalSeries object represents LFP data, store it inside an LFP object, then place the LFP object in a ProcessingModule named 'ecephys'. This is analogous to how we stored the SpatialSeries object inside of a Position object and stored the Position object in a ProcessingModule named 'behavior' earlier.
electrical_series = types.core.ElectricalSeries( ...
'starting_time', 0.0, ... % seconds
'starting_time_rate', 1000., ... % Hz
'data', randn(12, 100), ...
'electrodes', electrode_table_region, ...
'data_unit', 'volts');
lfp = types.core.LFP('ElectricalSeries', electrical_series);
ecephys_module = types.core.ProcessingModule(...
'description', 'extracellular electrophysiology');
ecephys_module.nwbdatainterface.set('LFP', lfp);
nwb.processing.set('ecephys', ecephys_module);

Spike times

Ragged arrays

Spike times are stored in another DynamicTable of subtype Units. The default Units table is at /units in the HDF5 file. You can add columns to the Units table just like you did for electrodes and trials. Here, we generate some random spike data and populate the table.
num_cells = 10;
firing_rate = 20;
spikes = cell(1, num_cells);
for ishank = 1:num_cells
spikes{ishank} = [];
for iunit = 1:poissrnd(20)
spikes{ishank}(end+1) = cumsum(exprnd(1/firing_rate));
end
end
spikes
spikes = 1×10 cell
 12345678910
11×29 double1×25 double1×16 double1×12 double1×23 double1×22 double1×16 double1×21 double1×22 double1×22 double
Spike times are an example of a ragged array- it's like a matrix, but each row has a different number of elements. We can represent this type of data as an indexed column of the units DynamicTable. These indexed columns have two components, the vector data object that holds the data and the vector index object that holds the indices in the vector that indicate the row breaks. You can use the convenience function util.create_indexed_column to create these objects.
[spike_times_vector, spike_times_index] = util.create_indexed_column( ...
spikes, '/units/spike_times');
nwb.units = types.core.Units( ...
'colnames', {'spike_times'}, ...
'description', 'units table', ...
'id', types.hdmf_common.ElementIdentifiers( ...
'data', int64(0:length(spikes) - 1)), ...
'spike_times', spike_times_vector, ...
'spike_times_index', spike_times_index);

Write the file

nwbExport(nwb, 'ecephys_tutorial.nwb')

Reading NWB data

Data arrays are read passively from the file. Calling TimeSeries.data does not read the data values, but presents an HDF5 object that can be indexed to read data. This allows you to conveniently work with datasets that are too large to fit in RAM all at once. load with no input arguments reads the entire dataset:
nwb2 = nwbRead('ecephys_tutorial.nwb');
nwb2.processing.get('ecephys'). ...
nwbdatainterface.get('LFP'). ...
electricalseries.get('ElectricalSeries'). ...
data.load;

Accessing data regions

If all you need is a data region, you can index a DataStub object like you would any normal array in MATLAB, as shown below. When indexing the dataset this way, only the selected region is read from disk into RAM. This allows you to handle very large datasets that would not fit entirely into RAM.
% read section of LFP
nwb2.processing.get('ecephys'). ...
nwbdatainterface.get('LFP'). ...
electricalseries.get('ElectricalSeries'). ...
data(1:5, 1:10)
ans = 5×10
-0.2473 -0.5852 0.1209 -0.5848 -1.3164 -1.4034 -0.8801 -0.2093 0.3934 0.6250 - 0.5297 0.4089 0.2502 -0.6511 -0.1110 -0.3925 -1.2349 -1.0213 -1.8974 -0.1869 - 2.4135 -2.0341 -1.8944 1.1283 -0.9568 -1.1572 0.8480 1.1798 2.5870 -0.4809 - 1.3274 -0.6456 0.2910 -0.0223 0.9700 0.4814 0.6749 0.2758 -0.2979 1.1617 - -0.2980 -0.7820 -1.0702 1.2701 -1.9685 -0.6576 1.7393 -0.6578 0.1026 -1.0548 -
% You can use the utility function |util.read_indexed_column| to read the
% spike times of a specific unit.
util.read_indexed_column( ...
nwb.units.spike_times_index, nwb.units.spike_times, 1)
ans = 29×1
0.0001 - 0.0287 - 0.1586 - 0.0126 - 0.0972 - 0.0785 - 0.0520 - 0.0556 - 0.0270 - 0.0492 +
nwb.intervals_trials = trials;

Extracellular electrophysiology

In order to store extracellular electrophysiology data, you first must create an electrodes table describing the electrodes that generated this data. Extracellular electrodes are stored in a electrodes table, which is also a DynamicTable. electrodes has several required fields: x, y, z, impedence, location, filtering, and electrode_group.

Electrode table

Since this is a DynamicTable, we can add additional metadata fields. We will be adding a "label" column to the table.
Here, we also demonstate another method for creating DynamicTables, by first creating a MATLAB native Table object and then calling util.table2nwb to convert this Table object into a DynamicTable.
nshanks = 4;
nchannels_per_shank = 3;
variables = {'x', 'y', 'z', 'imp', 'location', 'filtering', 'group', 'label'};
tbl = cell2table(cell(0, length(variables)), 'VariableNames', variables);
device = types.core.Device(...
'description', 'the best array', ...
'manufacturer', 'Probe Company 9000' ...
);
nwb.general_devices.set('array', device);
for ishank = 1:nshanks
electrode_group = types.core.ElectrodeGroup( ...
'description', ['electrode group for shank' num2str(ishank)], ...
'location', 'brain area', ...
'device', types.untyped.SoftLink(device) ...
);
nwb.general_extracellular_ephys.set(['shank' num2str(ishank)], electrode_group);
group_object_view = types.untyped.ObjectView(electrode_group);
for ielec = 1:nchannels_per_shank
electrode_label = ['shank' num2str(ishank) 'elec' num2str(ielec)];
tbl = [...
tbl; ...
{5.3, 1.5, 8.5, NaN, 'unknown', 'unknown', group_object_view, electrode_label} ...
];
end
end
tbl
tbl = 12×8 table
 xyzimplocationfilteringgrouplabel
15.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank1elec1'
25.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank1elec2'
35.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank1elec3'
45.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank2elec1'
55.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank2elec2'
65.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank2elec3'
75.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank3elec1'
85.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank3elec2'
95.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank3elec3'
105.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank4elec1'
115.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank4elec2'
125.30001.50008.5000NaN'unknown''unknown'1×1 ObjectView'shank4elec3'
electrode_table = util.table2nwb(tbl, 'all electrodes');
nwb.general_extracellular_ephys_electrodes = electrode_table;

Links

In the above loop, we create ElectrodeGroup objects. The electrodes table then uses an ObjectView in each row to link to the corresponding ElectrodeGroup object. An ObjectView is an object that allow you to create a link from one neurodata type referencing another.

ElectricalSeries

Voltage data are stored in ElectricalSeries objects. ElectricalSeries is a subclass of TimeSeries specialized for voltage data. In order to create our ElectricalSeries object, we will need to reference a set of rows in the electrodes table to indicate which electrodes were recorded. We will do this by creating a DynamicTableRegion, which is a type of link that allows you to reference specific rows of a DynamicTable, such as the electrodes table, by row indices.
Create a DynamicTableRegion that references all rows of the electrodes table.
electrode_table_region = types.hdmf_common.DynamicTableRegion( ...
'table', types.untyped.ObjectView(electrode_table), ...
'description', 'all electrodes', ...
'data', (0:height(tbl)-1)');
Now create an ElectricalSeries object to hold acquisition data collected during the experiment.
electrical_series = types.core.ElectricalSeries( ...
'starting_time', 0.0, ... % seconds
'starting_time_rate', 30000., ... % Hz
'data', randn(12, 3000), ...
'electrodes', electrode_table_region, ...
'data_unit', 'volts');
This is the voltage data recorded directly from our electrodes, so it goes in the acquisition group.
nwb.acquisition.set('ElectricalSeries', electrical_series);

LFP

Local field potential (LFP) refers in this case to data that has been downsampled and/or filtered from the original acquisition data and is used to analyze signals in the lower frequency range. Filtered and downsampled LFP data would also be stored in an ElectricalSeries. To help data analysis and visualization tools know that this ElectricalSeries object represents LFP data, store it inside an LFP object, then place the LFP object in a ProcessingModule named 'ecephys'. This is analogous to how we stored the SpatialSeries object inside of a Position object and stored the Position object in a ProcessingModule named 'behavior' earlier.
electrical_series = types.core.ElectricalSeries( ...
'starting_time', 0.0, ... % seconds
'starting_time_rate', 1000., ... % Hz
'data', randn(12, 100), ...
'electrodes', electrode_table_region, ...
'data_unit', 'volts');
lfp = types.core.LFP('ElectricalSeries', electrical_series);
ecephys_module = types.core.ProcessingModule(...
'description', 'extracellular electrophysiology');
ecephys_module.nwbdatainterface.set('LFP', lfp);
nwb.processing.set('ecephys', ecephys_module);

Spike times

Ragged arrays

Spike times are stored in another DynamicTable of subtype Units. The default Units table is at /units in the HDF5 file. You can add columns to the Units table just like you did for electrodes and trials. Here, we generate some random spike data and populate the table.
num_cells = 10;
firing_rate = 20;
spikes = cell(1, num_cells);
for ishank = 1:num_cells
spikes{ishank} = [];
for iunit = 1:poissrnd(20)
spikes{ishank}(end+1) = cumsum(exprnd(1/firing_rate));
end
end
spikes
spikes = 1×10 cell
 12345678910
11×28 double1×25 double1×22 double1×23 double1×18 double1×21 double1×18 double1×29 double1×15 double1×20 double
Spike times are an example of a ragged array- it's like a matrix, but each row has a different number of elements. We can represent this type of data as an indexed column of the units DynamicTable. These indexed columns have two components, the vector data object that holds the data and the vector index object that holds the indices in the vector that indicate the row breaks. You can use the convenience function util.create_indexed_column to create these objects.
[spike_times_vector, spike_times_index] = util.create_indexed_column(spikes);
nwb.units = types.core.Units( ...
'colnames', {'spike_times'}, ...
'description', 'units table', ...
'id', types.hdmf_common.ElementIdentifiers( ...
'data', int64(0:length(spikes) - 1) ...
), ...
'spike_times', spike_times_vector, ...
'spike_times_index', spike_times_index ...
);

Write the file

nwbExport(nwb, 'ecephys_tutorial.nwb')

Reading NWB data

Data arrays are read passively from the file. Calling TimeSeries.data does not read the data values, but presents an HDF5 object that can be indexed to read data. This allows you to conveniently work with datasets that are too large to fit in RAM all at once. load with no input arguments reads the entire dataset:
nwb2 = nwbRead('ecephys_tutorial.nwb');
nwb2.processing.get('ecephys'). ...
nwbdatainterface.get('LFP'). ...
electricalseries.get('ElectricalSeries'). ...
data.load;

Accessing data regions

If all you need is a data region, you can index a DataStub object like you would any normal array in MATLAB, as shown below. When indexing the dataset this way, only the selected region is read from disk into RAM. This allows you to handle very large datasets that would not fit entirely into RAM.
% read section of LFP
nwb2.processing.get('ecephys'). ...
nwbdatainterface.get('LFP'). ...
electricalseries.get('ElectricalSeries'). ...
data(1:5, 1:10)
ans = 5×10
0.3238 0.0568 0.1861 -1.0841 0.4015 1.5673 0.5038 0.9148 0.6305 -1.8499 + -0.2572 1.1631 -0.2843 -2.0889 -0.6124 -0.0981 -0.1522 -0.6394 -1.9411 0.2052 + -1.0528 -0.1458 0.1278 1.6205 1.2195 -0.3119 0.8782 -0.0532 -0.5970 1.4307 + -1.3472 -1.0314 -0.7384 0.0003 1.1219 0.0319 -1.0600 0.4216 -1.6961 0.5677 + -0.5865 2.4174 -0.6169 -0.6104 -0.8354 -0.3832 0.4795 -1.4911 0.3497 -0.2414 +
% You can use the utility function |util.read_indexed_column| to read the
% spike times of a specific unit.
util.read_indexed_column( ...
nwb.units.spike_times_index, nwb.units.spike_times, 1)
ans = 28×1
0.0192 + 0.0151 + 0.0079 + 0.0401 + 0.0393 + 0.0343 + 0.0657 + 0.0066 + 0.0230 + 0.0292

Learn more!

See the API documentation to learn what data types are available.

MATLAB tutorials

Python tutorials

See our tutorials for more details about your data type:
Check out other tutorials that teach advanced NWB topics: