From 3222f33258775c1836c0aee670b3be1c1f74531b Mon Sep 17 00:00:00 2001 From: Lawrence Date: Fri, 20 Apr 2018 16:47:54 -0400 Subject: [PATCH] basic read without links or refs now work. --- +file/fillClass.m | 29 +++++++++---- +file/fillConstructor.m | 59 ++++++++++++++++++++------- +file/fillValidators.m | 27 +++++++------ +io/parseDataset.m | 41 ++++++++++++------- +io/parseGroup.m | 53 ++++++++++++++++++------ +io/parseLink.m | 4 -- +types/+util/checkConstrained.m | 3 ++ +types/+util/checkDependent.m | 10 +++++ +types/+util/checkDims.m | 7 +++- +types/+util/checkDtype.m | 7 +++- nwbRead.m | 18 ++------- nwbfile.m | 72 ++++++++++++++++++++------------- 12 files changed, 221 insertions(+), 109 deletions(-) delete mode 100644 +io/parseLink.m create mode 100644 +types/+util/checkDependent.m diff --git a/+file/fillClass.m b/+file/fillClass.m index 9006a0ad..0bfc1773 100644 --- a/+file/fillClass.m +++ b/+file/fillClass.m @@ -11,6 +11,7 @@ optional = {}; readonly = {}; defaults = {}; +dependent = {}; %separate into readonly, required, and optional properties for i=1:length(allprops) pnm = allprops{i}; @@ -22,12 +23,23 @@ optional = [optional {pnm}]; end - if isa(prop, 'file.Attribute') && prop.readonly - readonly = [readonly {pnm}]; - end - - if isa(prop, 'file.Attribute') && ~isempty(prop.value) - defaults = [defaults {pnm}]; + if isa(prop, 'file.Attribute') + if prop.readonly + readonly = [readonly {pnm}]; + end + + if ~isempty(prop.value) + defaults = [defaults {pnm}]; + end + + if ~isempty(prop.dependent) + %extract prefix + parentname = strrep(pnm, ['_' prop.name], ''); + parent = classprops.named(parentname); + if ~parent.required + dependent = [dependent {pnm}]; + end + end end end non_inherited = setdiff(allprops, inherited); @@ -50,8 +62,8 @@ '% ' name ' ' class.doc]; %name, docstr propgroups = {... @()file.fillProps(classprops.named, ro_unique, 'SetAccess=protected')... - @()file.fillProps(classprops.named, req_unique)... - @()file.fillProps(classprops.named, opt_unique)... + @()file.fillProps(classprops.named, setdiff(req_unique, ro_unique))... + @()file.fillProps(classprops.named, setdiff(opt_unique, ro_unique))... }; docsep = {... '% READONLY'... @@ -70,6 +82,7 @@ constructorBody = file.fillConstructor(name,... depnm,... defaults,... %all defaults, regardless of inheritance + dependent,... req_unique,... opt_unique,... classprops); diff --git a/+file/fillConstructor.m b/+file/fillConstructor.m index b6d10585..e4580063 100644 --- a/+file/fillConstructor.m +++ b/+file/fillConstructor.m @@ -1,12 +1,12 @@ -function fcstr = fillConstructor(name, parentname, defaults, required, optional, props) +function fcstr = fillConstructor(name, parentname, defaults, dependent, required, optional, props) caps = upper(name); fcnbody = strjoin({['% ' caps ' Constructor for ' name]... ['% obj = ' caps '(parentname1,parentvalue1,..,parentvalueN,parentargN,name1,value1,...,nameN,valueN)']... }, newline); fcns = {... - @()fillParamDocs('REQUIRED', required, props.named)... - @()fillParamDocs('OPTIONAL', optional, props.named)... - @()fillBody(parentname, defaults, required, optional, props)... + @()fillParamDocs('REQUIRED', setdiff(required, dependent), props.named)... + @()fillParamDocs('OPTIONAL', setdiff(optional, dependent), props.named)... + @()fillBody(parentname, defaults, dependent, required, optional, props)... }; for i=1:length(fcns) fcn = fcns{i}; @@ -74,7 +74,7 @@ end end -function bodystr = fillBody(pname, defaults, required, optional, props) +function bodystr = fillBody(pname, defaults, dependent, required, optional, props) if isempty(defaults) bodystr = ''; else @@ -97,15 +97,46 @@ var = params{i}; bodystr = [bodystr newline 'addParameter(p, ''' var ''', []);']; end -req_unset = setdiff(required, defaults); %check required values that don't have a set value -req_body = strjoin({... - 'parse(p, varargin{:});'... - ['required = ' util.cellPrettyPrint(req_unset) ';']... - 'missing = intersect(p.UsingDefaults, required);'... - 'if ~isempty(missing)'... - ' error(''Missing Required Argument(s) { %s }'', strjoin(missing, '', ''));'... - 'end'}, newline); -bodystr = [bodystr newline req_body]; + +bodystr = [bodystr newline 'parse(p, varargin{:});']; + +%check required values that don't have a set value and are independent +req_unset = setdiff(required, [defaults dependent]); +if ~isempty(req_unset) + req_body = strjoin({... + ['required = ' util.cellPrettyPrint(req_unset) ';']... + 'missing = intersect(p.UsingDefaults, required);'... + 'if ~isempty(missing)'... + ' error(''Missing Required Argument(s) { %s }'', strjoin(missing, '', ''));'... + 'end'}, newline); + bodystr = [bodystr newline req_body]; +end + +%construct parent->dependent structure +dep_parents = struct(); +for i=1:length(dependent) + dep = dependent{i}; + dep_p = props.named(dep); + parent = dep_p.dependent; + if ~startsWith(dep, parent) + parent = strrep(dep, ['_' dep_p.name], ''); + end + if isfield(dep_parents, parent) + deps = dep_parents.(parent); + dep_parents.(parent) = [deps {dep}]; + else + dep_parents.(parent) = {dep}; + end +end + +%check each optional parent if they exist, then check their required dependents +dep_p_list = fieldnames(dep_parents); +for i=1:length(dep_p_list) + parent = dep_p_list{i}; + children = util.cellPrettyPrint(dep_parents.(parent)); + bodystr = [bodystr newline 'types.util.checkDependent(''' parent ''', ' children ', p.UsingDefaults);']; +end + for i=1:length(params) var = params{i}; bodystr = [bodystr newline 'obj.' var ' = p.Results.' var ';']; diff --git a/+file/fillValidators.m b/+file/fillValidators.m index c724cae5..8ae1a5b1 100644 --- a/+file/fillValidators.m +++ b/+file/fillValidators.m @@ -39,26 +39,28 @@ constr = {}; for i=1:length(prop.datasets) ds = prop.datasets(i); + if isempty(ds.type) + type = ds.dtype; + else + ds_namespace = namespacereg.getNamespace(ds.type).name; + type = ['types.' ds_namespace '.' ds.type]; + end + if isempty(ds.name) - if isempty(ds.type) - typespec = ds.dtype; - else - typespec = ds.type; - end - constr = [constr {typespec}]; - elseif isempty(ds.type) - namedprops.(ds.name) = ds.dtype; + constr = [constr {type}]; else - namedprops.(ds.name) = ds.type; + namedprops.(ds.name) = type; end end for i=1:length(prop.subgroups) sg = prop.subgroups(i); + sg_namespace = namespacereg.getNamespace(sg.type).name; + sgfullname = ['types.' sg_namespace '.' sg.type]; if isempty(sg.name) - constr = [constr sg.type]; + constr = [constr {sgfullname}]; else - namedprops.(sg.name) = sg.type; + namedprops.(sg.name) = sgfullname; end end @@ -81,7 +83,8 @@ elseif isa(prop, 'file.Attribute') fuvstr = fillDtypeValidation(name, prop.dtype, namespacereg); else %Link - fuvstr = fillDtypeValidation(name, prop.type, namespacereg); + namespace = namespacereg.getNamespace(prop.type).name; + fuvstr = fillDtypeValidation(name, ['types.' namespace '.' prop.type], namespacereg); end end diff --git a/+io/parseDataset.m b/+io/parseDataset.m index cde9468d..fbca61db 100644 --- a/+io/parseDataset.m +++ b/+io/parseDataset.m @@ -36,33 +36,46 @@ datatype = info.Datatype; if strcmp(datatype.Class, 'H5T_REFERENCE') - reftype = datatype.Type; - path = H5R.get_name(did, reftype, data); - props('ref') = []; - if strcmp(reftype, 'H5R_DATASET_REGION') - sid = H5R.get_region(did, reftype, data); - [start, finish] = H5S.get_select_bounds(sid); - H5S.close(sid); - reg = [start finish]; - else - reg = []; - end + [path, reg] = parseReference(did, datatype.Type, data); refs([fullpath '/ref']) = struct('path', path, 'region', reg); + props('ref') = []; elseif strcmp(datatype.Class, 'H5T_COMPOUND') t = table; compound = datatype.Type.Member; - isref = logical(size(compound)); for j=1:length(compound) comp = compound(j); if strcmp(comp.Datatype.Class, 'H5T_REFERENCE') - isref(j) = true; + refnames = data.(comp.Name); + reflist = repmat(struct('path', [], 'region', []), size(refnames)); + for k=1:size(refnames,2) + r = refnames(:,k); + [path, reg] = parseReference(did, comp.Datatype.Type, r); + reflist(k).path = path; + reflist(k).region = reg; + end + refs([fullpath '/' info.Name '.' comp.Name]) = reflist; + %for some reason, raw references are stored transposed. + data.(comp.Name) = cell(size(refnames, 2), 1); end end + props('table') = struct2table(data); else + keyboard; end kwargs = io.map2kwargs(props); parsed = eval([typename '(kwargs{:})']); H5D.close(did); H5F.close(fid); -end \ No newline at end of file +end + +function [target, region] = parseReference(did, type, data) +target = H5R.get_name(did, type, data); +region = []; +if strcmp(type, 'H5R_DATASET_REGION') + sid = H5R.get_region(did, type, data); + [start, finish] = H5S.get_select_bounds(sid); + H5S.close(sid); + region = [start finish]; +end +end diff --git a/+io/parseGroup.m b/+io/parseGroup.m index 9f1486a0..84fb588e 100644 --- a/+io/parseGroup.m +++ b/+io/parseGroup.m @@ -2,10 +2,11 @@ % NOTE, group name is in path format so we need to parse that out. % parsed is either a containers.Map containing properties mapped to values OR a % typed value -links = []; +links = containers.Map; refs = containers.Map; [~, root] = io.pathParts(info.Name); [props, typename] = io.parseAttributes(info.Attributes); +hasTypes = false; %parse datasets for i=1:length(info.Datasets) @@ -16,37 +17,65 @@ props = [props; ds]; else props(ds_info.Name) = ds; + hasTypes = true; end refs = [refs; dsrefs]; end -%parse links if present -for i=1:length(info.Links) - links = [links io.parseLink(info.Links)]; -end - %parse subgroups for i=1:length(info.Groups) g_info = info.Groups(i); [~, gname] = io.pathParts(g_info.Name); [subg, glinks, grefs] = io.parseGroup(filename, g_info); - props(gname) = subg; - links = [links glinks]; + if isa(subg, 'containers.Map') + props = [props; subg]; + else + props(gname) = subg; + hasTypes = true; + end + + links = [links; glinks]; refs = [refs; grefs]; end +%parse links and add to map of links +for i=1:length(info.Links) + l = info.Links(i); + fpname = [info.Name '/' l.Name]; + props(l.Name) = []; + links([info.Name '/' l.Name]) = l; + hasTypes = true; +end + if isempty(typename) - parsed = containers.Map; - keyboard; + %immediately elide prefix all property names with this but only if there are + %no typed objects in it. + propnames = keys(props); + if hasTypes + parsed = containers.Map({root}, {props}); + else + parsed = containers.Map; + for i=1:length(propnames) + pnm = propnames{i}; + p = props(pnm); + parsed([root '_' pnm]) = p; + end + end + + if isempty(parsed) + %special case where a directory is simply empty. Return itself but + %empty + parsed(root) = []; + end else %construct as kwargs and instantiate object + kwargs = io.map2kwargs(props); if isempty(root) %we are root - parsed = types.core.NWBFile; + parsed = types.core.NWBFile(kwargs{:}); return; end - kwargs = io.map2kwargs(props); parsed = eval([typename '(kwargs{:})']); end end \ No newline at end of file diff --git a/+io/parseLink.m b/+io/parseLink.m deleted file mode 100644 index 63af4918..00000000 --- a/+io/parseLink.m +++ /dev/null @@ -1,4 +0,0 @@ -function link = parseLink(info) -% just return a link -link = []; -end \ No newline at end of file diff --git a/+types/+util/checkConstrained.m b/+types/+util/checkConstrained.m index c413d0eb..6f8acf40 100644 --- a/+types/+util/checkConstrained.m +++ b/+types/+util/checkConstrained.m @@ -1,4 +1,7 @@ function checkConstrained(name, namedprops, constrained, val) +if isempty(val) + return; +end if ~isa(val, 'containers.Map') error('Property `%s` must be a containers.Map.', name); end diff --git a/+types/+util/checkDependent.m b/+types/+util/checkDependent.m new file mode 100644 index 00000000..8de45586 --- /dev/null +++ b/+types/+util/checkDependent.m @@ -0,0 +1,10 @@ +function checkDependent(parent, children, unconstructed) + if ~any(strcmp(parent, unconstructed)) + for i=1:length(children) + child = children{i}; + if any(strcmp(child, unconstructed)) + error('Dependent type `%s` is required for parent property `%s`', child, parent); + end + end + end +end \ No newline at end of file diff --git a/+types/+util/checkDims.m b/+types/+util/checkDims.m index 4f6e2411..deb385c1 100644 --- a/+types/+util/checkDims.m +++ b/+types/+util/checkDims.m @@ -1,8 +1,13 @@ function checkDims(valsize, validSizes) + if any(valsize == 0) + return; %ignore empty arrays + end validSizesStrings = cell(size(validSizes)); for i=1:length(validSizes) vs = validSizes{i}; - if length(valsize) ~= length(vs) + + if (length(vs) == 1 && (length(valsize) > 2 || ~any(valsize == 1))) ||... + (length(vs) > 1 && length(valsize) ~= length(vs)) continue; end diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m index ae2fb694..93a145cf 100644 --- a/+types/+util/checkDtype.m +++ b/+types/+util/checkDtype.m @@ -15,11 +15,11 @@ function checkDtype(name, type, val) error(errmsg); end - if strcmp(types, 'uint64') && val < 0 + if strcmp(type, 'uint64') && val < 0 error('Property `%s` must be greater than zero.', name); end case 'char' - if ~ischar(val) + if ~ischar(val) && ~iscellstr(val) error(errmsg); end otherwise %class or ref to class @@ -28,6 +28,9 @@ function checkDtype(name, type, val) end for i=1:length(val) subval = val{i}; + if isempty(subval) + continue; + end if ~isa(subval, type) error(errmsg); end diff --git a/nwbRead.m b/nwbRead.m index 42aa1b22..50f73976 100644 --- a/nwbRead.m +++ b/nwbRead.m @@ -25,18 +25,8 @@ else ff = fullfile(pwd, filename); end -[fp, ~, ~] = fileparts(ff); %complete filepath - -for lref = linkRefs - lr = lref{1}; - if isempty(lr.filename) - lr.ref = nwb(lr.path); - else - % we assume the external reference is to a dataset. - if ~java.io.File(lr.filename).isAbsolute - lr.filename = fullfile(fp, lr.filename); - end - lr.ref = h5read(lr.filename, lr.path); - end -end +keyboard; +%process refs +%process links +[fp, ~, ~] = fileparts(ff); end \ No newline at end of file diff --git a/nwbfile.m b/nwbfile.m index fc2eae87..474f60b0 100644 --- a/nwbfile.m +++ b/nwbfile.m @@ -1,30 +1,46 @@ -classdef nwbfile < handle - % nwbfile Root object representing data read from an NWB file. - % - % Requires that core and extension NWB types have been generated - % and reside in a 'types' package on the matlab path. - % - % Example. Construct an object from scratch for export: - % nwb = nwbfile; - % nwb.epochs = types.untyped.Group; - % nwb.epochs.stim = types.Epoch; - % nwbExport(nwb, 'epoch.nwb'); - % - % See also NWBREAD, GENERATECORE, GENERATEEXTENSIONS - methods - function obj = nwbfile(varargin) - obj = obj@types.core.NWBFile(varargin{:}); +classdef nwbfile < types.core.NWBFile + % nwbfile Root object representing data read from an NWB file. + % + % Requires that core and extension NWB types have been generated + % and reside in a 'types' package on the matlab path. + % + % Example. Construct an object from scratch for export: + % nwb = nwbfile; + % nwb.epochs = types.untyped.Group; + % nwb.epochs.stim = types.Epoch; + % nwbExport(nwb, 'epoch.nwb'); + % + % See also NWBREAD, GENERATECORE, GENERATEEXTENSIONS + methods + function obj = nwbfile(varargin) + p = inputParser; + p.KeepUnmatched = true; + p.PartialMatching = false; + p.StructExpand = false; + addParameter(p, 'nwb_version', []); + addParameter(p, 'file_create_date', []); + parse(p, varargin{:}); + if any(strcmp('nwb_version', p.UsingDefaults)) + varargin = [{'nwb_version'} {'1.0.2'}]; + end + if any(strcmp('file_create_date', p.UsingDefaults)) + varargin = [{'file_create_date'} {''}]; + end + obj = obj@types.core.NWBFile(varargin{:}); + end + + function export(obj, filename) + if exist(filename, 'file') + warning('Overwriting %s', filename); + delete(filename); + end + fid = H5F.create(filename); + export@types.core.NWBFile(obj, fid); + H5F.close(fid); + + if isempty(obj.file_create_date) + h5write(filename, '/file_create_date', datestr(datetime, 30)); + end + end end - - function export(obj, filename) - validateattributes(filename, {'string', 'char'}, {'scalartext'}); - if exist(filename, 'file') - warning('Overwriting %s', filename); - delete(filename); - end - fid = H5F.create(filename); - export@types.NWBFile(obj, fid); - H5F.close(fid); - end - end end \ No newline at end of file