From 8204c4824713d85f0af618cb5afc24247046586b Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 11 Dec 2023 14:42:43 -0800 Subject: [PATCH 01/78] config --- src/hdmf/build/manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 03f2856b8..b50e9c93f 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -480,6 +480,9 @@ def load_namespaces(self, **kwargs): self.register_container_type(new_ns, dt, container_cls) return deps + def load_config(self, **kwargs): + pass + @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"}, {"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, From bb40b5acbc10d6d65d8bf859950f9da82de61b17 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Jan 2024 14:36:45 -0800 Subject: [PATCH 02/78] rough draft --- docs/gallery/example_config.yaml | 16 ++++++++++++++++ src/hdmf/__init__.py | 6 +++++- src/hdmf/build/manager.py | 4 +--- src/hdmf/common/__init__.py | 17 +++++++++++++++++ src/hdmf/container.py | 23 +++++++++++++++++++++++ src/hdmf/term_set.py | 25 +++++++++++++++++++++++++ tests/unit/common/test_table.py | 4 ++++ tests/unit/test_container.py | 4 ++++ 8 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 docs/gallery/example_config.yaml diff --git a/docs/gallery/example_config.yaml b/docs/gallery/example_config.yaml new file mode 100644 index 000000000..c6ace1765 --- /dev/null +++ b/docs/gallery/example_config.yaml @@ -0,0 +1,16 @@ +- data_type: VectorData + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... +# - data_type: #Container2 +# namespace: +# namespace_version: +# fields: +# - name: +# doc: +# termset_path: +# - name: +# doc: +# termset_path: diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 2699a28af..da1ad3b77 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -3,9 +3,13 @@ from .container import Container, Data, DataRegion, HERDManager from .region import ListSlicer from .utils import docval, getargs -from .term_set import TermSet, TermSetWrapper +from .term_set import TermSet, TermSetWrapper, TermSetConfigurator +# a global TermSetConfigurator +global TS_CONFIG +TS_CONFIG = TermSetConfigurator() + @docval( {"name": "dataset", "type": None, "doc": "the HDF5 dataset to slice"}, {"name": "region", "type": None, "doc": "the region reference to use to slice"}, diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index b50e9c93f..029c17efd 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -410,6 +410,7 @@ def __init__(self, **kwargs): self.__data_types = dict() self.__default_mapper_cls = mapper_cls self.__class_generator = ClassGenerator() + self.__load_termset_config = True self.register_generator(CustomClassGenerator) self.register_generator(MCIClassGenerator) @@ -480,9 +481,6 @@ def load_namespaces(self, **kwargs): self.register_container_type(new_ns, dt, container_cls) return deps - def load_config(self, **kwargs): - pass - @docval({"name": "namespace", "type": str, "doc": "the namespace containing the data_type"}, {"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index e0782effe..f18eb7497 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -107,6 +107,23 @@ def available_namespaces(): return __TYPE_MAP.namespace_catalog.namespaces +@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', + 'default': '/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'}) #update path +def load_termset_config(config_path: str): + """ + Load the configuration file for validation on the fields defined for the objects within the file. + By default, the curated configuration file is used, but can take in a custom file. + """ + return __TS_CONFIG.load_termset_config(config_path) + + +def unload_termset_config(): + """ + Remove validation according to termset configuration file. + """ + return __TS_CONFIG.unload_termset_config() + + # a function to get the container class for a give type @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 229e20083..4d8474700 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -5,6 +5,7 @@ from typing import Type from uuid import uuid4 from warnings import warn +import yaml import h5py import numpy as np @@ -13,6 +14,8 @@ from .data_utils import DataIO, append_data, extend_data from .utils import docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict +from .term_set import TermSet, TermSetWrapper + def _set_exp(cls): """Set a class as being experimental""" @@ -232,6 +235,24 @@ def __init__(self, **kwargs): self.__read_io = None self.__obj = None + @docval({'name': 'fields', 'type': dict, 'doc': 'The fields/parameters/attibutes for the object.'}) + def init_validation(self, fields): + # load termset configuartion file from global Config + from . import TS_CONFIG #update path + # Before calling super().__init__() and before setting fields, check for config file for + # validation via TermSetWrapper. + with open(TS_CONFIG.path, 'r') as config: + termset_config = yaml.safe_load(config) + object_name = self.__class__.__name__ + + for obj_config in termset_config: + if obj_config['data_type'] == object_name: + for attr in obj_config['fields']: + if attr in fields: # make sure any custom fields are not handled (i.e., make an extension) + termset_path = obj_config['fields'][attr] + termset = TermSet(term_schema_path=termset_path) + fields[attr] = TermSetWrapper(value=fields[attr], termset=termset) + @property def read_io(self): """ @@ -785,6 +806,8 @@ class Data(AbstractContainer): @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'}) def __init__(self, **kwargs): + self.init_validation(fields=kwargs) + breakpoint() data = popargs('data', kwargs) super().__init__(**kwargs) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index f7169bdfd..4a196d5cc 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -304,3 +304,28 @@ def extend(self, arg): else: msg = ('"%s" is not in the term set.' % ', '.join([str(item) for item in bad_data])) raise ValueError(msg) + +class TermSetConfigurator: + """ + + """ + def __init__(self): + self.path = '/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml' + + # @property + # def config_path(self): + # return self.__config_path + + @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}) + def load_termset_config(config_path: str): + """ + Load the configuration file for validation on the fields defined for the objects within the file. + By default, the curated configuration file is used, but can take in a custom file. + """ + self.path = config_path + + def unload_termset_config(): + """ + Remove validation according to termset configuration file. + """ + self.path = None diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 7246a8ba8..9112c4e5f 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -36,6 +36,10 @@ except ImportError: LINKML_INSTALLED = False +class TestVDConfig(TestCase): + def test_init_config(self): + vd = VectorData(name='data', description='',data=['Homo sapiens']) + class TestDynamicTable(TestCase): diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index b5a2d87e8..ec74d5cb0 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -32,6 +32,10 @@ def test_link_and_get_resources(self): er_get = em.get_linked_resources() self.assertEqual(er, er_get) +class TestContainerConfig(TestCase): + def test_init_config(self): + obj = Container('obj1') + class TestContainer(TestCase): From b9b52f2d2f17c100faa39c914baa25b91b438080 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Jan 2024 14:45:42 -0800 Subject: [PATCH 03/78] move --- src/hdmf/__init__.py | 16 ++++++++++++++++ src/hdmf/common/__init__.py | 17 ----------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index da1ad3b77..61fa9f364 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -10,6 +10,22 @@ global TS_CONFIG TS_CONFIG = TermSetConfigurator() +@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', + 'default': '/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'}) #update path +def load_termset_config(config_path: str): + """ + Load the configuration file for validation on the fields defined for the objects within the file. + By default, the curated configuration file is used, but can take in a custom file. + """ + return __TS_CONFIG.load_termset_config(config_path) + + +def unload_termset_config(): + """ + Remove validation. + """ + return __TS_CONFIG.unload_termset_config() + @docval( {"name": "dataset", "type": None, "doc": "the HDF5 dataset to slice"}, {"name": "region", "type": None, "doc": "the region reference to use to slice"}, diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index f18eb7497..e0782effe 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -107,23 +107,6 @@ def available_namespaces(): return __TYPE_MAP.namespace_catalog.namespaces -@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', - 'default': '/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'}) #update path -def load_termset_config(config_path: str): - """ - Load the configuration file for validation on the fields defined for the objects within the file. - By default, the curated configuration file is used, but can take in a custom file. - """ - return __TS_CONFIG.load_termset_config(config_path) - - -def unload_termset_config(): - """ - Remove validation according to termset configuration file. - """ - return __TS_CONFIG.unload_termset_config() - - # a function to get the container class for a give type @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, From 5fa60cd1359ffb048008353673bcfdbe2223d07b Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 10 Jan 2024 21:35:37 -0800 Subject: [PATCH 04/78] testing --- src/hdmf/__init__.py | 4 ++-- src/hdmf/common/__init__.py | 1 + src/hdmf/container.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 61fa9f364..2e193f4cb 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -17,14 +17,14 @@ def load_termset_config(config_path: str): Load the configuration file for validation on the fields defined for the objects within the file. By default, the curated configuration file is used, but can take in a custom file. """ - return __TS_CONFIG.load_termset_config(config_path) + return TS_CONFIG.load_termset_config(config_path) def unload_termset_config(): """ Remove validation. """ - return __TS_CONFIG.unload_termset_config() + return TS_CONFIG.unload_termset_config() @docval( {"name": "dataset", "type": None, "doc": "the HDF5 dataset to slice"}, diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index e0782effe..7048f0b1b 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -233,3 +233,4 @@ def get_hdf5io(**kwargs): HERD = get_class('HERD', EXP_NAMESPACE) SimpleMultiContainer = get_class('SimpleMultiContainer', CORE_NAMESPACE) AlignedDynamicTable = get_class('AlignedDynamicTable', CORE_NAMESPACE) +breakpoint() diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 4d8474700..ead820ee2 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -16,7 +16,6 @@ from .term_set import TermSet, TermSetWrapper - def _set_exp(cls): """Set a class as being experimental""" cls._experimental = True From 988a442e77a6aa44fae3ab507b2062ce770cdbc0 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Fri, 12 Jan 2024 16:58:25 -0800 Subject: [PATCH 05/78] check --- src/hdmf/__init__.py | 1 - src/hdmf/container.py | 22 ++++++++++------------ src/hdmf/term_set.py | 10 ++++++++-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 2e193f4cb..15b1ba651 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -7,7 +7,6 @@ # a global TermSetConfigurator -global TS_CONFIG TS_CONFIG = TermSetConfigurator() @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', diff --git a/src/hdmf/container.py b/src/hdmf/container.py index ead820ee2..7fd7de23e 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -240,17 +240,16 @@ def init_validation(self, fields): from . import TS_CONFIG #update path # Before calling super().__init__() and before setting fields, check for config file for # validation via TermSetWrapper. - with open(TS_CONFIG.path, 'r') as config: - termset_config = yaml.safe_load(config) - object_name = self.__class__.__name__ - - for obj_config in termset_config: - if obj_config['data_type'] == object_name: - for attr in obj_config['fields']: - if attr in fields: # make sure any custom fields are not handled (i.e., make an extension) - termset_path = obj_config['fields'][attr] - termset = TermSet(term_schema_path=termset_path) - fields[attr] = TermSetWrapper(value=fields[attr], termset=termset) + termset_config = TS_CONFIG.load_termset_config() + object_name = self.__class__.__name__ + + for obj_config in termset_config: + if obj_config['data_type'] == object_name: + for attr in obj_config['fields']: + if attr in fields: # make sure any custom fields are not handled (i.e., make an extension) + termset_path = obj_config['fields'][attr] + termset = TermSet(term_schema_path=termset_path) + fields[attr] = TermSetWrapper(value=fields[attr], termset=termset) @property def read_io(self): @@ -801,7 +800,6 @@ class Data(AbstractContainer): """ A class for representing dataset containers """ - @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'}) def __init__(self, **kwargs): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 4a196d5cc..f8cb1eb08 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -316,13 +316,19 @@ def __init__(self): # def config_path(self): # return self.__config_path - @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}) + @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', + 'default': None}) def load_termset_config(config_path: str): """ Load the configuration file for validation on the fields defined for the objects within the file. By default, the curated configuration file is used, but can take in a custom file. """ - self.path = config_path + if config_path not None: + self.path = config_path + + with open(self.path, 'r') as config: + termset_config = yaml.safe_load(config) + return termset_config def unload_termset_config(): """ From 0b83e581599be37a57ade66e8c9d8a856e93ff61 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 17 Jan 2024 17:03:12 -0800 Subject: [PATCH 06/78] new way of thinking draft --- docs/gallery/example_config.yaml | 28 ++++++++----------- src/hdmf/__init__.py | 24 +++++++++++++--- src/hdmf/common/__init__.py | 1 - src/hdmf/common/hdmf-common-schema | 2 +- src/hdmf/container.py | 44 +++++++++++++++++++++++------- src/hdmf/term_set.py | 43 +++++++++++++++++++++-------- 6 files changed, 98 insertions(+), 44 deletions(-) diff --git a/docs/gallery/example_config.yaml b/docs/gallery/example_config.yaml index c6ace1765..a9277c0a7 100644 --- a/docs/gallery/example_config.yaml +++ b/docs/gallery/example_config.yaml @@ -1,16 +1,12 @@ -- data_type: VectorData - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... -# - data_type: #Container2 -# namespace: -# namespace_version: -# fields: -# - name: -# doc: -# termset_path: -# - name: -# doc: -# termset_path: +VectorData: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... +DataType2: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 15b1ba651..a40084da2 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -9,14 +9,30 @@ # a global TermSetConfigurator TS_CONFIG = TermSetConfigurator() +def get_termset_config(): + return TS_CONFIG.config + +def get_config_types(): + return TS_CONFIG.get_data_types() + @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', - 'default': '/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'}) #update path + 'default': None}) def load_termset_config(config_path: str): """ - Load the configuration file for validation on the fields defined for the objects within the file. - By default, the curated configuration file is used, but can take in a custom file. + If a user does not provide a config_path, then this method will unload any present configuration + and load the default curated configuration. + + If a user provides a config_path, then this method will: + - Search the current configuation for data_types that are already present. These data_types will be + replaced with the new configuration. + - If the data_type is not present, then they will be loaded alongside the default curated configuration. """ - return TS_CONFIG.load_termset_config(config_path) + if config_path is None: + TS_CONFIG.path = "/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml" + TS_CONFIG.load_termset_config() + + + def unload_termset_config(): diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 7048f0b1b..e0782effe 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -233,4 +233,3 @@ def get_hdf5io(**kwargs): HERD = get_class('HERD', EXP_NAMESPACE) SimpleMultiContainer = get_class('SimpleMultiContainer', CORE_NAMESPACE) AlignedDynamicTable = get_class('AlignedDynamicTable', CORE_NAMESPACE) -breakpoint() diff --git a/src/hdmf/common/hdmf-common-schema b/src/hdmf/common/hdmf-common-schema index 5b4cbb31d..4d2ddd638 160000 --- a/src/hdmf/common/hdmf-common-schema +++ b/src/hdmf/common/hdmf-common-schema @@ -1 +1 @@ -Subproject commit 5b4cbb31dbafcff51ca70bf218f464b186568151 +Subproject commit 4d2ddd6387c4e36f21f41964fe8873c083680b15 diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 7fd7de23e..e04aa12ae 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -238,18 +238,42 @@ def __init__(self, **kwargs): def init_validation(self, fields): # load termset configuartion file from global Config from . import TS_CONFIG #update path - # Before calling super().__init__() and before setting fields, check for config file for - # validation via TermSetWrapper. - termset_config = TS_CONFIG.load_termset_config() + + termset_config = TS_CONFIG.config + breakpoint() + if termset_config is None: + msg = 'TermSet Configuration is not loaded.' + raise ValueError(msg) object_name = self.__class__.__name__ - for obj_config in termset_config: - if obj_config['data_type'] == object_name: - for attr in obj_config['fields']: - if attr in fields: # make sure any custom fields are not handled (i.e., make an extension) - termset_path = obj_config['fields'][attr] - termset = TermSet(term_schema_path=termset_path) - fields[attr] = TermSetWrapper(value=fields[attr], termset=termset) + # Check that the object data_type is in the loaded namespace + from .common import get_type_map + container_types_dict = get_type_map().container_types + object_exists = False + for namespace in container_types_dict: + if object_name in container_types_dict[namespace]: + object_exists = True + else: + continue + + if not object_exists: + msg = "%s is not in the loaded Namespace(s)." % object_name + raise ValueError(msg) + + # Wrap supported fields with TermSetWrapper + obj_wrapped = False + if object_name in termset_config: + obj_wrapped = True + for attr in termset_config[object_name]['fields']: + if attr in fields: # make sure any custom fields are not handled (i.e., make an extension) + termset_path = termset_config[object_name]['fields'][attr] + termset = TermSet(term_schema_path=termset_path) + fields[attr] = TermSetWrapper(value=fields[attr], termset=termset) + + # Even if the data_type is in the namespace, it might not be in the configuration. + if not obj_wrapped: + msg = "%s is not in the loaded TermSet Configuration." % object_name + raise ValueError(msg) @property def read_io(self): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index f8cb1eb08..a26055b0f 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,6 +5,7 @@ import warnings import numpy as np from .data_utils import append_data, extend_data +import yaml class TermSet: @@ -310,28 +311,46 @@ class TermSetConfigurator: """ def __init__(self): - self.path = '/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml' + self.path = ['/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'] + self.config = None + self.load_termset_config() - # @property - # def config_path(self): - # return self.__config_path + def get_data_types(self): + """ + Return list of data_types within current configuration file. + """ + data_types = [] + for data_type_dict in self.config: + data_types.append(data_type_dict['data_type']) @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', 'default': None}) - def load_termset_config(config_path: str): + def load_termset_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. - By default, the curated configuration file is used, but can take in a custom file. """ - if config_path not None: - self.path = config_path + # Set self.config for __init__ + if self.config is None: + with open(self.path[0], 'r') as config: + termset_config = yaml.safe_load(config) + self.config = termset_config + + # # Check data_types within new config to see if they already exist in the current config + # with open(config_path, 'r') as config: + # termset_config = yaml.safe_load(config) + # for data_type_dict in termset_config: + # if data_type_dict['data_type'] in self.get_data_types(): + # pass + # + # + # # append path to new config to self.path + # if config_path is not None: + # self.path.append(config_path) - with open(self.path, 'r') as config: - termset_config = yaml.safe_load(config) - return termset_config def unload_termset_config(): """ Remove validation according to termset configuration file. """ - self.path = None + self.path = ['/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'] + self.config = None From fa9e0f025a9d3e56289e1f7927da36d4d6c44ea2 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 17 Jan 2024 17:10:04 -0800 Subject: [PATCH 07/78] support multiple config files --- src/hdmf/__init__.py | 11 +++-------- src/hdmf/term_set.py | 32 ++++++++++++-------------------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index a40084da2..482297986 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -12,9 +12,6 @@ def get_termset_config(): return TS_CONFIG.config -def get_config_types(): - return TS_CONFIG.get_data_types() - @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', 'default': None}) def load_termset_config(config_path: str): @@ -28,12 +25,10 @@ def load_termset_config(config_path: str): - If the data_type is not present, then they will be loaded alongside the default curated configuration. """ if config_path is None: - TS_CONFIG.path = "/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml" + TS_CONFIG.unload_termset_config() TS_CONFIG.load_termset_config() - - - - + else: + TS_CONFIG.load_termset_config(config_path) def unload_termset_config(): """ diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index a26055b0f..f4d52bff6 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -315,14 +315,6 @@ def __init__(self): self.config = None self.load_termset_config() - def get_data_types(self): - """ - Return list of data_types within current configuration file. - """ - data_types = [] - for data_type_dict in self.config: - data_types.append(data_type_dict['data_type']) - @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', 'default': None}) def load_termset_config(self,config_path): @@ -334,18 +326,18 @@ def load_termset_config(self,config_path): with open(self.path[0], 'r') as config: termset_config = yaml.safe_load(config) self.config = termset_config - - # # Check data_types within new config to see if they already exist in the current config - # with open(config_path, 'r') as config: - # termset_config = yaml.safe_load(config) - # for data_type_dict in termset_config: - # if data_type_dict['data_type'] in self.get_data_types(): - # pass - # - # - # # append path to new config to self.path - # if config_path is not None: - # self.path.append(config_path) + else: + # Check data_types within new config to see if they already exist in the current config + with open(config_path, 'r') as config: + termset_config = yaml.safe_load(config) + for data_type in termset_config: + if data_type in self.config: + self.config[data_type] = termset_config[data_type] + termset_config.pop(data_type) + self.config.update(termset_config) + + # append path to new config to self.path + self.path.append(config_path) def unload_termset_config(): From 66d0752eec749d81983ec8530c8b85d505778a69 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 18 Jan 2024 13:49:36 -0800 Subject: [PATCH 08/78] testing --- src/hdmf/__init__.py | 2 +- src/hdmf/container.py | 38 ++++++++++----- src/hdmf/hdmf_config.yaml | 12 +++++ src/hdmf/term_set.py | 9 ++-- tests/unit/common/test_table.py | 25 ++++++++++ tests/unit/test_config.yaml | 12 +++++ tests/unit/test_extension_config.yaml | 12 +++++ tests/unit/test_termset_config.py | 68 +++++++++++++++++++++++++++ 8 files changed, 162 insertions(+), 16 deletions(-) create mode 100644 src/hdmf/hdmf_config.yaml create mode 100644 tests/unit/test_config.yaml create mode 100644 tests/unit/test_extension_config.yaml create mode 100644 tests/unit/test_termset_config.py diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 482297986..a78bb1fac 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -7,7 +7,7 @@ # a global TermSetConfigurator -TS_CONFIG = TermSetConfigurator() +TS_CONFIG = TermSetConfigurator(path='src/hdmf/hdmf_config.yaml') def get_termset_config(): return TS_CONFIG.config diff --git a/src/hdmf/container.py b/src/hdmf/container.py index e04aa12ae..5dfc7e481 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -234,13 +234,12 @@ def __init__(self, **kwargs): self.__read_io = None self.__obj = None - @docval({'name': 'fields', 'type': dict, 'doc': 'The fields/parameters/attibutes for the object.'}) - def init_validation(self, fields): + @docval({'name': 'constructor_args', 'type': dict, 'doc': 'The fields/parameters/attibutes for the object.'}) + def init_validation(self, constructor_args): # load termset configuartion file from global Config from . import TS_CONFIG #update path termset_config = TS_CONFIG.config - breakpoint() if termset_config is None: msg = 'TermSet Configuration is not loaded.' raise ValueError(msg) @@ -248,7 +247,9 @@ def init_validation(self, fields): # Check that the object data_type is in the loaded namespace from .common import get_type_map - container_types_dict = get_type_map().container_types + tm = get_type_map() + + container_types_dict = tm.container_types object_exists = False for namespace in container_types_dict: if object_name in container_types_dict[namespace]: @@ -258,22 +259,37 @@ def init_validation(self, fields): if not object_exists: msg = "%s is not in the loaded Namespace(s)." % object_name - raise ValueError(msg) + raise warn(msg) # Wrap supported fields with TermSetWrapper obj_wrapped = False if object_name in termset_config: obj_wrapped = True for attr in termset_config[object_name]['fields']: - if attr in fields: # make sure any custom fields are not handled (i.e., make an extension) + obj_mapper = tm.get_map(self) + # get the spec according to attr name in schema + # Note: this is the name for the field in the config + spec = obj_mapper.get_attr_spec(attr) + + # In the case of dealing with datasets directly or not defined in the spec. + # (Data/VectorData/DynamicTable/etc) + if spec is None: + # constr_name= attr + msg = "Spec not found for %s" % attr + raise warn(msg) + # From the spec, get the corresponding constructor name + else: + constr_name = obj_mapper.get_const_arg(spec) + + if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) termset_path = termset_config[object_name]['fields'][attr] termset = TermSet(term_schema_path=termset_path) - fields[attr] = TermSetWrapper(value=fields[attr], termset=termset) + constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) # Even if the data_type is in the namespace, it might not be in the configuration. - if not obj_wrapped: + if object_exists and not obj_wrapped: msg = "%s is not in the loaded TermSet Configuration." % object_name - raise ValueError(msg) + raise warn(msg) @property def read_io(self): @@ -827,8 +843,8 @@ class Data(AbstractContainer): @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'}) def __init__(self, **kwargs): - self.init_validation(fields=kwargs) - breakpoint() + self.init_validation(constructor_args=kwargs) + # breakpoint() data = popargs('data', kwargs) super().__init__(**kwargs) diff --git a/src/hdmf/hdmf_config.yaml b/src/hdmf/hdmf_config.yaml new file mode 100644 index 000000000..a9277c0a7 --- /dev/null +++ b/src/hdmf/hdmf_config.yaml @@ -0,0 +1,12 @@ +VectorData: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... +DataType2: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index f4d52bff6..af9c34295 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -310,8 +310,10 @@ class TermSetConfigurator: """ """ - def __init__(self): - self.path = ['/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'] + @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuartion file.', + 'default': None}) + def __init__(self, **kwargs): + self.path = [kwargs['path']] self.config = None self.load_termset_config() @@ -339,10 +341,9 @@ def load_termset_config(self,config_path): # append path to new config to self.path self.path.append(config_path) - def unload_termset_config(): """ Remove validation according to termset configuration file. """ - self.path = ['/Users/mavaylon/Research/NWB/hdmf2/hdmf/docs/gallery/example_config.yaml'] + self.path = ['src/hdmf/hdmf_config.yaml'] self.config = None diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 9112c4e5f..eb37687f1 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -39,6 +39,31 @@ class TestVDConfig(TestCase): def test_init_config(self): vd = VectorData(name='data', description='',data=['Homo sapiens']) + tb = DynamicTable(name="with_table_columns", description='a test table', columns=[vd]) + from hdmf.common import get_type_map + htm = get_type_map() + om = htm.get_map(tb) + from datetime import datetime + from uuid import uuid4 + + # import numpy as np + # from dateutil.tz import tzlocal + # + # from pynwb import NWBHDF5IO, NWBFile + # from pynwb.behavior import SpatialSeries + # from pynwb import get_type_map + # tm = get_type_map() + # timestamps = np.linspace(0, 50) / 200 + # position_data = np.array([np.linspace(0, 10, 50), np.linspace(0, 8, 50)]).T + # + # position_spatial_series = SpatialSeries( + # name="SpatialSeries", + # description="Position (x, y) in an open field.", + # data=position_data, + # timestamps=timestamps, + # reference_frame="(0,0) is bottom left corner", + # ) + breakpoint() class TestDynamicTable(TestCase): diff --git a/tests/unit/test_config.yaml b/tests/unit/test_config.yaml new file mode 100644 index 000000000..a9277c0a7 --- /dev/null +++ b/tests/unit/test_config.yaml @@ -0,0 +1,12 @@ +VectorData: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... +DataType2: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... diff --git a/tests/unit/test_extension_config.yaml b/tests/unit/test_extension_config.yaml new file mode 100644 index 000000000..a9277c0a7 --- /dev/null +++ b/tests/unit/test_extension_config.yaml @@ -0,0 +1,12 @@ +VectorData: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... +DataType2: + namespace: + namespace_version: + fields: + data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml + field2: ... diff --git a/tests/unit/test_termset_config.py b/tests/unit/test_termset_config.py new file mode 100644 index 000000000..f37a90e37 --- /dev/null +++ b/tests/unit/test_termset_config.py @@ -0,0 +1,68 @@ +from hdmf.testing import TestCase +from hdmf import get_termset_config, load_termset_config, unload_termset_config +from hdmf.term_set import TermSetConfigurator + +class TestConfig(TestCase): + self.test_config = {} + self.test_merged_extension_config = {} + self.hdmf_config = {} + + def test_construct_config(self): + # add asserts for self.path and self.config + test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') + self.assertEqual(test_config.path, ['tests/unit/test_config.yaml']) + # self.assertEqual(test_config.config, None) + + def test_load_termset_config(self): + test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') + test_config.load_termset_config(path='tests/unit/test_extension_config.yaml') + self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) + # self.assertEqual(config.config, None) + + def test_unload_termset_config(self): + test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') + test_config.unload_termset_config() + self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) + self.assertEqual(config.config, None) + + def test_get_termset_config(self): + config = get_termset_config() + self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) + # self.assertEqual(config.config, None) + + def test_unload_global_config(self): + config = get_termset_config() + unload_termset_config() + self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) + self.assertEqual(config.config, None) + + def test_load_global_config_reset(self): + load_termset_config() + self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) + # self.assertEqual(config.config, None) + + def test_load_global_config_extension_config(self): + load_termset_config() + self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) + # self.assertEqual(config.config, None) + + def test_data(self): + pass + + def test_dataset_not_in_spec(self): + pass + + def test_attribute_not_in_spec(self): + pass + + def test_attriute_in_spec(self): + pass + + def test_dataset_in_spec(self): + pass + + def test_data_type_not_in_namespace(self): + pass + + def test_warn_not_wrapped(self): + pass From c77e06da7d7e8d90ec05a86928d714bac3a3060e Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 22 Jan 2024 12:35:39 -0800 Subject: [PATCH 09/78] placeholder' --- src/hdmf/__init__.py | 10 +-- src/hdmf/container.py | 102 ++++++++++++++++-------------- src/hdmf/term_set.py | 7 +- tests/unit/test_config.yaml | 2 +- tests/unit/test_config2.yaml | 23 +++++++ tests/unit/test_termset_config.py | 7 +- 6 files changed, 88 insertions(+), 63 deletions(-) create mode 100644 tests/unit/test_config2.yaml diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index a78bb1fac..730053e20 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -5,12 +5,14 @@ from .utils import docval, getargs from .term_set import TermSet, TermSetWrapper, TermSetConfigurator +import os -# a global TermSetConfigurator -TS_CONFIG = TermSetConfigurator(path='src/hdmf/hdmf_config.yaml') -def get_termset_config(): - return TS_CONFIG.config +# a global TermSetConfigurator +CUR_DIR = os.path.dirname(os.path.realpath(__file__)) +path_to_config = os.path.join(CUR_DIR, 'hdmf_config.yaml') +TS_CONFIG = TermSetConfigurator(path=path_to_config) +TS_CONFIG.unload_termset_config() @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', 'default': None}) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 5dfc7e481..a93a4624b 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -234,62 +234,68 @@ def __init__(self, **kwargs): self.__read_io = None self.__obj = None - @docval({'name': 'constructor_args', 'type': dict, 'doc': 'The fields/parameters/attibutes for the object.'}) - def init_validation(self, constructor_args): - # load termset configuartion file from global Config + def get_config(self): from . import TS_CONFIG #update path + return TS_CONFIG - termset_config = TS_CONFIG.config - if termset_config is None: - msg = 'TermSet Configuration is not loaded.' - raise ValueError(msg) - object_name = self.__class__.__name__ - - # Check that the object data_type is in the loaded namespace + def get_type_map(self): from .common import get_type_map tm = get_type_map() + return tm - container_types_dict = tm.container_types - object_exists = False - for namespace in container_types_dict: - if object_name in container_types_dict[namespace]: - object_exists = True - else: - continue - - if not object_exists: - msg = "%s is not in the loaded Namespace(s)." % object_name - raise warn(msg) - - # Wrap supported fields with TermSetWrapper - obj_wrapped = False - if object_name in termset_config: - obj_wrapped = True - for attr in termset_config[object_name]['fields']: - obj_mapper = tm.get_map(self) - # get the spec according to attr name in schema - # Note: this is the name for the field in the config - spec = obj_mapper.get_attr_spec(attr) - - # In the case of dealing with datasets directly or not defined in the spec. - # (Data/VectorData/DynamicTable/etc) - if spec is None: - # constr_name= attr - msg = "Spec not found for %s" % attr - raise warn(msg) - # From the spec, get the corresponding constructor name + @docval({'name': 'constructor_args', 'type': dict, + 'doc': 'The fields/parameters/attibutes for the object.'}) + def init_validation(self, constructor_args): + # load termset configuartion file from global Config + config = self.get_config() + termset_config = config.config + if termset_config is not None: + object_name = self.__class__.__name__ + + # Check that the object data_type is in the loaded namespace + tm = self.get_type_map() + + container_types_dict = tm.container_types + object_exists = False + for namespace in container_types_dict: + if object_name in container_types_dict[namespace]: + object_exists = True else: - constr_name = obj_mapper.get_const_arg(spec) + continue + + if not object_exists: + msg = "%s is not in the loaded Namespace(s)." % object_name + warn(msg) + + # Wrap supported fields with TermSetWrapper + obj_wrapped = False + if object_name in termset_config: + obj_wrapped = True + for attr in termset_config[object_name]['fields']: + obj_mapper = tm.get_map(self) + # get the spec according to attr name in schema + # Note: this is the name for the field in the config + spec = obj_mapper.get_attr_spec(attr) + + # In the case of dealing with datasets directly or not defined in the spec. + # (Data/VectorData/DynamicTable/etc) + if spec is None: + # constr_name= attr + msg = "Spec not found for %s" % attr + warn(msg) + # From the spec, get the corresponding constructor name + else: + constr_name = obj_mapper.get_const_arg(spec) - if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) - termset_path = termset_config[object_name]['fields'][attr] - termset = TermSet(term_schema_path=termset_path) - constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) + if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) + termset_path = termset_config[object_name]['fields'][attr] + termset = TermSet(term_schema_path=termset_path) + constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) - # Even if the data_type is in the namespace, it might not be in the configuration. - if object_exists and not obj_wrapped: - msg = "%s is not in the loaded TermSet Configuration." % object_name - raise warn(msg) + # Even if the data_type is in the namespace, it might not be in the configuration. + if object_exists and not obj_wrapped: + msg = "%s is not in the loaded TermSet Configuration." % object_name + warn(msg) @property def read_io(self): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index af9c34295..f270a5122 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -310,8 +310,7 @@ class TermSetConfigurator: """ """ - @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuartion file.', - 'default': None}) + @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuartion file.'}) def __init__(self, **kwargs): self.path = [kwargs['path']] self.config = None @@ -341,9 +340,9 @@ def load_termset_config(self,config_path): # append path to new config to self.path self.path.append(config_path) - def unload_termset_config(): + def unload_termset_config(self): """ Remove validation according to termset configuration file. """ - self.path = ['src/hdmf/hdmf_config.yaml'] + self.path = [] self.config = None diff --git a/tests/unit/test_config.yaml b/tests/unit/test_config.yaml index a9277c0a7..dbc5f48c2 100644 --- a/tests/unit/test_config.yaml +++ b/tests/unit/test_config.yaml @@ -4,7 +4,7 @@ VectorData: fields: data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml field2: ... -DataType2: +DynamicTable: namespace: namespace_version: fields: diff --git a/tests/unit/test_config2.yaml b/tests/unit/test_config2.yaml new file mode 100644 index 000000000..0b6abed93 --- /dev/null +++ b/tests/unit/test_config2.yaml @@ -0,0 +1,23 @@ +namespaces: + core: + version: + data_types: + VectorData: + data: '...' + field2: '...' + experimental: + version: + +# +# VectorData: +# namespace: +# namespace_version: +# fields: +# data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml +# field2: ... +# DynamicTable: +# namespace: +# namespace_version: +# fields: +# data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml +# field2: ... diff --git a/tests/unit/test_termset_config.py b/tests/unit/test_termset_config.py index f37a90e37..269207068 100644 --- a/tests/unit/test_termset_config.py +++ b/tests/unit/test_termset_config.py @@ -1,5 +1,5 @@ from hdmf.testing import TestCase -from hdmf import get_termset_config, load_termset_config, unload_termset_config +from hdmf import load_termset_config, unload_termset_config from hdmf.term_set import TermSetConfigurator class TestConfig(TestCase): @@ -25,11 +25,6 @@ def test_unload_termset_config(self): self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) self.assertEqual(config.config, None) - def test_get_termset_config(self): - config = get_termset_config() - self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) - # self.assertEqual(config.config, None) - def test_unload_global_config(self): config = get_termset_config() unload_termset_config() From 8353092660e7ba586c9ed672f5ab0b2df889b84c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 20:35:50 +0000 Subject: [PATCH 10/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/test_config2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_config2.yaml b/tests/unit/test_config2.yaml index 0b6abed93..7b4a2b9ed 100644 --- a/tests/unit/test_config2.yaml +++ b/tests/unit/test_config2.yaml @@ -8,7 +8,7 @@ namespaces: experimental: version: -# +# # VectorData: # namespace: # namespace_version: From ed140c97add47578b7634aec6fd4a943caddbacd Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 25 Jan 2024 13:38:18 -0800 Subject: [PATCH 11/78] update --- src/hdmf/__init__.py | 31 ----------- src/hdmf/build/manager.py | 17 ++++-- src/hdmf/common/__init__.py | 12 +++++ src/hdmf/container.py | 91 +++++++++++++-------------------- src/hdmf/term_set.py | 32 ++++++------ tests/unit/common/test_table.py | 14 ++--- tests/unit/test_config2.yaml | 26 +++------- tests/unit/test_container.py | 12 +++-- 8 files changed, 100 insertions(+), 135 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index 730053e20..d67d685fb 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -3,41 +3,10 @@ from .container import Container, Data, DataRegion, HERDManager from .region import ListSlicer from .utils import docval, getargs -from .term_set import TermSet, TermSetWrapper, TermSetConfigurator import os -# a global TermSetConfigurator -CUR_DIR = os.path.dirname(os.path.realpath(__file__)) -path_to_config = os.path.join(CUR_DIR, 'hdmf_config.yaml') -TS_CONFIG = TermSetConfigurator(path=path_to_config) -TS_CONFIG.unload_termset_config() - -@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', - 'default': None}) -def load_termset_config(config_path: str): - """ - If a user does not provide a config_path, then this method will unload any present configuration - and load the default curated configuration. - - If a user provides a config_path, then this method will: - - Search the current configuation for data_types that are already present. These data_types will be - replaced with the new configuration. - - If the data_type is not present, then they will be loaded alongside the default curated configuration. - """ - if config_path is None: - TS_CONFIG.unload_termset_config() - TS_CONFIG.load_termset_config() - else: - TS_CONFIG.load_termset_config(config_path) - -def unload_termset_config(): - """ - Remove validation. - """ - return TS_CONFIG.unload_termset_config() - @docval( {"name": "dataset", "type": None, "doc": "the HDF5 dataset to slice"}, {"name": "region", "type": None, "doc": "the region reference to use to slice"}, diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 029c17efd..d17946baa 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -5,6 +5,7 @@ from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator from ..container import AbstractContainer, Container, Data +from ..term_set import TermSetConfigurator from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog from ..spec.spec import BaseStorageSpec from ..utils import docval, getargs, ExtenderMeta, get_docval @@ -391,13 +392,15 @@ def data_type(self): class TypeMap: - ''' A class to maintain the map between ObjectMappers and AbstractContainer classes - ''' + """ + A class to maintain the map between ObjectMappers and AbstractContainer classes + """ @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None}, - {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}) + {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}, + {'name': 'config_path', 'type': str, 'doc': 'The path to the TermSet config yaml.', 'default': None}) def __init__(self, **kwargs): - namespaces, mapper_cls = getargs('namespaces', 'mapper_cls', kwargs) + namespaces, mapper_cls, config_path = getargs('namespaces', 'mapper_cls', 'config_path', kwargs) if namespaces is None: namespaces = NamespaceCatalog() if mapper_cls is None: @@ -410,7 +413,7 @@ def __init__(self, **kwargs): self.__data_types = dict() self.__default_mapper_cls = mapper_cls self.__class_generator = ClassGenerator() - self.__load_termset_config = True + self.ts_config = TermSetConfigurator(path=config_path) self.register_generator(CustomClassGenerator) self.register_generator(MCIClassGenerator) @@ -418,6 +421,10 @@ def __init__(self, **kwargs): def namespace_catalog(self): return self.__ns_catalog + @property + def data_types(self): + return self.__data_types + @property def container_types(self): return self.__container_types diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index e0782effe..cd3b98120 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -20,6 +20,14 @@ # a global type map global __TYPE_MAP +def load_termset_config(config_path: str): + """ + This method will: + - Search the current configuration for data_types that are already present. These data_types will be + replaced with the new configuration. + - If the data_type is not present, then they will be loaded alongside the default curated configuration. + """ + __TYPE_MAP.ts_config.load_termset_config(config_path) # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, @@ -37,10 +45,14 @@ def register_class(**kwargs): def _dec(cls): _set_exp(cls) __TYPE_MAP.register_container_type(namespace, data_type, cls) + cls.type_map = __TYPE_MAP + cls.namespace = namespace return cls else: def _dec(cls): __TYPE_MAP.register_container_type(namespace, data_type, cls) + cls.type_map = __TYPE_MAP + cls.namespace = namespace return cls if container_cls is None: diff --git a/src/hdmf/container.py b/src/hdmf/container.py index a93a4624b..611cf9fb5 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -234,68 +234,49 @@ def __init__(self, **kwargs): self.__read_io = None self.__obj = None - def get_config(self): - from . import TS_CONFIG #update path - return TS_CONFIG - - def get_type_map(self): - from .common import get_type_map - tm = get_type_map() - return tm - @docval({'name': 'constructor_args', 'type': dict, 'doc': 'The fields/parameters/attibutes for the object.'}) def init_validation(self, constructor_args): + """ + + """ # load termset configuartion file from global Config - config = self.get_config() - termset_config = config.config + configurator = self.type_map.ts_config + termset_config = configurator.config if termset_config is not None: - object_name = self.__class__.__name__ - - # Check that the object data_type is in the loaded namespace - tm = self.get_type_map() - - container_types_dict = tm.container_types - object_exists = False - for namespace in container_types_dict: - if object_name in container_types_dict[namespace]: - object_exists = True - else: - continue - - if not object_exists: - msg = "%s is not in the loaded Namespace(s)." % object_name + # check to see that the namespace for the container is in the config + if self.namespace not in self.type_map.container_types: + breakpoint() + msg = "%s not found within loaded configuration." % self.namespace warn(msg) + else: + # check to see that the container type is in the config under the namespace + config_namespace = termset_config['namespaces'][self.namespace] + object_name = self.__class__.__name__ - # Wrap supported fields with TermSetWrapper - obj_wrapped = False - if object_name in termset_config: - obj_wrapped = True - for attr in termset_config[object_name]['fields']: - obj_mapper = tm.get_map(self) - # get the spec according to attr name in schema - # Note: this is the name for the field in the config - spec = obj_mapper.get_attr_spec(attr) - - # In the case of dealing with datasets directly or not defined in the spec. - # (Data/VectorData/DynamicTable/etc) - if spec is None: - # constr_name= attr - msg = "Spec not found for %s" % attr - warn(msg) - # From the spec, get the corresponding constructor name - else: - constr_name = obj_mapper.get_const_arg(spec) - - if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) - termset_path = termset_config[object_name]['fields'][attr] - termset = TermSet(term_schema_path=termset_path) - constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) - - # Even if the data_type is in the namespace, it might not be in the configuration. - if object_exists and not obj_wrapped: - msg = "%s is not in the loaded TermSet Configuration." % object_name - warn(msg) + if object_name not in config_namespace['data_types']: + breakpoint() + msg = '%s not found within the configuration for %s' % (object_name, self.namespace) + else: + for attr in config_namespace['data_types'][object_name]: + obj_mapper = self.type_map.get_map(self) + # get the spec according to attr name in schema + # Note: this is the name for the field in the config + spec = obj_mapper.get_attr_spec(attr) + + # In the case of dealing with datasets directly or not defined in the spec. + # (Data/VectorData/DynamicTable/etc) + breakpoint() + if spec is None: + msg = "Spec not found for %s" % attr + warn(msg) + # From the spec, get the corresponding constructor name + else: + constr_name = obj_mapper.get_const_arg(spec) + if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) + termset_path = config_namespace['data_types'][object_name][attr] + termset = TermSet(term_schema_path=termset_path) + constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) @property def read_io(self): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index f270a5122..9b19f5de5 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -310,35 +310,35 @@ class TermSetConfigurator: """ """ - @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuartion file.'}) + @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuration file.', 'default': None}) def __init__(self, **kwargs): - self.path = [kwargs['path']] self.config = None - self.load_termset_config() + if kwargs['path'] is None: + self.path = [] + else: + self.path = [kwargs['path']] + self.load_termset_config(config_path=self.path[0]) - @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.', - 'default': None}) + @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}) def load_termset_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. """ - # Set self.config for __init__ - if self.config is None: - with open(self.path[0], 'r') as config: - termset_config = yaml.safe_load(config) - self.config = termset_config - else: - # Check data_types within new config to see if they already exist in the current config - with open(config_path, 'r') as config: - termset_config = yaml.safe_load(config) + with open(config_path, 'r') as config: + termset_config = yaml.safe_load(config) + if self.config is None: # set the initial config/load after config has been unloaded + self.config = termset_config + if len(self.path)==0: # for loading after an unloaded config + self.path.append(config_path) + else: # append to the existing config for data_type in termset_config: if data_type in self.config: self.config[data_type] = termset_config[data_type] termset_config.pop(data_type) self.config.update(termset_config) - # append path to new config to self.path - self.path.append(config_path) + # append path to new config to self.path + self.path.append(config_path) def unload_termset_config(self): """ diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index eb37687f1..b3da97a2d 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -18,6 +18,7 @@ DynamicTableRegion, get_manager, SimpleMultiContainer, + load_termset_config ) from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file from hdmf.utils import StrDataset @@ -38,13 +39,14 @@ class TestVDConfig(TestCase): def test_init_config(self): + load_termset_config(config_path='/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/test_config2.yaml') vd = VectorData(name='data', description='',data=['Homo sapiens']) - tb = DynamicTable(name="with_table_columns", description='a test table', columns=[vd]) - from hdmf.common import get_type_map - htm = get_type_map() - om = htm.get_map(tb) - from datetime import datetime - from uuid import uuid4 + # tb = DynamicTable(name="with_table_columns", description='a test table', columns=[vd]) + # from hdmf.common import get_type_map + # htm = get_type_map() + # om = htm.get_map(tb) + # from datetime import datetime + # from uuid import uuid4 # import numpy as np # from dateutil.tz import tzlocal diff --git a/tests/unit/test_config2.yaml b/tests/unit/test_config2.yaml index 7b4a2b9ed..20e63af8f 100644 --- a/tests/unit/test_config2.yaml +++ b/tests/unit/test_config2.yaml @@ -1,23 +1,11 @@ namespaces: - core: - version: + hdmf-common: + version: TBD data_types: VectorData: - data: '...' + data: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' field2: '...' - experimental: - version: - -# -# VectorData: -# namespace: -# namespace_version: -# fields: -# data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml -# field2: ... -# DynamicTable: -# namespace: -# namespace_version: -# fields: -# data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml -# field2: ... + type2: + field1: '...' + hdmf-experimental: + version: TBD diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index ec74d5cb0..352b5faa7 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -190,15 +190,21 @@ def test_set_modified_parent(self): child_obj.set_modified() self.assertTrue(child_obj.parent.modified) + def test_vd(self): + col1 = VectorData( + name='Species_1', + description='...', + data=['Homo sapiens'], + ) def test_all_children(self): col1 = VectorData( name='Species_1', description='...', data=['Homo sapiens'], ) - species = DynamicTable(name='species', description='My species', columns=[col1]) - obj = species.all_objects - self.assertEqual(sorted(list(obj.keys())), sorted([species.object_id, species.id.object_id, col1.object_id])) + # species = DynamicTable(name='species', description='My species', columns=[col1]) + # obj = species.all_objects + # self.assertEqual(sorted(list(obj.keys())), sorted([species.object_id, species.id.object_id, col1.object_id])) def test_add_child(self): """Test that add child creates deprecation warning and also properly sets child's parent and modified From 0478c394aa833fc6686eaf2bee462aff6eeef510 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Thu, 25 Jan 2024 13:38:49 -0800 Subject: [PATCH 12/78] Delete docs/gallery/example_config.yaml --- docs/gallery/example_config.yaml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 docs/gallery/example_config.yaml diff --git a/docs/gallery/example_config.yaml b/docs/gallery/example_config.yaml deleted file mode 100644 index a9277c0a7..000000000 --- a/docs/gallery/example_config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -VectorData: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... -DataType2: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... From e432e6d66c4222bacaf0f0b9c4698e203dd89fa6 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 25 Jan 2024 13:40:54 -0800 Subject: [PATCH 13/78] clean up --- src/hdmf/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index d67d685fb..bbf265254 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -3,6 +3,7 @@ from .container import Container, Data, DataRegion, HERDManager from .region import ListSlicer from .utils import docval, getargs +from .term_set import TermSet, TermSetWrapper, TermSetConfigurator import os From 9e6c2e329783afba9f559a9a3c32fba385f4b00f Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 25 Jan 2024 13:44:37 -0800 Subject: [PATCH 14/78] clean up --- src/hdmf/build/manager.py | 4 ---- src/hdmf/hdmf_config.yaml | 12 ----------- tests/unit/common/test_table.py | 30 --------------------------- tests/unit/test_config.yaml | 23 ++++++++++---------- tests/unit/test_config2.yaml | 11 ---------- tests/unit/test_container.py | 16 +++----------- tests/unit/test_extension_config.yaml | 23 ++++++++++---------- 7 files changed, 25 insertions(+), 94 deletions(-) delete mode 100644 src/hdmf/hdmf_config.yaml delete mode 100644 tests/unit/test_config2.yaml diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index d17946baa..c3189756f 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -421,10 +421,6 @@ def __init__(self, **kwargs): def namespace_catalog(self): return self.__ns_catalog - @property - def data_types(self): - return self.__data_types - @property def container_types(self): return self.__container_types diff --git a/src/hdmf/hdmf_config.yaml b/src/hdmf/hdmf_config.yaml deleted file mode 100644 index a9277c0a7..000000000 --- a/src/hdmf/hdmf_config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -VectorData: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... -DataType2: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index b3da97a2d..c8d5c5311 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -37,36 +37,6 @@ except ImportError: LINKML_INSTALLED = False -class TestVDConfig(TestCase): - def test_init_config(self): - load_termset_config(config_path='/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/test_config2.yaml') - vd = VectorData(name='data', description='',data=['Homo sapiens']) - # tb = DynamicTable(name="with_table_columns", description='a test table', columns=[vd]) - # from hdmf.common import get_type_map - # htm = get_type_map() - # om = htm.get_map(tb) - # from datetime import datetime - # from uuid import uuid4 - - # import numpy as np - # from dateutil.tz import tzlocal - # - # from pynwb import NWBHDF5IO, NWBFile - # from pynwb.behavior import SpatialSeries - # from pynwb import get_type_map - # tm = get_type_map() - # timestamps = np.linspace(0, 50) / 200 - # position_data = np.array([np.linspace(0, 10, 50), np.linspace(0, 8, 50)]).T - # - # position_spatial_series = SpatialSeries( - # name="SpatialSeries", - # description="Position (x, y) in an open field.", - # data=position_data, - # timestamps=timestamps, - # reference_frame="(0,0) is bottom left corner", - # ) - breakpoint() - class TestDynamicTable(TestCase): diff --git a/tests/unit/test_config.yaml b/tests/unit/test_config.yaml index dbc5f48c2..20e63af8f 100644 --- a/tests/unit/test_config.yaml +++ b/tests/unit/test_config.yaml @@ -1,12 +1,11 @@ -VectorData: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... -DynamicTable: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... +namespaces: + hdmf-common: + version: TBD + data_types: + VectorData: + data: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' + field2: '...' + type2: + field1: '...' + hdmf-experimental: + version: TBD diff --git a/tests/unit/test_config2.yaml b/tests/unit/test_config2.yaml deleted file mode 100644 index 20e63af8f..000000000 --- a/tests/unit/test_config2.yaml +++ /dev/null @@ -1,11 +0,0 @@ -namespaces: - hdmf-common: - version: TBD - data_types: - VectorData: - data: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' - field2: '...' - type2: - field1: '...' - hdmf-experimental: - version: TBD diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index 352b5faa7..b5a2d87e8 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -32,10 +32,6 @@ def test_link_and_get_resources(self): er_get = em.get_linked_resources() self.assertEqual(er, er_get) -class TestContainerConfig(TestCase): - def test_init_config(self): - obj = Container('obj1') - class TestContainer(TestCase): @@ -190,21 +186,15 @@ def test_set_modified_parent(self): child_obj.set_modified() self.assertTrue(child_obj.parent.modified) - def test_vd(self): - col1 = VectorData( - name='Species_1', - description='...', - data=['Homo sapiens'], - ) def test_all_children(self): col1 = VectorData( name='Species_1', description='...', data=['Homo sapiens'], ) - # species = DynamicTable(name='species', description='My species', columns=[col1]) - # obj = species.all_objects - # self.assertEqual(sorted(list(obj.keys())), sorted([species.object_id, species.id.object_id, col1.object_id])) + species = DynamicTable(name='species', description='My species', columns=[col1]) + obj = species.all_objects + self.assertEqual(sorted(list(obj.keys())), sorted([species.object_id, species.id.object_id, col1.object_id])) def test_add_child(self): """Test that add child creates deprecation warning and also properly sets child's parent and modified diff --git a/tests/unit/test_extension_config.yaml b/tests/unit/test_extension_config.yaml index a9277c0a7..20e63af8f 100644 --- a/tests/unit/test_extension_config.yaml +++ b/tests/unit/test_extension_config.yaml @@ -1,12 +1,11 @@ -VectorData: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... -DataType2: - namespace: - namespace_version: - fields: - data: /Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml - field2: ... +namespaces: + hdmf-common: + version: TBD + data_types: + VectorData: + data: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' + field2: '...' + type2: + field1: '...' + hdmf-experimental: + version: TBD From fd73ae09c5711b92435e8bceb9e85a92e41e280d Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 25 Jan 2024 13:45:50 -0800 Subject: [PATCH 15/78] clean up --- tests/unit/common/test_table.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index c8d5c5311..6c24c2b5a 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -17,8 +17,7 @@ EnumData, DynamicTableRegion, get_manager, - SimpleMultiContainer, - load_termset_config + SimpleMultiContainer ) from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file from hdmf.utils import StrDataset From 6a9c956a6f69c63074711d21fda457c0d544aee0 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 29 Jan 2024 09:37:59 -0800 Subject: [PATCH 16/78] checkpoint --- src/hdmf/container.py | 79 ++++++++++++++++++++++++++++++++++++++----- src/hdmf/term_set.py | 43 ++++++++++++++++++----- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 611cf9fb5..b9d2e9cce 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -6,6 +6,7 @@ from uuid import uuid4 from warnings import warn import yaml +import os import h5py import numpy as np @@ -72,12 +73,12 @@ class AbstractContainer(metaclass=ExtenderMeta): # Override the _setter factor function, so directives that apply to # Container do not get used on Data @classmethod - def _setter(cls, field): + def _setter(cls, field): # dada """ Make a setter function for creating a :py:func:`property` """ name = field['name'] - + # breakpoint() if not field.get('settable', True): return None @@ -87,10 +88,66 @@ def setter(self, val): if name in self.fields: msg = "can't set attribute '%s' -- already set" % name raise AttributeError(msg) - self.fields[name] = val + self.fields[name] = self._field_config(arg_name=name, val=val) + # self.fields[name] = val return setter + + def _field_config(self, arg_name, val): + """ + + """ + # load termset configuartion file from global Config + # if arg_name == 'location': + # breakpoint() + configurator = self.type_map.ts_config + if len(configurator.path)>0: + CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) + termset_config = configurator.config + + + # check to see that the namespace for the container is in the config + if self.namespace not in self.type_map.container_types: + msg = "%s not found within loaded configuration." % self.namespace + warn(msg) + else: + # check to see that the container type is in the config under the namespace + config_namespace = termset_config['namespaces'][self.namespace] + object_name = self.__class__.__name__ + + if object_name not in config_namespace['data_types']: + msg = '%s not found within the configuration for %s' % (object_name, self.namespace) + else: + for attr in config_namespace['data_types'][object_name]: + # if the attr has been manually wrapped then skip checking the config for the attr + if type(attr) == TermSetWrapper: + continue + + obj_mapper = self.type_map.get_map(self) + # get the spec according to attr name in schema + # Note: this is the name for the field in the config + + spec = obj_mapper.get_attr_spec(attr) + + # In the case of dealing with datasets directly or not defined in the spec. + # (Data/VectorData/DynamicTable/etc) + if spec is None: + msg = "Spec not found for %s" % attr + warn(msg) + # From the spec, get the corresponding constructor name + else: + constr_name = obj_mapper.get_const_arg(spec) + if constr_name == arg_name: # make sure any custom fields are not handled (i.e., make an extension) + termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) + termset = TermSet(term_schema_path=termset_path) + val = TermSetWrapper(value=val, termset=termset) + return val + return val + else: + return val + + @classmethod def _getter(cls, field): """ @@ -242,11 +299,12 @@ def init_validation(self, constructor_args): """ # load termset configuartion file from global Config configurator = self.type_map.ts_config + CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) termset_config = configurator.config + if termset_config is not None: # check to see that the namespace for the container is in the config if self.namespace not in self.type_map.container_types: - breakpoint() msg = "%s not found within loaded configuration." % self.namespace warn(msg) else: @@ -255,18 +313,21 @@ def init_validation(self, constructor_args): object_name = self.__class__.__name__ if object_name not in config_namespace['data_types']: - breakpoint() msg = '%s not found within the configuration for %s' % (object_name, self.namespace) else: for attr in config_namespace['data_types'][object_name]: + # if the attr has been manually wrapped then skip checking the config for the attr + if type(attr) == TermSetWrapper: + continue + obj_mapper = self.type_map.get_map(self) # get the spec according to attr name in schema # Note: this is the name for the field in the config + spec = obj_mapper.get_attr_spec(attr) # In the case of dealing with datasets directly or not defined in the spec. # (Data/VectorData/DynamicTable/etc) - breakpoint() if spec is None: msg = "Spec not found for %s" % attr warn(msg) @@ -274,7 +335,7 @@ def init_validation(self, constructor_args): else: constr_name = obj_mapper.get_const_arg(spec) if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) - termset_path = config_namespace['data_types'][object_name][attr] + termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) termset = TermSet(term_schema_path=termset_path) constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) @@ -546,6 +607,7 @@ class Container(AbstractContainer): def _setter(cls, field): """Returns a list of setter functions for the given field to be added to the class during class declaration.""" super_setter = AbstractContainer._setter(field) + # breakpoint() ret = [super_setter] # create setter with check for required name # the AbstractContainer that is passed to the setter must have name = required_name @@ -572,6 +634,7 @@ def container_setter(self, val): idx2 = len(ret) - 1 def container_setter(self, val): + # breakpoint() ret[idx2](self, val) # call the previous setter if val is not None: if isinstance(val, (tuple, list)): @@ -830,8 +893,6 @@ class Data(AbstractContainer): @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'}, {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'}) def __init__(self, **kwargs): - self.init_validation(constructor_args=kwargs) - # breakpoint() data = popargs('data', kwargs) super().__init__(**kwargs) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 9b19f5de5..b05167106 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -319,26 +319,51 @@ def __init__(self, **kwargs): self.path = [kwargs['path']] self.load_termset_config(config_path=self.path[0]) + def get_config(self, object_name, namespace): + try: + namespace_config = self.config['namespaces'][namespace] + except KeyError: + msg = 'The namespace %s was not found within the configuration.' % namespace + raise ValueError(msg) + + try: + type_config = namespace_config['data_types'][object_name] + return type_config + except KeyError: + msg = '%s was not found within the configuration for that namespace.' % object_name + raise ValueError(msg) + @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}) def load_termset_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. """ + with open(config_path, 'r') as config: termset_config = yaml.safe_load(config) if self.config is None: # set the initial config/load after config has been unloaded self.config = termset_config if len(self.path)==0: # for loading after an unloaded config self.path.append(config_path) - else: # append to the existing config - for data_type in termset_config: - if data_type in self.config: - self.config[data_type] = termset_config[data_type] - termset_config.pop(data_type) - self.config.update(termset_config) - - # append path to new config to self.path - self.path.append(config_path) + else: # append/replace to the existing config + if config_path in self.path: + msg = 'This configuration file path already exists within the configurator.' + raise ValueError(msg) + else: + for namespace in termset_config: + if namespace not in self.config: # append namespace config if not present within self.config + self.config['namespaces'][namespace] = termset_config['namespaces'][namespace] + else: # check for any needed overrides within existing namespace configs + for data_type in termset_config['namespaces'][namespace]['data_types']: + if data_type in self.config['namespaces'][namespace]['data_types']: + replace_config = termset_config['namespaces'][namespace]['data_types'][data_type] + self.config['namespaces'][namespace]['data_types'][data_type] = replace_config + else: # append to config + new_data_type_config = termset_config['namespaces'][namespace]['data_types'][data_type] + self.config['namespaces'][namespace]['data_types'] = new_data_type_config + + # append path to self.path + self.path.append(config_path) def unload_termset_config(self): """ From ec36ee3d38a8ee22cc18ffe6a353b57e0143a447 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 31 Jan 2024 09:42:14 -0800 Subject: [PATCH 17/78] need to clean --- src/hdmf/common/__init__.py | 9 ++++----- src/hdmf/container.py | 13 +++++++------ src/hdmf/term_set.py | 3 ++- tests/unit/test_config.yaml | 4 ++++ tests/unit/test_extension_config.yaml | 5 +++++ 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index cd3b98120..ffbf8f672 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -20,13 +20,12 @@ # a global type map global __TYPE_MAP -def load_termset_config(config_path: str): +def load_termset_config(config_path: str = None): """ - This method will: - - Search the current configuration for data_types that are already present. These data_types will be - replaced with the new configuration. - - If the data_type is not present, then they will be loaded alongside the default curated configuration. + This method will either load the default config or the config provided by the path. """ + if config_path is None: + config_path = path_to_config __TYPE_MAP.ts_config.load_termset_config(config_path) # a function to register a container classes with the global map diff --git a/src/hdmf/container.py b/src/hdmf/container.py index e584c9e45..c1a3aa3a4 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -37,7 +37,7 @@ class HERDManager: This class manages whether to set/attach an instance of HERD to the subclass. """ - @docval({'name': 'herd', 'type': 'hdmf.common.resources.HERD', + @docval({'name': 'herd', 'type': 'HERD', 'doc': 'The external resources to be used for the container.'},) def link_resources(self, **kwargs): """ @@ -78,7 +78,8 @@ def _setter(cls, field): # dada Make a setter function for creating a :py:func:`property` """ name = field['name'] - # breakpoint() + # if name == 'electrodes': + # breakpoint() if not field.get('settable', True): return None @@ -89,7 +90,6 @@ def setter(self, val): msg = "can't set attribute '%s' -- already set" % name raise AttributeError(msg) self.fields[name] = self._field_config(arg_name=name, val=val) - # self.fields[name] = val return setter @@ -99,8 +99,6 @@ def _field_config(self, arg_name, val): """ # load termset configuartion file from global Config - # if arg_name == 'location': - # breakpoint() configurator = self.type_map.ts_config if len(configurator.path)>0: CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) @@ -120,6 +118,8 @@ def _field_config(self, arg_name, val): msg = '%s not found within the configuration for %s' % (object_name, self.namespace) else: for attr in config_namespace['data_types'][object_name]: + # if object_name == 'NWBFile': + # breakpoint() # if the attr has been manually wrapped then skip checking the config for the attr if type(attr) == TermSetWrapper: continue @@ -141,6 +141,7 @@ def _field_config(self, arg_name, val): if constr_name == arg_name: # make sure any custom fields are not handled (i.e., make an extension) termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) termset = TermSet(term_schema_path=termset_path) + val = TermSetWrapper(value=val, termset=termset) return val return val @@ -496,7 +497,7 @@ def set_modified(self, **kwargs): def children(self): return tuple(self.__children) - @docval({'name': 'child', 'type': 'hdmf.container.Container', + @docval({'name': 'child', 'type': 'Container', 'doc': 'the child Container for this Container', 'default': None}) def add_child(self, **kwargs): warn(DeprecationWarning('add_child is deprecated. Set the parent attribute instead.')) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index b05167106..bb242cca8 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -350,10 +350,11 @@ def load_termset_config(self,config_path): msg = 'This configuration file path already exists within the configurator.' raise ValueError(msg) else: - for namespace in termset_config: + for namespace in termset_config['namespaces']: if namespace not in self.config: # append namespace config if not present within self.config self.config['namespaces'][namespace] = termset_config['namespaces'][namespace] else: # check for any needed overrides within existing namespace configs + breakpoint() for data_type in termset_config['namespaces'][namespace]['data_types']: if data_type in self.config['namespaces'][namespace]['data_types']: replace_config = termset_config['namespaces'][namespace]['data_types'][data_type] diff --git a/tests/unit/test_config.yaml b/tests/unit/test_config.yaml index 20e63af8f..0462131ef 100644 --- a/tests/unit/test_config.yaml +++ b/tests/unit/test_config.yaml @@ -9,3 +9,7 @@ namespaces: field1: '...' hdmf-experimental: version: TBD + data_types: + type1: + field1: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' + field2: '...' diff --git a/tests/unit/test_extension_config.yaml b/tests/unit/test_extension_config.yaml index 20e63af8f..e0bf09d0b 100644 --- a/tests/unit/test_extension_config.yaml +++ b/tests/unit/test_extension_config.yaml @@ -9,3 +9,8 @@ namespaces: field1: '...' hdmf-experimental: version: TBD + core: + data_types: + NWBFile: + experimenter: experimenter_termset2.yaml + blah: experimenter.yaml From fd8995a1d4b81ab24d29b6e608bf0ddab1ebb716 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 31 Jan 2024 10:23:55 -0800 Subject: [PATCH 18/78] partial clean up --- src/hdmf/__init__.py | 2 - src/hdmf/container.py | 84 +++++--------------- tests/unit/test_termset_config.py | 126 +++++++++++++++--------------- 3 files changed, 83 insertions(+), 129 deletions(-) diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index bbf265254..b32a7f579 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -5,8 +5,6 @@ from .utils import docval, getargs from .term_set import TermSet, TermSetWrapper, TermSetConfigurator -import os - @docval( {"name": "dataset", "type": None, "doc": "the HDF5 dataset to slice"}, diff --git a/src/hdmf/container.py b/src/hdmf/container.py index c1a3aa3a4..f4dc8ff1e 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -73,13 +73,11 @@ class AbstractContainer(metaclass=ExtenderMeta): # Override the _setter factor function, so directives that apply to # Container do not get used on Data @classmethod - def _setter(cls, field): # dada + def _setter(cls, field): """ Make a setter function for creating a :py:func:`property` """ name = field['name'] - # if name == 'electrodes': - # breakpoint() if not field.get('settable', True): return None @@ -96,7 +94,9 @@ def setter(self, val): def _field_config(self, arg_name, val): """ - + This method will be called in the setter. The termset configuration will be used (if loaded) + to check for a defined TermSet associated with the field. If found, the value of the field + will be wrapped with a TermSetWrapper. """ # load termset configuartion file from global Config configurator = self.type_map.ts_config @@ -109,6 +109,7 @@ def _field_config(self, arg_name, val): if self.namespace not in self.type_map.container_types: msg = "%s not found within loaded configuration." % self.namespace warn(msg) + return val else: # check to see that the container type is in the config under the namespace config_namespace = termset_config['namespaces'][self.namespace] @@ -116,13 +117,11 @@ def _field_config(self, arg_name, val): if object_name not in config_namespace['data_types']: msg = '%s not found within the configuration for %s' % (object_name, self.namespace) + warn(msg) + return val else: for attr in config_namespace['data_types'][object_name]: - # if object_name == 'NWBFile': - # breakpoint() - # if the attr has been manually wrapped then skip checking the config for the attr - if type(attr) == TermSetWrapper: - continue + obj_mapper = self.type_map.get_map(self) # get the spec according to attr name in schema @@ -133,18 +132,25 @@ def _field_config(self, arg_name, val): # In the case of dealing with datasets directly or not defined in the spec. # (Data/VectorData/DynamicTable/etc) if spec is None: - msg = "Spec not found for %s" % attr + msg = "Spec not found for %s." % attr warn(msg) + return val # From the spec, get the corresponding constructor name else: constr_name = obj_mapper.get_const_arg(spec) - if constr_name == arg_name: # make sure any custom fields are not handled (i.e., make an extension) + if constr_name == arg_name: termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) termset = TermSet(term_schema_path=termset_path) - - val = TermSetWrapper(value=val, termset=termset) + # If the val has been manually wrapped then skip checking the config for the attr + if type(val) == TermSetWrapper: + return val + else: + val = TermSetWrapper(value=val, termset=termset) + return val + else: + msg = "The name of the field does not match the field name in the schema. Field value not wrapped." + warn(msg) return val - return val else: return val @@ -292,54 +298,6 @@ def __init__(self, **kwargs): self.__read_io = None self.__obj = None - @docval({'name': 'constructor_args', 'type': dict, - 'doc': 'The fields/parameters/attibutes for the object.'}) - def init_validation(self, constructor_args): - """ - - """ - # load termset configuartion file from global Config - configurator = self.type_map.ts_config - CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) - termset_config = configurator.config - - if termset_config is not None: - # check to see that the namespace for the container is in the config - if self.namespace not in self.type_map.container_types: - msg = "%s not found within loaded configuration." % self.namespace - warn(msg) - else: - # check to see that the container type is in the config under the namespace - config_namespace = termset_config['namespaces'][self.namespace] - object_name = self.__class__.__name__ - - if object_name not in config_namespace['data_types']: - msg = '%s not found within the configuration for %s' % (object_name, self.namespace) - else: - for attr in config_namespace['data_types'][object_name]: - # if the attr has been manually wrapped then skip checking the config for the attr - if type(attr) == TermSetWrapper: - continue - - obj_mapper = self.type_map.get_map(self) - # get the spec according to attr name in schema - # Note: this is the name for the field in the config - - spec = obj_mapper.get_attr_spec(attr) - - # In the case of dealing with datasets directly or not defined in the spec. - # (Data/VectorData/DynamicTable/etc) - if spec is None: - msg = "Spec not found for %s" % attr - warn(msg) - # From the spec, get the corresponding constructor name - else: - constr_name = obj_mapper.get_const_arg(spec) - if constr_name in constructor_args: # make sure any custom fields are not handled (i.e., make an extension) - termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) - termset = TermSet(term_schema_path=termset_path) - constructor_args[attr] = TermSetWrapper(value=constructor_args[attr], termset=termset) - @property def read_io(self): """ @@ -608,7 +566,6 @@ class Container(AbstractContainer): def _setter(cls, field): """Returns a list of setter functions for the given field to be added to the class during class declaration.""" super_setter = AbstractContainer._setter(field) - # breakpoint() ret = [super_setter] # create setter with check for required name # the AbstractContainer that is passed to the setter must have name = required_name @@ -635,7 +592,6 @@ def container_setter(self, val): idx2 = len(ret) - 1 def container_setter(self, val): - # breakpoint() ret[idx2](self, val) # call the previous setter if val is not None: if isinstance(val, (tuple, list)): diff --git a/tests/unit/test_termset_config.py b/tests/unit/test_termset_config.py index 269207068..18dcc6ca1 100644 --- a/tests/unit/test_termset_config.py +++ b/tests/unit/test_termset_config.py @@ -1,63 +1,63 @@ -from hdmf.testing import TestCase -from hdmf import load_termset_config, unload_termset_config -from hdmf.term_set import TermSetConfigurator - -class TestConfig(TestCase): - self.test_config = {} - self.test_merged_extension_config = {} - self.hdmf_config = {} - - def test_construct_config(self): - # add asserts for self.path and self.config - test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') - self.assertEqual(test_config.path, ['tests/unit/test_config.yaml']) - # self.assertEqual(test_config.config, None) - - def test_load_termset_config(self): - test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') - test_config.load_termset_config(path='tests/unit/test_extension_config.yaml') - self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) - # self.assertEqual(config.config, None) - - def test_unload_termset_config(self): - test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') - test_config.unload_termset_config() - self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) - self.assertEqual(config.config, None) - - def test_unload_global_config(self): - config = get_termset_config() - unload_termset_config() - self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) - self.assertEqual(config.config, None) - - def test_load_global_config_reset(self): - load_termset_config() - self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) - # self.assertEqual(config.config, None) - - def test_load_global_config_extension_config(self): - load_termset_config() - self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) - # self.assertEqual(config.config, None) - - def test_data(self): - pass - - def test_dataset_not_in_spec(self): - pass - - def test_attribute_not_in_spec(self): - pass - - def test_attriute_in_spec(self): - pass - - def test_dataset_in_spec(self): - pass - - def test_data_type_not_in_namespace(self): - pass - - def test_warn_not_wrapped(self): - pass +# from hdmf.testing import TestCase +# from hdmf import load_termset_config, unload_termset_config +# from hdmf.term_set import TermSetConfigurator +# +# class TestConfig(TestCase): +# self.test_config = {} +# self.test_merged_extension_config = {} +# self.hdmf_config = {} +# +# def test_construct_config(self): +# # add asserts for self.path and self.config +# test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') +# self.assertEqual(test_config.path, ['tests/unit/test_config.yaml']) +# # self.assertEqual(test_config.config, None) +# +# def test_load_termset_config(self): +# test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') +# test_config.load_termset_config(path='tests/unit/test_extension_config.yaml') +# self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) +# # self.assertEqual(config.config, None) +# +# def test_unload_termset_config(self): +# test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') +# test_config.unload_termset_config() +# self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) +# self.assertEqual(config.config, None) +# +# def test_unload_global_config(self): +# config = get_termset_config() +# unload_termset_config() +# self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) +# self.assertEqual(config.config, None) +# +# def test_load_global_config_reset(self): +# load_termset_config() +# self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) +# # self.assertEqual(config.config, None) +# +# def test_load_global_config_extension_config(self): +# load_termset_config() +# self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) +# # self.assertEqual(config.config, None) +# +# def test_data(self): +# pass +# +# def test_dataset_not_in_spec(self): +# pass +# +# def test_attribute_not_in_spec(self): +# pass +# +# def test_attriute_in_spec(self): +# pass +# +# def test_dataset_in_spec(self): +# pass +# +# def test_data_type_not_in_namespace(self): +# pass +# +# def test_warn_not_wrapped(self): +# pass From 041099edef65a24c024bfb3bdcd8896f975b4c12 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 31 Jan 2024 10:32:10 -0800 Subject: [PATCH 19/78] warn --- src/hdmf/container.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index f4dc8ff1e..e62fb68b2 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -143,6 +143,8 @@ def _field_config(self, arg_name, val): termset = TermSet(term_schema_path=termset_path) # If the val has been manually wrapped then skip checking the config for the attr if type(val) == TermSetWrapper: + msg = "Field value already wrapped with TermSetWrapper." + warn(msg) return val else: val = TermSetWrapper(value=val, termset=termset) From 145513a72540d848a4d5c01ee785aaec8601da9f Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 12 Mar 2024 15:53:15 -0700 Subject: [PATCH 20/78] yaml changes --- pyproject.toml | 3 +- requirements-min.txt | 1 - requirements-opt.txt | 1 - src/hdmf/container.py | 1 - src/hdmf/term_set.py | 7 ++- .../schemasheets/nwb_static_enums.yaml | 58 +++++++------------ 6 files changed, 26 insertions(+), 45 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee8037be5..0ff3ea237 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,7 @@ zarr = ["zarr>=2.12.0"] tqdm = ["tqdm>=4.41.0"] termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'", "schemasheets>=0.1.23; python_version >= '3.9'", - "oaklib>=0.5.12; python_version >= '3.9'", - "pyyaml>=6.0.1; python_version >= '3.9'"] + "oaklib>=0.5.12; python_version >= '3.9'"] [project.urls] "Homepage" = "https://github.com/hdmf-dev/hdmf" diff --git a/requirements-min.txt b/requirements-min.txt index a437fc588..af3d8672c 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -10,6 +10,5 @@ scipy==1.4 # linkml-runtime==1.5.5; python_version >= "3.9" # schemasheets==0.1.23; python_version >= "3.9" # oaklib==0.5.12; python_version >= "3.9" -# pyyaml==6.0.1; python_version >= "3.9" tqdm==4.41.0 zarr==2.12.0 diff --git a/requirements-opt.txt b/requirements-opt.txt index 644fc80be..a89b0cede 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -5,4 +5,3 @@ linkml-runtime==1.6.0; python_version >= "3.9" schemasheets==0.1.24; python_version >= "3.9" oaklib==0.5.20; python_version >= "3.9" pydantic==1.10.13 # linkml-runtime 1.6.0 and related packages require pydantic<2 -pyyaml==6.0.1; python_version >= "3.9" diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 9db48aa6f..ef658c2d9 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -5,7 +5,6 @@ from typing import Type from uuid import uuid4 from warnings import warn -import yaml import os import h5py diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index bb242cca8..cb601b0e3 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,9 +5,11 @@ import warnings import numpy as np from .data_utils import append_data, extend_data -import yaml +from ruamel.yaml import YAML +yaml=YAML(typ='safe') # Use ruamel to avoid multple YAML packages + class TermSet: """ Class for implementing term sets from ontologies and other resources used to define the @@ -163,7 +165,6 @@ def __schemasheets_convert(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: - import yaml from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover @@ -340,7 +341,7 @@ def load_termset_config(self,config_path): """ with open(config_path, 'r') as config: - termset_config = yaml.safe_load(config) + termset_config = yaml.load(config) if self.config is None: # set the initial config/load after config has been unloaded self.config = termset_config if len(self.path)==0: # for loading after an unloaded config diff --git a/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml b/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml index 222205959..52af4b8a7 100644 --- a/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml +++ b/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml @@ -2,8 +2,7 @@ classes: BrainSample: slot_usage: cell_type: {} - slots: - - cell_type + slots: [cell_type] default_prefix: TEMP default_range: string description: this schema demonstrates the use of static enums @@ -11,42 +10,27 @@ enums: NeuronOrGlialCellTypeEnum: description: Enumeration to capture various cell types found in the brain. permissible_values: - ASTROCYTE: - description: Characteristic star-shaped glial cells in the brain and spinal - cord. - meaning: CL:0000127 - INTERNEURON: - description: Neurons whose axons (and dendrites) are limited to a single brain - area. - meaning: CL:0000099 - MICROGLIAL_CELL: - description: Microglia are the resident immune cells of the brain and constantly - patrol the cerebral microenvironment to respond to pathogens and damage. - meaning: CL:0000129 - MOTOR_NEURON: - description: Neurons whose cell body is located in the motor cortex, brainstem - or the spinal cord, and whose axon (fiber) projects to the spinal cord or - outside of the spinal cord to directly or indirectly control effector organs, - mainly muscles and glands. - meaning: CL:0000100 - OLIGODENDROCYTE: - description: Type of neuroglia whose main functions are to provide support - and insulation to axons within the central nervous system (CNS) of jawed - vertebrates. - meaning: CL:0000128 - PYRAMIDAL_NEURON: - description: Neurons with a pyramidal shaped cell body (soma) and two distinct - dendritic trees. - meaning: CL:0000598 + ASTROCYTE: {description: Characteristic star-shaped glial cells in the brain + and spinal cord., meaning: 'CL:0000127'} + INTERNEURON: {description: Neurons whose axons (and dendrites) are limited to + a single brain area., meaning: 'CL:0000099'} + MICROGLIAL_CELL: {description: Microglia are the resident immune cells of the + brain and constantly patrol the cerebral microenvironment to respond to + pathogens and damage., meaning: 'CL:0000129'} + MOTOR_NEURON: {description: 'Neurons whose cell body is located in the motor + cortex, brainstem or the spinal cord, and whose axon (fiber) projects to + the spinal cord or outside of the spinal cord to directly or indirectly + control effector organs, mainly muscles and glands.', meaning: 'CL:0000100'} + OLIGODENDROCYTE: {description: Type of neuroglia whose main functions are to + provide support and insulation to axons within the central nervous system + (CNS) of jawed vertebrates., meaning: 'CL:0000128'} + PYRAMIDAL_NEURON: {description: Neurons with a pyramidal shaped cell body (soma) + and two distinct dendritic trees., meaning: 'CL:0000598'} id: https://w3id.org/linkml/examples/nwb_static_enums -imports: -- linkml:types +imports: ['linkml:types'] name: nwb_static_enums -prefixes: - CL: http://purl.obolibrary.org/obo/CL_ - TEMP: https://example.org/TEMP/ - linkml: https://w3id.org/linkml/ +prefixes: {CL: 'http://purl.obolibrary.org/obo/CL_', TEMP: 'https://example.org/TEMP/', + linkml: 'https://w3id.org/linkml/'} slots: - cell_type: - required: true + cell_type: {required: true} title: static enums example From bb4b422526953f104891e5751b230ecc6900e370 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 12 Mar 2024 16:10:08 -0700 Subject: [PATCH 21/78] revert --- pyproject.toml | 3 +- requirements-min.txt | 1 + requirements-opt.txt | 1 + src/hdmf/term_set.py | 4 +- .../schemasheets/nwb_static_enums.yaml | 58 ++++++++++++------- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0ff3ea237..ee8037be5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,8 @@ zarr = ["zarr>=2.12.0"] tqdm = ["tqdm>=4.41.0"] termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'", "schemasheets>=0.1.23; python_version >= '3.9'", - "oaklib>=0.5.12; python_version >= '3.9'"] + "oaklib>=0.5.12; python_version >= '3.9'", + "pyyaml>=6.0.1; python_version >= '3.9'"] [project.urls] "Homepage" = "https://github.com/hdmf-dev/hdmf" diff --git a/requirements-min.txt b/requirements-min.txt index af3d8672c..a437fc588 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -10,5 +10,6 @@ scipy==1.4 # linkml-runtime==1.5.5; python_version >= "3.9" # schemasheets==0.1.23; python_version >= "3.9" # oaklib==0.5.12; python_version >= "3.9" +# pyyaml==6.0.1; python_version >= "3.9" tqdm==4.41.0 zarr==2.12.0 diff --git a/requirements-opt.txt b/requirements-opt.txt index a89b0cede..644fc80be 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -5,3 +5,4 @@ linkml-runtime==1.6.0; python_version >= "3.9" schemasheets==0.1.24; python_version >= "3.9" oaklib==0.5.20; python_version >= "3.9" pydantic==1.10.13 # linkml-runtime 1.6.0 and related packages require pydantic<2 +pyyaml==6.0.1; python_version >= "3.9" diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index cb601b0e3..21931f7ea 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,11 +5,8 @@ import warnings import numpy as np from .data_utils import append_data, extend_data -from ruamel.yaml import YAML -yaml=YAML(typ='safe') # Use ruamel to avoid multple YAML packages - class TermSet: """ Class for implementing term sets from ontologies and other resources used to define the @@ -165,6 +162,7 @@ def __schemasheets_convert(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: + import yaml from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover diff --git a/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml b/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml index 52af4b8a7..222205959 100644 --- a/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml +++ b/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml @@ -2,7 +2,8 @@ classes: BrainSample: slot_usage: cell_type: {} - slots: [cell_type] + slots: + - cell_type default_prefix: TEMP default_range: string description: this schema demonstrates the use of static enums @@ -10,27 +11,42 @@ enums: NeuronOrGlialCellTypeEnum: description: Enumeration to capture various cell types found in the brain. permissible_values: - ASTROCYTE: {description: Characteristic star-shaped glial cells in the brain - and spinal cord., meaning: 'CL:0000127'} - INTERNEURON: {description: Neurons whose axons (and dendrites) are limited to - a single brain area., meaning: 'CL:0000099'} - MICROGLIAL_CELL: {description: Microglia are the resident immune cells of the - brain and constantly patrol the cerebral microenvironment to respond to - pathogens and damage., meaning: 'CL:0000129'} - MOTOR_NEURON: {description: 'Neurons whose cell body is located in the motor - cortex, brainstem or the spinal cord, and whose axon (fiber) projects to - the spinal cord or outside of the spinal cord to directly or indirectly - control effector organs, mainly muscles and glands.', meaning: 'CL:0000100'} - OLIGODENDROCYTE: {description: Type of neuroglia whose main functions are to - provide support and insulation to axons within the central nervous system - (CNS) of jawed vertebrates., meaning: 'CL:0000128'} - PYRAMIDAL_NEURON: {description: Neurons with a pyramidal shaped cell body (soma) - and two distinct dendritic trees., meaning: 'CL:0000598'} + ASTROCYTE: + description: Characteristic star-shaped glial cells in the brain and spinal + cord. + meaning: CL:0000127 + INTERNEURON: + description: Neurons whose axons (and dendrites) are limited to a single brain + area. + meaning: CL:0000099 + MICROGLIAL_CELL: + description: Microglia are the resident immune cells of the brain and constantly + patrol the cerebral microenvironment to respond to pathogens and damage. + meaning: CL:0000129 + MOTOR_NEURON: + description: Neurons whose cell body is located in the motor cortex, brainstem + or the spinal cord, and whose axon (fiber) projects to the spinal cord or + outside of the spinal cord to directly or indirectly control effector organs, + mainly muscles and glands. + meaning: CL:0000100 + OLIGODENDROCYTE: + description: Type of neuroglia whose main functions are to provide support + and insulation to axons within the central nervous system (CNS) of jawed + vertebrates. + meaning: CL:0000128 + PYRAMIDAL_NEURON: + description: Neurons with a pyramidal shaped cell body (soma) and two distinct + dendritic trees. + meaning: CL:0000598 id: https://w3id.org/linkml/examples/nwb_static_enums -imports: ['linkml:types'] +imports: +- linkml:types name: nwb_static_enums -prefixes: {CL: 'http://purl.obolibrary.org/obo/CL_', TEMP: 'https://example.org/TEMP/', - linkml: 'https://w3id.org/linkml/'} +prefixes: + CL: http://purl.obolibrary.org/obo/CL_ + TEMP: https://example.org/TEMP/ + linkml: https://w3id.org/linkml/ slots: - cell_type: {required: true} + cell_type: + required: true title: static enums example From 55479e34b8443f93fc15df7263820cfc3e3eedb3 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 12 Mar 2024 16:25:15 -0700 Subject: [PATCH 22/78] except --- src/hdmf/container.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index ef658c2d9..143cc2553 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -98,7 +98,11 @@ def _field_config(self, arg_name, val): will be wrapped with a TermSetWrapper. """ # load termset configuartion file from global Config - configurator = self.type_map.ts_config + try: + configurator = self.type_map.ts_config + except AttributeError: # This is for containers that are not registered, e.g., testing classes. + return val + if len(configurator.path)>0: CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) termset_config = configurator.config From 3d67fbce89f3fa4aad2de8a6bd81c16d28c5a63d Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 13 Mar 2024 14:29:14 -0700 Subject: [PATCH 23/78] clean up --- src/hdmf/common/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index ffbf8f672..a3c3a80e4 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -20,14 +20,6 @@ # a global type map global __TYPE_MAP -def load_termset_config(config_path: str = None): - """ - This method will either load the default config or the config provided by the path. - """ - if config_path is None: - config_path = path_to_config - __TYPE_MAP.ts_config.load_termset_config(config_path) - # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, {'name': 'namespace', 'type': str, 'doc': 'the name of the namespace', 'default': CORE_NAMESPACE}, From 5b7787afa5df54fb1c48f4d0d4db0db167f50c46 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 13 Mar 2024 16:53:22 -0700 Subject: [PATCH 24/78] warning tests --- src/hdmf/common/__init__.py | 26 +++++++++ src/hdmf/container.py | 93 +++++++++++++++------------------ src/hdmf/term_set.py | 9 ++-- tests/unit/common/test_table.py | 49 +++++++++++++++-- tests/unit/hdmf_config.yaml | 6 +++ 5 files changed, 126 insertions(+), 57 deletions(-) create mode 100644 tests/unit/hdmf_config.yaml diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index a3c3a80e4..217f6cba1 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -20,6 +20,32 @@ # a global type map global __TYPE_MAP +@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}, + {'name': 'return_map', 'type': bool, 'doc': 'Bool to return type_map', 'default': False}, + is_method=False) +def load_termset_config(**kwargs): + """ + This method will either load the default config or the config provided by the path. + """ + config_path, return_map = kwargs['config_path'], kwargs['return_map'] + __TYPE_MAP.ts_config.load_termset_config(config_path) + + if return_map: # This helps with testing + return __TYPE_MAP + +def get_loaded_config(): + if __TYPE_MAP.ts_config.config is None: + msg = "No configuration is loaded." + raise ValueError(msg) + else: + return __TYPE_MAP.ts_config.config + +def unload_termset_config(): + """ + Remove validation. + """ + return __TYPE_MAP.ts_config.unload_termset_config() + # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, {'name': 'namespace', 'type': str, 'doc': 'the name of the namespace', 'default': CORE_NAMESPACE}, diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 143cc2553..df1b3a9eb 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -97,68 +97,61 @@ def _field_config(self, arg_name, val): to check for a defined TermSet associated with the field. If found, the value of the field will be wrapped with a TermSetWrapper. """ - # load termset configuartion file from global Config + # load termset configuration file from global Config try: + """ + Even though the path field in the configurator can be a list of paths, the config + itself is only one file. When a user loads custom configs, the config is appended/modified. + The modificiations are not written to file, avoiding permanent modifications. + """ configurator = self.type_map.ts_config - except AttributeError: # This is for containers that are not registered, e.g., testing classes. - return val - - if len(configurator.path)>0: CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) termset_config = configurator.config + except AttributeError: # This is for containers that are not registered, e.g., testing classes. + return val + # check to see that the namespace for the container is in the config + if self.namespace not in self.type_map.container_types: + msg = "%s not found within loaded configuration." % self.namespace + warn(msg) + return val + else: + # check to see that the container type is in the config under the namespace + config_namespace = termset_config['namespaces'][self.namespace] + object_name = self.__class__.__name__ - # check to see that the namespace for the container is in the config - if self.namespace not in self.type_map.container_types: - msg = "%s not found within loaded configuration." % self.namespace + if object_name not in config_namespace['data_types']: + msg = '%s not found within the configuration for %s' % (object_name, self.namespace) warn(msg) return val else: - # check to see that the container type is in the config under the namespace - config_namespace = termset_config['namespaces'][self.namespace] - object_name = self.__class__.__name__ - - if object_name not in config_namespace['data_types']: - msg = '%s not found within the configuration for %s' % (object_name, self.namespace) - warn(msg) - return val - else: - for attr in config_namespace['data_types'][object_name]: - - - obj_mapper = self.type_map.get_map(self) - # get the spec according to attr name in schema - # Note: this is the name for the field in the config - - spec = obj_mapper.get_attr_spec(attr) - - # In the case of dealing with datasets directly or not defined in the spec. - # (Data/VectorData/DynamicTable/etc) - if spec is None: - msg = "Spec not found for %s." % attr + for attr in config_namespace['data_types'][object_name]: + obj_mapper = self.type_map.get_map(self) + # get the spec according to attr name in schema + # Note: this is the name for the field in the config + + spec = obj_mapper.get_attr_spec(attr) + + # In the case of dealing with datasets directly or not defined in the spec. + # (Data/VectorData/DynamicTable/etc) + if spec is None: + msg = "Spec not found for %s." % attr + warn(msg) + return val + # From the spec, get the corresponding constructor name + else: + constr_name = obj_mapper.get_const_arg(spec) + termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) + termset = TermSet(term_schema_path=termset_path) + # If the val has been manually wrapped then skip checking the config for the attr + if type(val) == TermSetWrapper: + msg = "Field value already wrapped with TermSetWrapper." warn(msg) return val - # From the spec, get the corresponding constructor name else: - constr_name = obj_mapper.get_const_arg(spec) - if constr_name == arg_name: - termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) - termset = TermSet(term_schema_path=termset_path) - # If the val has been manually wrapped then skip checking the config for the attr - if type(val) == TermSetWrapper: - msg = "Field value already wrapped with TermSetWrapper." - warn(msg) - return val - else: - val = TermSetWrapper(value=val, termset=termset) - return val - else: - msg = "The name of the field does not match the field name in the schema. Field value not wrapped." - warn(msg) - return val - else: - return val - + val = TermSetWrapper(value=val, termset=termset) + return val + @classmethod def _getter(cls, field): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 21931f7ea..65f33d3c0 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,6 +5,7 @@ import warnings import numpy as np from .data_utils import append_data, extend_data +import yaml class TermSet: @@ -162,7 +163,6 @@ def __schemasheets_convert(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: - import yaml from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover @@ -307,7 +307,9 @@ def extend(self, arg): class TermSetConfigurator: """ - + This class allows users to toggle on/off a global configuration for defined data types. + When toggled on, every instance of a configuration file supported data type will be validated + according the corresponding TermSet. """ @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuration file.', 'default': None}) def __init__(self, **kwargs): @@ -337,9 +339,8 @@ def load_termset_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. """ - with open(config_path, 'r') as config: - termset_config = yaml.load(config) + termset_config = yaml.load(config, Loader=yaml.FullLoader) if self.config is None: # set the initial config/load after config has been unloaded self.config = termset_config if len(self.path)==0: # for loading after an unloaded config diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 6c24c2b5a..0fde826dd 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -5,7 +5,7 @@ import pandas as pd import unittest -from hdmf import Container +from hdmf import Container, Data from hdmf import TermSet, TermSetWrapper from hdmf.backends.hdf5 import H5DataIO, HDF5IO from hdmf.backends.hdf5.h5tools import H5_TEXT, H5PY_3 @@ -17,10 +17,14 @@ EnumData, DynamicTableRegion, get_manager, - SimpleMultiContainer + SimpleMultiContainer, + load_termset_config, + unload_termset_config, + get_loaded_config, + get_type_map ) from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file -from hdmf.utils import StrDataset +from hdmf.utils import StrDataset, popargs, docval from hdmf.data_utils import DataChunkIterator from tests.unit.helpers.utils import ( @@ -2767,3 +2771,42 @@ def test_columns(self): {'name': 'col3', 'description': '...'}, {'name': 'col4', 'description': '...'}) ) self.assertEqual(self.foo2.__columns__, self.foo3.__columns__) + + +class ExtensionContainer(Container): + __fields__ = ("description",) + + def __init__(self, **kwargs): + description, namespace, type_map = popargs('description', 'namespace', 'type_map', kwargs) + super().__init__(**kwargs) + self.namespace = namespace + self.type_map = type_map + self.description = description + + +class TestTermSetConfig(TestCase): + def setUp(self): + unload_termset_config() + self.tm = load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) + + def test_load_config(self): + config = get_loaded_config() + self.assertEqual(config, + {'namespaces': {'hdmf-common': {'version': '3.12.2', + 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}}}}}) + + def test_validate_with_config(self): + data = VectorData(name='foo', data=[0], description='Homo sapiens') + + def test_namespace_warn(self): + with self.assertWarns(Warning): + container = ExtensionContainer(name='foo', namespace='foo', type_map=self.tm, description='Homo sapiens') + + def test_container_type_warn(self): + with self.assertWarns(Warning): + container = ExtensionContainer(name='foo', namespace='hdmf-common', type_map=self.tm, description='Homo sapiens') + + def test_already_wrapped_warn(self): + with self.assertWarns(Warning): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + data = VectorData(name='foo', data=[0], description=TermSetWrapper(value='Homo sapiens', termset=terms)) diff --git a/tests/unit/hdmf_config.yaml b/tests/unit/hdmf_config.yaml new file mode 100644 index 000000000..375810d39 --- /dev/null +++ b/tests/unit/hdmf_config.yaml @@ -0,0 +1,6 @@ +namespaces: + hdmf-common: + version: 3.12.2 + data_types: + VectorData: + description: example_test_term_set.yaml From e979bf8d4799326c9d81cdf5bf4db319553ff187 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 23:53:35 +0000 Subject: [PATCH 25/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index df1b3a9eb..9b0c0c893 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -151,7 +151,7 @@ def _field_config(self, arg_name, val): else: val = TermSetWrapper(value=val, termset=termset) return val - + @classmethod def _getter(cls, field): From b8064041a4c19c247d081158d4c47c367f93b426 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 14:24:34 -0700 Subject: [PATCH 26/78] tests --- src/hdmf/container.py | 7 +++++-- tests/unit/common/test_table.py | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 9b0c0c893..ff2e7eed3 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -105,8 +105,11 @@ def _field_config(self, arg_name, val): The modificiations are not written to file, avoiding permanent modifications. """ configurator = self.type_map.ts_config - CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) - termset_config = configurator.config + if len(configurator.path)>0: + CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) + termset_config = configurator.config + else: + return val except AttributeError: # This is for containers that are not registered, e.g., testing classes. return val diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 0fde826dd..1342f8d9a 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -2786,9 +2786,15 @@ def __init__(self, **kwargs): class TestTermSetConfig(TestCase): def setUp(self): + if not LINKML_INSTALLED: + self.skipTest("optional LinkML module is not installed") + unload_termset_config() self.tm = load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) + def tearDown(self): + unload_termset_config() + def test_load_config(self): config = get_loaded_config() self.assertEqual(config, @@ -2810,3 +2816,5 @@ def test_already_wrapped_warn(self): with self.assertWarns(Warning): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') data = VectorData(name='foo', data=[0], description=TermSetWrapper(value='Homo sapiens', termset=terms)) + + unload_termset_config() From 901cb56bd0258c7f1fe7e4f2ea98c58daabad3f9 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 14:32:14 -0700 Subject: [PATCH 27/78] tests --- src/hdmf/term_set.py | 14 ++++++-------- tests/unit/common/test_table.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 65f33d3c0..92f2ec617 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,7 +5,6 @@ import warnings import numpy as np from .data_utils import append_data, extend_data -import yaml class TermSet: @@ -163,6 +162,7 @@ def __schemasheets_convert(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: + import yaml from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover @@ -262,13 +262,6 @@ def __getitem__(self, val): """ return self.__value[val] - # uncomment when DataChunkIterator objects can be wrapped by TermSet - # def __next__(self): - # """ - # Return the next item of a wrapped iterator. - # """ - # return self.__value.__next__() - # def __len__(self): return len(self.__value) @@ -313,6 +306,11 @@ class TermSetConfigurator: """ @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuration file.', 'default': None}) def __init__(self, **kwargs): + try: + import yaml + except ImportError: + msg = "Install yaml" + raise ValueError(msg) self.config = None if kwargs['path'] is None: self.path = [] diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 1342f8d9a..4083849e5 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -36,9 +36,10 @@ try: import linkml_runtime # noqa: F401 - LINKML_INSTALLED = True + import yaml + REQUIREMENTS_INSTALLED = True except ImportError: - LINKML_INSTALLED = False + REQUIREMENTS_INSTALLED = False class TestDynamicTable(TestCase): @@ -135,7 +136,7 @@ def test_constructor_all_columns_are_iterators(self): # now test that when we supply id's that the error goes away _ = DynamicTable(name="TestTable", description="", columns=[column], id=list(range(3))) - @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_add_col_validate(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') col1 = VectorData( @@ -154,7 +155,7 @@ def test_add_col_validate(self): expected_df.index.name = 'id' pd.testing.assert_frame_equal(species.to_dataframe(), expected_df) - @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_add_col_validate_bad_data(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') col1 = VectorData( @@ -169,7 +170,7 @@ def test_add_col_validate_bad_data(self): data=TermSetWrapper(value=['bad data'], termset=terms)) - @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_add_row_validate(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') col1 = VectorData( @@ -191,7 +192,7 @@ def test_add_row_validate(self): expected_df.index.name = 'id' pd.testing.assert_frame_equal(species.to_dataframe(), expected_df) - @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_add_row_validate_bad_data_one_col(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') col1 = VectorData( @@ -208,7 +209,7 @@ def test_add_row_validate_bad_data_one_col(self): with self.assertRaises(ValueError): species.add_row(Species_1='bad', Species_2='Ursus arctos horribilis') - @unittest.skipIf(not LINKML_INSTALLED, "optional LinkML module is not installed") + @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_add_row_validate_bad_data_all_col(self): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') col1 = VectorData( @@ -2786,7 +2787,7 @@ def __init__(self, **kwargs): class TestTermSetConfig(TestCase): def setUp(self): - if not LINKML_INSTALLED: + if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") unload_termset_config() From 809d511eeac5b47c6bc5a5a356ad6ef850a31b7e Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 14:32:47 -0700 Subject: [PATCH 28/78] tests --- src/hdmf/term_set.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 92f2ec617..a5ecaf095 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -306,11 +306,6 @@ class TermSetConfigurator: """ @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuration file.', 'default': None}) def __init__(self, **kwargs): - try: - import yaml - except ImportError: - msg = "Install yaml" - raise ValueError(msg) self.config = None if kwargs['path'] is None: self.path = [] From 6b99a8a35afa05e7a76b979702ac19957f6cd750 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 14:44:11 -0700 Subject: [PATCH 29/78] ruff --- src/hdmf/container.py | 1 - src/hdmf/term_set.py | 11 ++++++++--- tests/unit/common/test_table.py | 24 +++++++++++++++--------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index ff2e7eed3..d8882835e 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -143,7 +143,6 @@ def _field_config(self, arg_name, val): return val # From the spec, get the corresponding constructor name else: - constr_name = obj_mapper.get_const_arg(spec) termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) termset = TermSet(term_schema_path=termset_path) # If the val has been manually wrapped then skip checking the config for the attr diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index a5ecaf095..414656d21 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -332,6 +332,12 @@ def load_termset_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. """ + try: + import yaml + except ImportError: + msg = "Install yaml." + raise ValueError(msg) + with open(config_path, 'r') as config: termset_config = yaml.load(config, Loader=yaml.FullLoader) if self.config is None: # set the initial config/load after config has been unloaded @@ -347,14 +353,13 @@ def load_termset_config(self,config_path): if namespace not in self.config: # append namespace config if not present within self.config self.config['namespaces'][namespace] = termset_config['namespaces'][namespace] else: # check for any needed overrides within existing namespace configs - breakpoint() for data_type in termset_config['namespaces'][namespace]['data_types']: if data_type in self.config['namespaces'][namespace]['data_types']: replace_config = termset_config['namespaces'][namespace]['data_types'][data_type] self.config['namespaces'][namespace]['data_types'][data_type] = replace_config else: # append to config - new_data_type_config = termset_config['namespaces'][namespace]['data_types'][data_type] - self.config['namespaces'][namespace]['data_types'] = new_data_type_config + new_config = termset_config['namespaces'][namespace]['data_types'][data_type] + self.config['namespaces'][namespace]['data_types'] = new_config # append path to self.path self.path.append(config_path) diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index 4083849e5..34072c583 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -5,7 +5,7 @@ import pandas as pd import unittest -from hdmf import Container, Data +from hdmf import Container from hdmf import TermSet, TermSetWrapper from hdmf.backends.hdf5 import H5DataIO, HDF5IO from hdmf.backends.hdf5.h5tools import H5_TEXT, H5PY_3 @@ -20,11 +20,9 @@ SimpleMultiContainer, load_termset_config, unload_termset_config, - get_loaded_config, - get_type_map -) + get_loaded_config) from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file -from hdmf.utils import StrDataset, popargs, docval +from hdmf.utils import StrDataset, popargs from hdmf.data_utils import DataChunkIterator from tests.unit.helpers.utils import ( @@ -36,7 +34,6 @@ try: import linkml_runtime # noqa: F401 - import yaml REQUIREMENTS_INSTALLED = True except ImportError: REQUIREMENTS_INSTALLED = False @@ -2804,18 +2801,27 @@ def test_load_config(self): def test_validate_with_config(self): data = VectorData(name='foo', data=[0], description='Homo sapiens') + self.assertEqual(data.description.value, 'Homo sapiens') def test_namespace_warn(self): with self.assertWarns(Warning): - container = ExtensionContainer(name='foo', namespace='foo', type_map=self.tm, description='Homo sapiens') + ExtensionContainer(name='foo', + namespace='foo', + type_map=self.tm, + description='Homo sapiens') def test_container_type_warn(self): with self.assertWarns(Warning): - container = ExtensionContainer(name='foo', namespace='hdmf-common', type_map=self.tm, description='Homo sapiens') + ExtensionContainer(name='foo', + namespace='hdmf-common', + type_map=self.tm, + description='Homo sapiens') def test_already_wrapped_warn(self): with self.assertWarns(Warning): terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') - data = VectorData(name='foo', data=[0], description=TermSetWrapper(value='Homo sapiens', termset=terms)) + VectorData(name='foo', + data=[0], + description=TermSetWrapper(value='Homo sapiens', termset=terms)) unload_termset_config() From f38ca1d2f952cc6cbb68b757d3a33bc5756e6421 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 14:49:09 -0700 Subject: [PATCH 30/78] update --- src/hdmf/container.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index d8882835e..d0e0c33b0 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -141,9 +141,10 @@ def _field_config(self, arg_name, val): msg = "Spec not found for %s." % attr warn(msg) return val - # From the spec, get the corresponding constructor name else: - termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][attr]) + # From the spec, get the corresponding constructor name + constr_name = obj_mapper.get_const_arg(spec) + termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][constr_name]) termset = TermSet(term_schema_path=termset_path) # If the val has been manually wrapped then skip checking the config for the attr if type(val) == TermSetWrapper: From 57e7bf085cb973db1a9749dee33ba2f54fa79f3c Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 14:52:30 -0700 Subject: [PATCH 31/78] update --- src/hdmf/common/__init__.py | 2 +- src/hdmf/term_set.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 217f6cba1..f4137bbf5 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -20,7 +20,7 @@ # a global type map global __TYPE_MAP -@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}, +@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}, {'name': 'return_map', 'type': bool, 'doc': 'Bool to return type_map', 'default': False}, is_method=False) def load_termset_config(**kwargs): diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 414656d21..04619d8d0 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -327,7 +327,7 @@ def get_config(self, object_name, namespace): msg = '%s was not found within the configuration for that namespace.' % object_name raise ValueError(msg) - @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuartion file.'}) + @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}) def load_termset_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. From 2d32acf41df9993615132809d2a3fe879a384978 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 15:59:16 -0700 Subject: [PATCH 32/78] cov --- src/hdmf/term_set.py | 2 +- tests/unit/hdmf_config2.yaml | 13 +++++++ tests/unit/test_term_set.py | 57 +++++++++++++++++++++++++++- tests/unit/test_termset_config.py | 63 ------------------------------- 4 files changed, 69 insertions(+), 66 deletions(-) create mode 100644 tests/unit/hdmf_config2.yaml delete mode 100644 tests/unit/test_termset_config.py diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 04619d8d0..566d10df5 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -334,7 +334,7 @@ def load_termset_config(self,config_path): """ try: import yaml - except ImportError: + except ImportError: # pragma: no cover msg = "Install yaml." raise ValueError(msg) diff --git a/tests/unit/hdmf_config2.yaml b/tests/unit/hdmf_config2.yaml new file mode 100644 index 000000000..8970d81ac --- /dev/null +++ b/tests/unit/hdmf_config2.yaml @@ -0,0 +1,13 @@ +namespaces: + hdmf-common: + version: 3.12.2 + data_types: + Data: + description: example_test_term_set.yaml + EnumData: + description: example_test_term_set.yaml + namespace2: + version: 0 + data_types: + MythicData: + description: example_test_term_set.yaml diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index b4a469438..ffdd3a2fb 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -1,8 +1,8 @@ import os -from hdmf.term_set import TermSet, TermSetWrapper +from hdmf.term_set import TermSet, TermSetWrapper, TermSetConfigurator from hdmf.testing import TestCase, remove_test_file -from hdmf.common import VectorData +from hdmf.common import VectorData, unload_termset_config import numpy as np @@ -215,3 +215,56 @@ def test_wrapper_extend_error(self): data_obj = VectorData(name='species', description='...', data=self.wrapped_list) with self.assertRaises(ValueError): data_obj.extend(['bad_data']) + +class TestTermSetConfig(TestCase): + def setUp(self): + if not REQUIREMENTS_INSTALLED: + self.skipTest("optional LinkML module is not installed") + + def tearDown(self): + unload_termset_config() + + def test_config_path(self): + path = 'tests/unit/hdmf_config.yaml' + tc = TermSetConfigurator(path=path) + self.assertEqual(tc.path, [path]) + + def test_get_config(self): + path = 'tests/unit/hdmf_config.yaml' + tc = TermSetConfigurator(path=path) + self.assertEqual(tc.get_config('VectorData', 'hdmf-common'), + {'description': 'example_test_term_set.yaml'}) + + def test_get_config_namespace_error(self): + path = 'tests/unit/hdmf_config.yaml' + tc = TermSetConfigurator(path=path) + with self.assertRaises(ValueError): + tc.get_config('VectorData', 'hdmf-common11') + + def test_get_config_container_error(self): + path = 'tests/unit/hdmf_config.yaml' + tc = TermSetConfigurator(path=path) + with self.assertRaises(ValueError): + tc.get_config('VectorData11', 'hdmf-common') + + def test_already_loaded_path_error(self): + path = 'tests/unit/hdmf_config.yaml' + tc = TermSetConfigurator(path=path) + with self.assertRaises(ValueError): + tc.load_termset_config(config_path=path) + + def test_load_two_unique_configs(self): + path = 'tests/unit/hdmf_config.yaml' + path2 = 'tests/unit/hdmf_config2.yaml' + tc = TermSetConfigurator(path=path) + tc.load_termset_config(config_path=path2) + self.assertEqual(tc.path, [path, path2]) + + # def test_append_namespace(self): + # pass + # + # def test_replace_config(self): + # pass + # + # def test_append_config(self): + # pass diff --git a/tests/unit/test_termset_config.py b/tests/unit/test_termset_config.py deleted file mode 100644 index 18dcc6ca1..000000000 --- a/tests/unit/test_termset_config.py +++ /dev/null @@ -1,63 +0,0 @@ -# from hdmf.testing import TestCase -# from hdmf import load_termset_config, unload_termset_config -# from hdmf.term_set import TermSetConfigurator -# -# class TestConfig(TestCase): -# self.test_config = {} -# self.test_merged_extension_config = {} -# self.hdmf_config = {} -# -# def test_construct_config(self): -# # add asserts for self.path and self.config -# test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') -# self.assertEqual(test_config.path, ['tests/unit/test_config.yaml']) -# # self.assertEqual(test_config.config, None) -# -# def test_load_termset_config(self): -# test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') -# test_config.load_termset_config(path='tests/unit/test_extension_config.yaml') -# self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) -# # self.assertEqual(config.config, None) -# -# def test_unload_termset_config(self): -# test_config = TermSetConfigurator(path='tests/unit/test_config.yaml') -# test_config.unload_termset_config() -# self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) -# self.assertEqual(config.config, None) -# -# def test_unload_global_config(self): -# config = get_termset_config() -# unload_termset_config() -# self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) -# self.assertEqual(config.config, None) -# -# def test_load_global_config_reset(self): -# load_termset_config() -# self.assertEqual(config.path, ['src/hdmf/hdmf_config.yaml']) -# # self.assertEqual(config.config, None) -# -# def test_load_global_config_extension_config(self): -# load_termset_config() -# self.assertEqual(config.path, ['tests/unit/test_config.yaml', 'tests/unit/test_extension_config.yaml']) -# # self.assertEqual(config.config, None) -# -# def test_data(self): -# pass -# -# def test_dataset_not_in_spec(self): -# pass -# -# def test_attribute_not_in_spec(self): -# pass -# -# def test_attriute_in_spec(self): -# pass -# -# def test_dataset_in_spec(self): -# pass -# -# def test_data_type_not_in_namespace(self): -# pass -# -# def test_warn_not_wrapped(self): -# pass From 4fd01166dba428090923a97ce7533e74fcef571f Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 14 Mar 2024 16:24:51 -0700 Subject: [PATCH 33/78] tests --- src/hdmf/term_set.py | 4 ++-- tests/unit/hdmf_config2.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 566d10df5..77f04ba79 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -350,7 +350,7 @@ def load_termset_config(self,config_path): raise ValueError(msg) else: for namespace in termset_config['namespaces']: - if namespace not in self.config: # append namespace config if not present within self.config + if namespace not in self.config['namespaces']: # append namespace config if not present self.config['namespaces'][namespace] = termset_config['namespaces'][namespace] else: # check for any needed overrides within existing namespace configs for data_type in termset_config['namespaces'][namespace]['data_types']: @@ -359,7 +359,7 @@ def load_termset_config(self,config_path): self.config['namespaces'][namespace]['data_types'][data_type] = replace_config else: # append to config new_config = termset_config['namespaces'][namespace]['data_types'][data_type] - self.config['namespaces'][namespace]['data_types'] = new_config + self.config['namespaces'][namespace]['data_types'][data_type] = new_config # append path to self.path self.path.append(config_path) diff --git a/tests/unit/hdmf_config2.yaml b/tests/unit/hdmf_config2.yaml index 8970d81ac..adcc54a94 100644 --- a/tests/unit/hdmf_config2.yaml +++ b/tests/unit/hdmf_config2.yaml @@ -6,6 +6,8 @@ namespaces: description: example_test_term_set.yaml EnumData: description: example_test_term_set.yaml + VectorData: + description: ... namespace2: version: 0 data_types: From 39ea8c27d3a17ad44189cddda5386fe48d0ab27f Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Fri, 15 Mar 2024 10:10:42 -0700 Subject: [PATCH 34/78] tests/clean --- tests/unit/common/test_table.py | 63 +------------------------ tests/unit/test_term_set.py | 83 +++++++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 71 deletions(-) diff --git a/tests/unit/common/test_table.py b/tests/unit/common/test_table.py index b5877953b..f2d03332f 100644 --- a/tests/unit/common/test_table.py +++ b/tests/unit/common/test_table.py @@ -17,12 +17,9 @@ EnumData, DynamicTableRegion, get_manager, - SimpleMultiContainer, - load_termset_config, - unload_termset_config, - get_loaded_config) + SimpleMultiContainer) from hdmf.testing import TestCase, H5RoundTripMixin, remove_test_file -from hdmf.utils import StrDataset, popargs +from hdmf.utils import StrDataset from hdmf.data_utils import DataChunkIterator from tests.unit.helpers.utils import ( @@ -2837,59 +2834,3 @@ def test_columns(self): {'name': 'col3', 'description': '...'}, {'name': 'col4', 'description': '...'}) ) self.assertEqual(self.foo2.__columns__, self.foo3.__columns__) - - -class ExtensionContainer(Container): - __fields__ = ("description",) - - def __init__(self, **kwargs): - description, namespace, type_map = popargs('description', 'namespace', 'type_map', kwargs) - super().__init__(**kwargs) - self.namespace = namespace - self.type_map = type_map - self.description = description - - -class TestTermSetConfig(TestCase): - def setUp(self): - if not REQUIREMENTS_INSTALLED: - self.skipTest("optional LinkML module is not installed") - - unload_termset_config() - self.tm = load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) - - def tearDown(self): - unload_termset_config() - - def test_load_config(self): - config = get_loaded_config() - self.assertEqual(config, - {'namespaces': {'hdmf-common': {'version': '3.12.2', - 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}}}}}) - - def test_validate_with_config(self): - data = VectorData(name='foo', data=[0], description='Homo sapiens') - self.assertEqual(data.description.value, 'Homo sapiens') - - def test_namespace_warn(self): - with self.assertWarns(Warning): - ExtensionContainer(name='foo', - namespace='foo', - type_map=self.tm, - description='Homo sapiens') - - def test_container_type_warn(self): - with self.assertWarns(Warning): - ExtensionContainer(name='foo', - namespace='hdmf-common', - type_map=self.tm, - description='Homo sapiens') - - def test_already_wrapped_warn(self): - with self.assertWarns(Warning): - terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') - VectorData(name='foo', - data=[0], - description=TermSetWrapper(value='Homo sapiens', termset=terms)) - - unload_termset_config() diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index ffdd3a2fb..0ae5af730 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -1,9 +1,11 @@ import os +import numpy as np +from hdmf import Container from hdmf.term_set import TermSet, TermSetWrapper, TermSetConfigurator from hdmf.testing import TestCase, remove_test_file -from hdmf.common import VectorData, unload_termset_config -import numpy as np +from hdmf.common import VectorData, unload_termset_config, get_loaded_config, load_termset_config +from hdmf.utils import popargs CUR_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -224,6 +226,20 @@ def setUp(self): def tearDown(self): unload_termset_config() + def test_get_loaded_config(self): + load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) + loaded_config = get_loaded_config() + config = {'namespaces': + {'hdmf-common': + {'version': '3.12.2', + 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}}}}} + + self.assertEqual(loaded_config, config) + + def test_get_loaded_config_error(self): + with self.assertRaises(ValueError): + get_loaded_config() + def test_config_path(self): path = 'tests/unit/hdmf_config.yaml' tc = TermSetConfigurator(path=path) @@ -260,11 +276,58 @@ def test_load_two_unique_configs(self): tc.load_termset_config(config_path=path2) self.assertEqual(tc.path, [path, path2]) - # def test_append_namespace(self): - # pass - # - # def test_replace_config(self): - # pass - # - # def test_append_config(self): - # pass + +class ExtensionContainer(Container): + __fields__ = ("description",) + + def __init__(self, **kwargs): + description, namespace, type_map = popargs('description', 'namespace', 'type_map', kwargs) + super().__init__(**kwargs) + self.namespace = namespace + self.type_map = type_map + self.description = description + + +class TestTermSetConfigVectorData(TestCase): + def setUp(self): + if not REQUIREMENTS_INSTALLED: + self.skipTest("optional LinkML module is not installed") + + unload_termset_config() + self.tm = load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) + + def tearDown(self): + unload_termset_config() + + def test_load_config(self): + config = get_loaded_config() + self.assertEqual(config, + {'namespaces': {'hdmf-common': {'version': '3.12.2', + 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}}}}}) + + def test_validate_with_config(self): + data = VectorData(name='foo', data=[0], description='Homo sapiens') + self.assertEqual(data.description.value, 'Homo sapiens') + + def test_namespace_warn(self): + with self.assertWarns(Warning): + ExtensionContainer(name='foo', + namespace='foo', + type_map=self.tm, + description='Homo sapiens') + + def test_container_type_warn(self): + with self.assertWarns(Warning): + ExtensionContainer(name='foo', + namespace='hdmf-common', + type_map=self.tm, + description='Homo sapiens') + + def test_already_wrapped_warn(self): + with self.assertWarns(Warning): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') + VectorData(name='foo', + data=[0], + description=TermSetWrapper(value='Homo sapiens', termset=terms)) + + unload_termset_config() From 0602a530a66c54d29777b22fbf12b6906ce6f183 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Fri, 15 Mar 2024 10:45:50 -0700 Subject: [PATCH 35/78] coverage' git push --- tests/unit/hdmf_config.yaml | 2 ++ tests/unit/test_term_set.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/unit/hdmf_config.yaml b/tests/unit/hdmf_config.yaml index 375810d39..422917f95 100644 --- a/tests/unit/hdmf_config.yaml +++ b/tests/unit/hdmf_config.yaml @@ -4,3 +4,5 @@ namespaces: data_types: VectorData: description: example_test_term_set.yaml + VectorIndex: + data: ... diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 0ae5af730..7345018bd 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -4,7 +4,7 @@ from hdmf import Container from hdmf.term_set import TermSet, TermSetWrapper, TermSetConfigurator from hdmf.testing import TestCase, remove_test_file -from hdmf.common import VectorData, unload_termset_config, get_loaded_config, load_termset_config +from hdmf.common import VectorIndex, VectorData, unload_termset_config, get_loaded_config, load_termset_config from hdmf.utils import popargs @@ -293,7 +293,6 @@ def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") - unload_termset_config() self.tm = load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) def tearDown(self): @@ -331,3 +330,12 @@ def test_already_wrapped_warn(self): description=TermSetWrapper(value='Homo sapiens', termset=terms)) unload_termset_config() + + def test_warn_field_not_in_spec(self): + with self.assertWarns(Warning): + col1 = VectorData(name='col1', + description='Homo sapiens', + data=['1a', '1b', '1c', '2a']) + VectorIndex(name='col1_index', + target=col1, + data=[3, 4]) From d66145d4b22f88d1de3bd4b27213177ef6953928 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Fri, 15 Mar 2024 10:57:59 -0700 Subject: [PATCH 36/78] coverage' git push --- tests/unit/test_term_set.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 7345018bd..71882a55d 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -229,10 +229,9 @@ def tearDown(self): def test_get_loaded_config(self): load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) loaded_config = get_loaded_config() - config = {'namespaces': - {'hdmf-common': - {'version': '3.12.2', - 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}}}}} + config = {'namespaces': {'hdmf-common': {'version': '3.12.2', + 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}, + 'VectorIndex': {'data': '...'}}}}} self.assertEqual(loaded_config, config) @@ -302,7 +301,8 @@ def test_load_config(self): config = get_loaded_config() self.assertEqual(config, {'namespaces': {'hdmf-common': {'version': '3.12.2', - 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}}}}}) + 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}, + 'VectorIndex': {'data': '...'}}}}}) def test_validate_with_config(self): data = VectorData(name='foo', data=[0], description='Homo sapiens') From eb3970b387d28f99205f80008434e044d0674014 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Fri, 15 Mar 2024 11:41:07 -0700 Subject: [PATCH 37/78] final clean ups --- src/hdmf/common/__init__.py | 5 ++++- src/hdmf/container.py | 16 +++++++++------- src/hdmf/term_set.py | 13 ++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index f4137bbf5..45818ad84 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -34,6 +34,9 @@ def load_termset_config(**kwargs): return __TYPE_MAP def get_loaded_config(): + """ + This method returns the entire config file. + """ if __TYPE_MAP.ts_config.config is None: msg = "No configuration is loaded." raise ValueError(msg) @@ -42,7 +45,7 @@ def get_loaded_config(): def unload_termset_config(): """ - Remove validation. + Unload the configuration file. """ return __TYPE_MAP.ts_config.unload_termset_config() diff --git a/src/hdmf/container.py b/src/hdmf/container.py index d0e0c33b0..ce7545dbc 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -96,21 +96,23 @@ def _field_config(self, arg_name, val): This method will be called in the setter. The termset configuration will be used (if loaded) to check for a defined TermSet associated with the field. If found, the value of the field will be wrapped with a TermSetWrapper. + + Even though the path field in the configurator can be a list of paths, the config + itself is only one file. When a user loads custom configs, the config is appended/modified. + The modificiations are not written to file, avoiding permanent modifications. """ # load termset configuration file from global Config try: - """ - Even though the path field in the configurator can be a list of paths, the config - itself is only one file. When a user loads custom configs, the config is appended/modified. - The modificiations are not written to file, avoiding permanent modifications. - """ configurator = self.type_map.ts_config if len(configurator.path)>0: + # The type_map has a config always set; however, when toggled off, the config path path is empty. CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) termset_config = configurator.config else: return val - except AttributeError: # This is for containers that are not registered, e.g., testing classes. + except AttributeError: + # These are for classes that have not been registered, e.g., some test containers. + # Without register_class, the type_map parameter is not defined. return val # check to see that the namespace for the container is in the config @@ -130,9 +132,9 @@ def _field_config(self, arg_name, val): else: for attr in config_namespace['data_types'][object_name]: obj_mapper = self.type_map.get_map(self) + # get the spec according to attr name in schema # Note: this is the name for the field in the config - spec = obj_mapper.get_attr_spec(attr) # In the case of dealing with datasets directly or not defined in the spec. diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 77f04ba79..e13f80707 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -313,7 +313,14 @@ def __init__(self, **kwargs): self.path = [kwargs['path']] self.load_termset_config(config_path=self.path[0]) - def get_config(self, object_name, namespace): + @docval({'name': 'data_type', 'type': str, + 'doc': 'The desired data type within the configuration file.'}, + {'name': 'namespace', 'type': str, + 'doc': 'The namespace for the data type. '}) + def get_config(self, data_type, namespace): + """ + Return the config for that data type in the given namespace. + """ try: namespace_config = self.config['namespaces'][namespace] except KeyError: @@ -321,10 +328,10 @@ def get_config(self, object_name, namespace): raise ValueError(msg) try: - type_config = namespace_config['data_types'][object_name] + type_config = namespace_config['data_types'][data_type] return type_config except KeyError: - msg = '%s was not found within the configuration for that namespace.' % object_name + msg = '%s was not found within the configuration for that namespace.' % data_type raise ValueError(msg) @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}) From e9e26f6d9510ace74f98d43ffeff052b5763ada8 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Fri, 15 Mar 2024 11:43:52 -0700 Subject: [PATCH 38/78] final clean ups --- src/hdmf/build/manager.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index c3189756f..59efe1092 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -397,10 +397,9 @@ class TypeMap: """ @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None}, - {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}, - {'name': 'config_path', 'type': str, 'doc': 'The path to the TermSet config yaml.', 'default': None}) + {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}) def __init__(self, **kwargs): - namespaces, mapper_cls, config_path = getargs('namespaces', 'mapper_cls', 'config_path', kwargs) + namespaces, mapper_cls = getargs('namespaces', 'mapper_cls', kwargs) if namespaces is None: namespaces = NamespaceCatalog() if mapper_cls is None: @@ -413,7 +412,7 @@ def __init__(self, **kwargs): self.__data_types = dict() self.__default_mapper_cls = mapper_cls self.__class_generator = ClassGenerator() - self.ts_config = TermSetConfigurator(path=config_path) + self.ts_config = TermSetConfigurator() self.register_generator(CustomClassGenerator) self.register_generator(MCIClassGenerator) From f6d5a558ff9c58f1b3fc30de5c9b426474936358 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 15 Mar 2024 12:22:07 -0700 Subject: [PATCH 39/78] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eddf8270..d4c43cf4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## HDMF 3.13.0 (Upcoming) ### Enhancements +- Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` accordding to a configuration file. @mavaylon1 [1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Added docs page that lists limitations of support for the HDMF specification language. @rly [#1069](https://github.com/hdmf-dev/hdmf/pull/1069) - Added warning when using `add_row` or `add_column` to add a ragged array to `DynamicTable` without an index parameter. @stephprince [#1066](https://github.com/hdmf-dev/hdmf/pull/1066) From 626a316b440a430371888b455e820902883c9143 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 15 Mar 2024 12:23:53 -0700 Subject: [PATCH 40/78] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4c43cf4b..6939138b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## HDMF 3.13.0 (Upcoming) ### Enhancements -- Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` accordding to a configuration file. @mavaylon1 [1016](https://github.com/hdmf-dev/hdmf/pull/1016) +- Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Added docs page that lists limitations of support for the HDMF specification language. @rly [#1069](https://github.com/hdmf-dev/hdmf/pull/1069) - Added warning when using `add_row` or `add_column` to add a ragged array to `DynamicTable` without an index parameter. @stephprince [#1066](https://github.com/hdmf-dev/hdmf/pull/1066) From 9728c664a3677a388d0af48420b08da058b8ab82 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Mon, 18 Mar 2024 19:33:12 -0700 Subject: [PATCH 41/78] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6939138b3..1101206d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## HDMF 3.13.0 (Upcoming) ### Enhancements -- Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [1016](https://github.com/hdmf-dev/hdmf/pull/1016) +- Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Added docs page that lists limitations of support for the HDMF specification language. @rly [#1069](https://github.com/hdmf-dev/hdmf/pull/1069) - Added warning when using `add_row` or `add_column` to add a ragged array to `DynamicTable` without an index parameter. @stephprince [#1066](https://github.com/hdmf-dev/hdmf/pull/1066) From e654e3d72b8b8047e92cfa7130540cd6a7b3f5bb Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 19 Mar 2024 00:09:47 -0700 Subject: [PATCH 42/78] Update src/hdmf/container.py --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index ce7545dbc..962130277 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -99,7 +99,7 @@ def _field_config(self, arg_name, val): Even though the path field in the configurator can be a list of paths, the config itself is only one file. When a user loads custom configs, the config is appended/modified. - The modificiations are not written to file, avoiding permanent modifications. + The modifications are not written to file, avoiding permanent modifications. """ # load termset configuration file from global Config try: From ddd66ab032d5d68d39d4b255fa44ba02e0b4892f Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 19 Mar 2024 00:10:38 -0700 Subject: [PATCH 43/78] Update src/hdmf/container.py --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 962130277..708cfea80 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -105,7 +105,7 @@ def _field_config(self, arg_name, val): try: configurator = self.type_map.ts_config if len(configurator.path)>0: - # The type_map has a config always set; however, when toggled off, the config path path is empty. + # The type_map has a config always set; however, when toggled off, the config path is empty. CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) termset_config = configurator.config else: From 5518adcc303364dafc873f393df0b89193b8b223 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 19 Mar 2024 00:31:30 -0700 Subject: [PATCH 44/78] Update src/hdmf/term_set.py --- src/hdmf/term_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index e13f80707..1405ea739 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -302,7 +302,7 @@ class TermSetConfigurator: """ This class allows users to toggle on/off a global configuration for defined data types. When toggled on, every instance of a configuration file supported data type will be validated - according the corresponding TermSet. + according to the corresponding TermSet. """ @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuration file.', 'default': None}) def __init__(self, **kwargs): From 790df9b29781f7cf311c996943b3b321b4332158 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 19 Mar 2024 00:32:19 -0700 Subject: [PATCH 45/78] Update src/hdmf/term_set.py --- src/hdmf/term_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 1405ea739..db19f4045 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -316,7 +316,7 @@ def __init__(self, **kwargs): @docval({'name': 'data_type', 'type': str, 'doc': 'The desired data type within the configuration file.'}, {'name': 'namespace', 'type': str, - 'doc': 'The namespace for the data type. '}) + 'doc': 'The namespace for the data type.'}) def get_config(self, data_type, namespace): """ Return the config for that data type in the given namespace. From 4f617cbcb30c8cef05948d4641789f18d14a8c1e Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 19 Mar 2024 07:02:41 -0700 Subject: [PATCH 46/78] in progress --- src/hdmf/build/manager.py | 2 ++ src/hdmf/common/__init__.py | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 59efe1092..3398cf007 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -5,6 +5,7 @@ from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator from ..container import AbstractContainer, Container, Data +from ..common import get_type_map() from ..term_set import TermSetConfigurator from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog from ..spec.spec import BaseStorageSpec @@ -734,6 +735,7 @@ def register_container_type(self, **kwargs): self.__container_types.setdefault(namespace, dict()) self.__container_types[namespace][data_type] = container_cls self.__data_types.setdefault(container_cls, (namespace, data_type)) + container_cls.__type_map = get_type_map() if not isinstance(container_cls, TypeSource): setattr(container_cls, spec.type_key(), data_type) setattr(container_cls, 'namespace', namespace) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 45818ad84..2cbedf50e 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -65,14 +65,14 @@ def register_class(**kwargs): def _dec(cls): _set_exp(cls) __TYPE_MAP.register_container_type(namespace, data_type, cls) - cls.type_map = __TYPE_MAP - cls.namespace = namespace + # cls.type_map = __TYPE_MAP + # cls.namespace = namespace return cls else: def _dec(cls): __TYPE_MAP.register_container_type(namespace, data_type, cls) - cls.type_map = __TYPE_MAP - cls.namespace = namespace + # cls.type_map = __TYPE_MAP + # cls.namespace = namespace return cls if container_cls is None: @@ -183,6 +183,9 @@ def get_type_map(**kwargs): type_map.load_namespaces(extensions) elif isinstance(extensions, TypeMap): type_map.merge(extensions) + global_config = __TYPE_MAP.ts_config + if global_config is not None: + type_map.ts_config = global_config return type_map From 08766d3e459370bb44d247e443da83f550723c20 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Tue, 19 Mar 2024 07:03:02 -0700 Subject: [PATCH 47/78] Update src/hdmf/container.py Co-authored-by: Ryan Ly --- src/hdmf/container.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 708cfea80..e88bd417b 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -144,16 +144,16 @@ def _field_config(self, arg_name, val): warn(msg) return val else: - # From the spec, get the corresponding constructor name - constr_name = obj_mapper.get_const_arg(spec) - termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][constr_name]) - termset = TermSet(term_schema_path=termset_path) # If the val has been manually wrapped then skip checking the config for the attr if type(val) == TermSetWrapper: msg = "Field value already wrapped with TermSetWrapper." warn(msg) return val else: + # From the spec, get the corresponding constructor name + constr_name = obj_mapper.get_const_arg(spec) + termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][constr_name]) + termset = TermSet(term_schema_path=termset_path) val = TermSetWrapper(value=val, termset=termset) return val From 465b2a5343a331d071eb70896a37cd1b0afb6560 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Tue, 19 Mar 2024 07:21:56 -0700 Subject: [PATCH 48/78] Update tests/unit/test_term_set.py Co-authored-by: Ryan Ly --- tests/unit/test_term_set.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 71882a55d..f89d8f84c 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -332,10 +332,10 @@ def test_already_wrapped_warn(self): unload_termset_config() def test_warn_field_not_in_spec(self): + col1 = VectorData(name='col1', + description='Homo sapiens', + data=['1a', '1b', '1c', '2a']) with self.assertWarns(Warning): - col1 = VectorData(name='col1', - description='Homo sapiens', - data=['1a', '1b', '1c', '2a']) VectorIndex(name='col1_index', target=col1, data=[3, 4]) From 9c6a451c99ae3e649c682b10ee6d3ac6b869725b Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Tue, 19 Mar 2024 07:22:13 -0700 Subject: [PATCH 49/78] Update tests/unit/test_term_set.py Co-authored-by: Ryan Ly --- tests/unit/test_term_set.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index f89d8f84c..922b391e2 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -323,8 +323,8 @@ def test_container_type_warn(self): description='Homo sapiens') def test_already_wrapped_warn(self): + terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') with self.assertWarns(Warning): - terms = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') VectorData(name='foo', data=[0], description=TermSetWrapper(value='Homo sapiens', termset=terms)) From bdb04102a8aa3cf4d12c1146cf3c35df86f5ddb9 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 19 Mar 2024 07:23:47 -0700 Subject: [PATCH 50/78] in progress --- src/hdmf/build/manager.py | 4 ++-- src/hdmf/common/__init__.py | 5 +---- tests/unit/test_config.yaml | 4 ++-- tests/unit/test_term_set.py | 7 ++++--- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 3398cf007..33183243f 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -5,7 +5,6 @@ from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator from ..container import AbstractContainer, Container, Data -from ..common import get_type_map() from ..term_set import TermSetConfigurator from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog from ..spec.spec import BaseStorageSpec @@ -735,7 +734,8 @@ def register_container_type(self, **kwargs): self.__container_types.setdefault(namespace, dict()) self.__container_types[namespace][data_type] = container_cls self.__data_types.setdefault(container_cls, (namespace, data_type)) - container_cls.__type_map = get_type_map() + # from hdmf.common import get_type_map # circular import + # container_cls.type_map = get_type_map() if not isinstance(container_cls, TypeSource): setattr(container_cls, spec.type_key(), data_type) setattr(container_cls, 'namespace', namespace) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 2cbedf50e..cc4ba0e69 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -21,7 +21,6 @@ global __TYPE_MAP @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}, - {'name': 'return_map', 'type': bool, 'doc': 'Bool to return type_map', 'default': False}, is_method=False) def load_termset_config(**kwargs): """ @@ -30,9 +29,6 @@ def load_termset_config(**kwargs): config_path, return_map = kwargs['config_path'], kwargs['return_map'] __TYPE_MAP.ts_config.load_termset_config(config_path) - if return_map: # This helps with testing - return __TYPE_MAP - def get_loaded_config(): """ This method returns the entire config file. @@ -78,6 +74,7 @@ def _dec(cls): if container_cls is None: return _dec else: + container_cls.type_map = get_type_map() _dec(container_cls) diff --git a/tests/unit/test_config.yaml b/tests/unit/test_config.yaml index 0462131ef..f02cca71e 100644 --- a/tests/unit/test_config.yaml +++ b/tests/unit/test_config.yaml @@ -3,7 +3,7 @@ namespaces: version: TBD data_types: VectorData: - data: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' + data: 'tests/unit/example_test_term_set.yaml' field2: '...' type2: field1: '...' @@ -11,5 +11,5 @@ namespaces: version: TBD data_types: type1: - field1: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' + field1: 'tests/unit/example_test_term_set.yaml' field2: '...' diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 922b391e2..8cc02a83b 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -4,7 +4,8 @@ from hdmf import Container from hdmf.term_set import TermSet, TermSetWrapper, TermSetConfigurator from hdmf.testing import TestCase, remove_test_file -from hdmf.common import VectorIndex, VectorData, unload_termset_config, get_loaded_config, load_termset_config +from hdmf.common import (VectorIndex, VectorData, unload_termset_config, get_loaded_config, load_termset_config, + get_type_map) from hdmf.utils import popargs @@ -227,7 +228,7 @@ def tearDown(self): unload_termset_config() def test_get_loaded_config(self): - load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) + load_termset_config(config_path='tests/unit/hdmf_config.yaml') loaded_config = get_loaded_config() config = {'namespaces': {'hdmf-common': {'version': '3.12.2', 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}, @@ -292,7 +293,7 @@ def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") - self.tm = load_termset_config(config_path='tests/unit/hdmf_config.yaml', return_map=True) + self.tm = get_type_map() def tearDown(self): unload_termset_config() From be1752dab8289e951cd78290b3aeb5f9de409fd3 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 19 Mar 2024 07:25:02 -0700 Subject: [PATCH 51/78] in progress --- tests/unit/test_config.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 tests/unit/test_config.yaml diff --git a/tests/unit/test_config.yaml b/tests/unit/test_config.yaml deleted file mode 100644 index f02cca71e..000000000 --- a/tests/unit/test_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -namespaces: - hdmf-common: - version: TBD - data_types: - VectorData: - data: 'tests/unit/example_test_term_set.yaml' - field2: '...' - type2: - field1: '...' - hdmf-experimental: - version: TBD - data_types: - type1: - field1: 'tests/unit/example_test_term_set.yaml' - field2: '...' From 46c8f9b61eb1c3950531f8a266629f9899c6f41d Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 19 Mar 2024 07:53:50 -0700 Subject: [PATCH 52/78] in progress --- src/hdmf/common/__init__.py | 8 ++++---- src/hdmf/container.py | 7 ++++--- src/hdmf/term_set.py | 1 + tests/unit/test_term_set.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index cc4ba0e69..870c3f00c 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -26,7 +26,7 @@ def load_termset_config(**kwargs): """ This method will either load the default config or the config provided by the path. """ - config_path, return_map = kwargs['config_path'], kwargs['return_map'] + config_path = kwargs['config_path'] __TYPE_MAP.ts_config.load_termset_config(config_path) def get_loaded_config(): @@ -61,20 +61,20 @@ def register_class(**kwargs): def _dec(cls): _set_exp(cls) __TYPE_MAP.register_container_type(namespace, data_type, cls) - # cls.type_map = __TYPE_MAP + cls.type_map = __TYPE_MAP # cls.namespace = namespace return cls else: def _dec(cls): __TYPE_MAP.register_container_type(namespace, data_type, cls) - # cls.type_map = __TYPE_MAP + cls.type_map = __TYPE_MAP # cls.namespace = namespace return cls if container_cls is None: return _dec else: - container_cls.type_map = get_type_map() + # container_cls.type_map = get_type_map() _dec(container_cls) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index e88bd417b..9887d3f34 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -150,9 +150,10 @@ def _field_config(self, arg_name, val): warn(msg) return val else: - # From the spec, get the corresponding constructor name - constr_name = obj_mapper.get_const_arg(spec) - termset_path = os.path.join(CUR_DIR, config_namespace['data_types'][object_name][constr_name]) + # From the spec, get the mapped attribute name + mapped_attr_name = obj_mapper.get_attribute(spec) + termset_path = os.path.join(CUR_DIR, + config_namespace['data_types'][object_name][mapped_attr_name]) termset = TermSet(term_schema_path=termset_path) val = TermSetWrapper(value=val, termset=termset) return val diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index db19f4045..d17c15e56 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -361,6 +361,7 @@ def load_termset_config(self,config_path): self.config['namespaces'][namespace] = termset_config['namespaces'][namespace] else: # check for any needed overrides within existing namespace configs for data_type in termset_config['namespaces'][namespace]['data_types']: + # NOTE: these two branches effectively do the same thing, but are split for clarity. if data_type in self.config['namespaces'][namespace]['data_types']: replace_config = termset_config['namespaces'][namespace]['data_types'][data_type] self.config['namespaces'][namespace]['data_types'][data_type] = replace_config diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 8cc02a83b..d1634eb9b 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -292,7 +292,7 @@ class TestTermSetConfigVectorData(TestCase): def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") - + load_termset_config(config_path='tests/unit/hdmf_config.yaml') self.tm = get_type_map() def tearDown(self): From 3574bd043506ccb5f0c91effad3878df45d649d0 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 19 Mar 2024 08:05:39 -0700 Subject: [PATCH 53/78] in progress --- src/hdmf/container.py | 16 +++++++++++----- tests/unit/test_container.py | 5 +++++ tests/unit/test_term_set.py | 8 ++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 9887d3f34..9f00ce7d0 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -90,6 +90,12 @@ def setter(self, val): return setter + def get_data_type(self): + """ + Method that return data type. + """ + return getattr(self, self._data_type_attr) + def _field_config(self, arg_name, val): """ @@ -123,14 +129,14 @@ def _field_config(self, arg_name, val): else: # check to see that the container type is in the config under the namespace config_namespace = termset_config['namespaces'][self.namespace] - object_name = self.__class__.__name__ + data_type = getattr(self, self._data_type_attr) - if object_name not in config_namespace['data_types']: - msg = '%s not found within the configuration for %s' % (object_name, self.namespace) + if data_type not in config_namespace['data_types']: + msg = '%s not found within the configuration for %s' % (data_type, self.namespace) warn(msg) return val else: - for attr in config_namespace['data_types'][object_name]: + for attr in config_namespace['data_types'][data_type]: obj_mapper = self.type_map.get_map(self) # get the spec according to attr name in schema @@ -153,7 +159,7 @@ def _field_config(self, arg_name, val): # From the spec, get the mapped attribute name mapped_attr_name = obj_mapper.get_attribute(spec) termset_path = os.path.join(CUR_DIR, - config_namespace['data_types'][object_name][mapped_attr_name]) + config_namespace['data_types'][data_type][mapped_attr_name]) termset = TermSet(term_schema_path=termset_path) val = TermSetWrapper(value=val, termset=termset) return val diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index b5a2d87e8..5c9948222 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -58,6 +58,11 @@ def test_new(self): self.assertFalse(child_obj._in_construct_mode) self.assertTrue(child_obj.modified) + def test_get_data_type(self): + obj = Container('obj1') + dt = obj.get_data_type() + self.assertEqual(dt, 'Container') + def test_new_object_id_none(self): """Test that passing object_id=None to __new__ is OK and results in a non-None object ID being assigned. """ diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index d1634eb9b..4ca24019f 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -274,7 +274,15 @@ def test_load_two_unique_configs(self): path2 = 'tests/unit/hdmf_config2.yaml' tc = TermSetConfigurator(path=path) tc.load_termset_config(config_path=path2) + config = {'namespaces': {'hdmf-common': {'version': '3.12.2', + 'data_types': {'VectorData': {'description': '...'}, + 'VectorIndex': {'data': '...'}, + 'Data': {'description': 'example_test_term_set.yaml'}, + 'EnumData': {'description': 'example_test_term_set.yaml'}}}, + 'namespace2': {'version': 0, + 'data_types': {'MythicData': {'description': 'example_test_term_set.yaml'}}}}} self.assertEqual(tc.path, [path, path2]) + self.assertEqual(tc.config, config) class ExtensionContainer(Container): From 207683b845970a1d985227ed0ed867e2bdb7cf58 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Tue, 19 Mar 2024 08:09:49 -0700 Subject: [PATCH 54/78] clean tests --- tests/unit/test_term_set.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 4ca24019f..b646f9944 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -289,10 +289,10 @@ class ExtensionContainer(Container): __fields__ = ("description",) def __init__(self, **kwargs): - description, namespace, type_map = popargs('description', 'namespace', 'type_map', kwargs) + description, namespace = popargs('description', 'namespace', kwargs) super().__init__(**kwargs) self.namespace = namespace - self.type_map = type_map + self.type_map = get_type_map() self.description = description @@ -301,7 +301,6 @@ def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") load_termset_config(config_path='tests/unit/hdmf_config.yaml') - self.tm = get_type_map() def tearDown(self): unload_termset_config() @@ -321,14 +320,12 @@ def test_namespace_warn(self): with self.assertWarns(Warning): ExtensionContainer(name='foo', namespace='foo', - type_map=self.tm, description='Homo sapiens') def test_container_type_warn(self): with self.assertWarns(Warning): ExtensionContainer(name='foo', namespace='hdmf-common', - type_map=self.tm, description='Homo sapiens') def test_already_wrapped_warn(self): From 8d4458893996f6fdcc91ed7e7540af7fd4abc1cb Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 14:35:31 -0700 Subject: [PATCH 55/78] checkpoint of updates --- requirements-opt.txt | 7 +-- src/hdmf/__init__.py | 2 +- src/hdmf/build/manager.py | 4 +- src/hdmf/common/hdmf-common-schema | 2 +- src/hdmf/container.py | 2 +- src/hdmf/term_set.py | 11 ++-- tests/unit/hdmf_config.yaml | 3 +- tests/unit/hdmf_config2.yaml | 9 ++- tests/unit/test_term_set.py | 26 ++++----- .../schemasheets/nwb_static_enums.yaml | 58 +++++++------------ 10 files changed, 57 insertions(+), 67 deletions(-) diff --git a/requirements-opt.txt b/requirements-opt.txt index 644fc80be..3bb8833da 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -1,8 +1,7 @@ # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF tqdm==4.66.1 zarr==2.16.1 -linkml-runtime==1.6.0; python_version >= "3.9" -schemasheets==0.1.24; python_version >= "3.9" -oaklib==0.5.20; python_version >= "3.9" -pydantic==1.10.13 # linkml-runtime 1.6.0 and related packages require pydantic<2 +linkml-runtime==1.7.4; python_version >= "3.9" +schemasheets==0.2.1; python_version >= "3.9" +oaklib==0.5.32; python_version >= "3.9" pyyaml==6.0.1; python_version >= "3.9" diff --git a/src/hdmf/__init__.py b/src/hdmf/__init__.py index b32a7f579..6fc72a117 100644 --- a/src/hdmf/__init__.py +++ b/src/hdmf/__init__.py @@ -3,7 +3,7 @@ from .container import Container, Data, DataRegion, HERDManager from .region import ListSlicer from .utils import docval, getargs -from .term_set import TermSet, TermSetWrapper, TermSetConfigurator +from .term_set import TermSet, TermSetWrapper, TypeConfigurator @docval( diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 33183243f..a8976cf25 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -5,7 +5,7 @@ from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator from ..container import AbstractContainer, Container, Data -from ..term_set import TermSetConfigurator +from ..term_set import TypeConfigurator from ..spec import DatasetSpec, GroupSpec, NamespaceCatalog from ..spec.spec import BaseStorageSpec from ..utils import docval, getargs, ExtenderMeta, get_docval @@ -412,7 +412,7 @@ def __init__(self, **kwargs): self.__data_types = dict() self.__default_mapper_cls = mapper_cls self.__class_generator = ClassGenerator() - self.ts_config = TermSetConfigurator() + self.ts_config = TypeConfigurator() self.register_generator(CustomClassGenerator) self.register_generator(MCIClassGenerator) diff --git a/src/hdmf/common/hdmf-common-schema b/src/hdmf/common/hdmf-common-schema index 4d2ddd638..5b4cbb31d 160000 --- a/src/hdmf/common/hdmf-common-schema +++ b/src/hdmf/common/hdmf-common-schema @@ -1 +1 @@ -Subproject commit 4d2ddd6387c4e36f21f41964fe8873c083680b15 +Subproject commit 5b4cbb31dbafcff51ca70bf218f464b186568151 diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 9f00ce7d0..20ef886d3 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -159,7 +159,7 @@ def _field_config(self, arg_name, val): # From the spec, get the mapped attribute name mapped_attr_name = obj_mapper.get_attribute(spec) termset_path = os.path.join(CUR_DIR, - config_namespace['data_types'][data_type][mapped_attr_name]) + config_namespace['data_types'][data_type][mapped_attr_name]['termset']) termset = TermSet(term_schema_path=termset_path) val = TermSetWrapper(value=val, termset=termset) return val diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index d17c15e56..c082ffcff 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -162,12 +162,13 @@ def __schemasheets_convert(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: - import yaml + from ruamel.yaml import YAML from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover msg = "Install schemasheets." raise ValueError(msg) + schema_maker = SchemaMaker() tsv_file_paths = glob.glob(self.schemasheets_folder + "/*.tsv") schema = schema_maker.create_schema(tsv_file_paths) @@ -175,6 +176,7 @@ def __schemasheets_convert(self): schemasheet_schema_path = os.path.join(self.schemasheets_folder, f"{schema_dict['name']}.yaml") with open(schemasheet_schema_path, "w") as f: + yaml=YAML(typ='safe') yaml.dump(schema_dict, f) return schemasheet_schema_path @@ -298,7 +300,7 @@ def extend(self, arg): msg = ('"%s" is not in the term set.' % ', '.join([str(item) for item in bad_data])) raise ValueError(msg) -class TermSetConfigurator: +class TypeConfigurator: """ This class allows users to toggle on/off a global configuration for defined data types. When toggled on, every instance of a configuration file supported data type will be validated @@ -340,13 +342,14 @@ def load_termset_config(self,config_path): Load the configuration file for validation on the fields defined for the objects within the file. """ try: - import yaml + from ruamel.yaml import YAML except ImportError: # pragma: no cover msg = "Install yaml." raise ValueError(msg) with open(config_path, 'r') as config: - termset_config = yaml.load(config, Loader=yaml.FullLoader) + yaml=YAML(typ='safe') + termset_config = yaml.load(config) if self.config is None: # set the initial config/load after config has been unloaded self.config = termset_config if len(self.path)==0: # for loading after an unloaded config diff --git a/tests/unit/hdmf_config.yaml b/tests/unit/hdmf_config.yaml index 422917f95..92ec2f321 100644 --- a/tests/unit/hdmf_config.yaml +++ b/tests/unit/hdmf_config.yaml @@ -3,6 +3,7 @@ namespaces: version: 3.12.2 data_types: VectorData: - description: example_test_term_set.yaml + description: + termset: example_test_term_set.yaml VectorIndex: data: ... diff --git a/tests/unit/hdmf_config2.yaml b/tests/unit/hdmf_config2.yaml index adcc54a94..0aecacf51 100644 --- a/tests/unit/hdmf_config2.yaml +++ b/tests/unit/hdmf_config2.yaml @@ -3,13 +3,16 @@ namespaces: version: 3.12.2 data_types: Data: - description: example_test_term_set.yaml + description: + termset: example_test_term_set.yaml EnumData: - description: example_test_term_set.yaml + description: + termset: example_test_term_set.yaml VectorData: description: ... namespace2: version: 0 data_types: MythicData: - description: example_test_term_set.yaml + description: + termset: example_test_term_set.yaml diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index b646f9944..b4f7f0634 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -2,7 +2,7 @@ import numpy as np from hdmf import Container -from hdmf.term_set import TermSet, TermSetWrapper, TermSetConfigurator +from hdmf.term_set import TermSet, TermSetWrapper, TypeConfigurator from hdmf.testing import TestCase, remove_test_file from hdmf.common import (VectorIndex, VectorData, unload_termset_config, get_loaded_config, load_termset_config, get_type_map) @@ -231,7 +231,7 @@ def test_get_loaded_config(self): load_termset_config(config_path='tests/unit/hdmf_config.yaml') loaded_config = get_loaded_config() config = {'namespaces': {'hdmf-common': {'version': '3.12.2', - 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}, + 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, 'VectorIndex': {'data': '...'}}}}} self.assertEqual(loaded_config, config) @@ -242,45 +242,45 @@ def test_get_loaded_config_error(self): def test_config_path(self): path = 'tests/unit/hdmf_config.yaml' - tc = TermSetConfigurator(path=path) + tc = TypeConfigurator(path=path) self.assertEqual(tc.path, [path]) def test_get_config(self): path = 'tests/unit/hdmf_config.yaml' - tc = TermSetConfigurator(path=path) + tc = TypeConfigurator(path=path) self.assertEqual(tc.get_config('VectorData', 'hdmf-common'), - {'description': 'example_test_term_set.yaml'}) + {'description': {'termset': 'example_test_term_set.yaml'}}) def test_get_config_namespace_error(self): path = 'tests/unit/hdmf_config.yaml' - tc = TermSetConfigurator(path=path) + tc = TypeConfigurator(path=path) with self.assertRaises(ValueError): tc.get_config('VectorData', 'hdmf-common11') def test_get_config_container_error(self): path = 'tests/unit/hdmf_config.yaml' - tc = TermSetConfigurator(path=path) + tc = TypeConfigurator(path=path) with self.assertRaises(ValueError): tc.get_config('VectorData11', 'hdmf-common') def test_already_loaded_path_error(self): path = 'tests/unit/hdmf_config.yaml' - tc = TermSetConfigurator(path=path) + tc = TypeConfigurator(path=path) with self.assertRaises(ValueError): tc.load_termset_config(config_path=path) def test_load_two_unique_configs(self): path = 'tests/unit/hdmf_config.yaml' path2 = 'tests/unit/hdmf_config2.yaml' - tc = TermSetConfigurator(path=path) + tc = TypeConfigurator(path=path) tc.load_termset_config(config_path=path2) config = {'namespaces': {'hdmf-common': {'version': '3.12.2', 'data_types': {'VectorData': {'description': '...'}, 'VectorIndex': {'data': '...'}, - 'Data': {'description': 'example_test_term_set.yaml'}, - 'EnumData': {'description': 'example_test_term_set.yaml'}}}, + 'Data': {'description': {'termset': 'example_test_term_set.yaml'}}, + 'EnumData': {'description': {'termset': 'example_test_term_set.yaml'}}}}, 'namespace2': {'version': 0, - 'data_types': {'MythicData': {'description': 'example_test_term_set.yaml'}}}}} + 'data_types': {'MythicData': {'description': {'termset': 'example_test_term_set.yaml'}}}}}} self.assertEqual(tc.path, [path, path2]) self.assertEqual(tc.config, config) @@ -309,7 +309,7 @@ def test_load_config(self): config = get_loaded_config() self.assertEqual(config, {'namespaces': {'hdmf-common': {'version': '3.12.2', - 'data_types': {'VectorData': {'description': 'example_test_term_set.yaml'}, + 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, 'VectorIndex': {'data': '...'}}}}}) def test_validate_with_config(self): diff --git a/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml b/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml index 222205959..52af4b8a7 100644 --- a/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml +++ b/tests/unit/test_term_set_input/schemasheets/nwb_static_enums.yaml @@ -2,8 +2,7 @@ classes: BrainSample: slot_usage: cell_type: {} - slots: - - cell_type + slots: [cell_type] default_prefix: TEMP default_range: string description: this schema demonstrates the use of static enums @@ -11,42 +10,27 @@ enums: NeuronOrGlialCellTypeEnum: description: Enumeration to capture various cell types found in the brain. permissible_values: - ASTROCYTE: - description: Characteristic star-shaped glial cells in the brain and spinal - cord. - meaning: CL:0000127 - INTERNEURON: - description: Neurons whose axons (and dendrites) are limited to a single brain - area. - meaning: CL:0000099 - MICROGLIAL_CELL: - description: Microglia are the resident immune cells of the brain and constantly - patrol the cerebral microenvironment to respond to pathogens and damage. - meaning: CL:0000129 - MOTOR_NEURON: - description: Neurons whose cell body is located in the motor cortex, brainstem - or the spinal cord, and whose axon (fiber) projects to the spinal cord or - outside of the spinal cord to directly or indirectly control effector organs, - mainly muscles and glands. - meaning: CL:0000100 - OLIGODENDROCYTE: - description: Type of neuroglia whose main functions are to provide support - and insulation to axons within the central nervous system (CNS) of jawed - vertebrates. - meaning: CL:0000128 - PYRAMIDAL_NEURON: - description: Neurons with a pyramidal shaped cell body (soma) and two distinct - dendritic trees. - meaning: CL:0000598 + ASTROCYTE: {description: Characteristic star-shaped glial cells in the brain + and spinal cord., meaning: 'CL:0000127'} + INTERNEURON: {description: Neurons whose axons (and dendrites) are limited to + a single brain area., meaning: 'CL:0000099'} + MICROGLIAL_CELL: {description: Microglia are the resident immune cells of the + brain and constantly patrol the cerebral microenvironment to respond to + pathogens and damage., meaning: 'CL:0000129'} + MOTOR_NEURON: {description: 'Neurons whose cell body is located in the motor + cortex, brainstem or the spinal cord, and whose axon (fiber) projects to + the spinal cord or outside of the spinal cord to directly or indirectly + control effector organs, mainly muscles and glands.', meaning: 'CL:0000100'} + OLIGODENDROCYTE: {description: Type of neuroglia whose main functions are to + provide support and insulation to axons within the central nervous system + (CNS) of jawed vertebrates., meaning: 'CL:0000128'} + PYRAMIDAL_NEURON: {description: Neurons with a pyramidal shaped cell body (soma) + and two distinct dendritic trees., meaning: 'CL:0000598'} id: https://w3id.org/linkml/examples/nwb_static_enums -imports: -- linkml:types +imports: ['linkml:types'] name: nwb_static_enums -prefixes: - CL: http://purl.obolibrary.org/obo/CL_ - TEMP: https://example.org/TEMP/ - linkml: https://w3id.org/linkml/ +prefixes: {CL: 'http://purl.obolibrary.org/obo/CL_', TEMP: 'https://example.org/TEMP/', + linkml: 'https://w3id.org/linkml/'} slots: - cell_type: - required: true + cell_type: {required: true} title: static enums example From c1797917d1e1dfae3f9effabd4040f3cf5e377ea Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 14:36:38 -0700 Subject: [PATCH 56/78] checkpoint of updates --- tests/unit/test_extension_config.yaml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 tests/unit/test_extension_config.yaml diff --git a/tests/unit/test_extension_config.yaml b/tests/unit/test_extension_config.yaml deleted file mode 100644 index e0bf09d0b..000000000 --- a/tests/unit/test_extension_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -namespaces: - hdmf-common: - version: TBD - data_types: - VectorData: - data: '/Users/mavaylon/Research/NWB/hdmf2/hdmf/tests/unit/example_test_term_set.yaml' - field2: '...' - type2: - field1: '...' - hdmf-experimental: - version: TBD - core: - data_types: - NWBFile: - experimenter: experimenter_termset2.yaml - blah: experimenter.yaml From f108c07a829829997d21603d32790c3cca9436fc Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 14:38:08 -0700 Subject: [PATCH 57/78] checkpoint of updates --- requirements-opt.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-opt.txt b/requirements-opt.txt index 3bb8833da..5ce083df5 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -4,4 +4,3 @@ zarr==2.16.1 linkml-runtime==1.7.4; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" oaklib==0.5.32; python_version >= "3.9" -pyyaml==6.0.1; python_version >= "3.9" From 9d27717681e8810e293ce847c3f3234d8c436f3b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 21:42:07 +0000 Subject: [PATCH 58/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements-opt.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-opt.txt b/requirements-opt.txt index 5a21a284d..5ce083df5 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -4,4 +4,3 @@ zarr==2.16.1 linkml-runtime==1.7.4; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" oaklib==0.5.32; python_version >= "3.9" - From 6b19e6d17147cd4f58b10a70d8052d65dba16832 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 15:00:51 -0700 Subject: [PATCH 59/78] copy --- src/hdmf/build/manager.py | 13 +++++++++---- src/hdmf/common/__init__.py | 3 --- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index a8976cf25..a44e45c2b 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -397,14 +397,18 @@ class TypeMap: """ @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None}, - {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}) + {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}, + {'name': 'type_config', 'type': TypeConfigurator, 'doc': 'The TypeConfigurator to use', + 'default': None}) def __init__(self, **kwargs): - namespaces, mapper_cls = getargs('namespaces', 'mapper_cls', kwargs) + namespaces, mapper_cls, type_config = getargs('namespaces', 'mapper_cls', 'type_config', kwargs) if namespaces is None: namespaces = NamespaceCatalog() if mapper_cls is None: from .objectmapper import ObjectMapper # avoid circular import mapper_cls = ObjectMapper + if type_config is None: + type_config = TypeConfigurator() self.__ns_catalog = namespaces self.__mappers = dict() # already constructed ObjectMapper classes self.__mapper_cls = dict() # the ObjectMapper class to use for each container type @@ -412,7 +416,8 @@ def __init__(self, **kwargs): self.__data_types = dict() self.__default_mapper_cls = mapper_cls self.__class_generator = ClassGenerator() - self.ts_config = TypeConfigurator() + self.ts_config = type_config + self.register_generator(CustomClassGenerator) self.register_generator(MCIClassGenerator) @@ -425,7 +430,7 @@ def container_types(self): return self.__container_types def __copy__(self): - ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls) + ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls, self.ts_config) ret.merge(self) return ret diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 870c3f00c..f1ff98031 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -180,9 +180,6 @@ def get_type_map(**kwargs): type_map.load_namespaces(extensions) elif isinstance(extensions, TypeMap): type_map.merge(extensions) - global_config = __TYPE_MAP.ts_config - if global_config is not None: - type_map.ts_config = global_config return type_map From 7b25393951a7ae81338f4cbd3258a5e4eb4c2148 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 15:13:47 -0700 Subject: [PATCH 60/78] clean up --- src/hdmf/build/manager.py | 2 -- src/hdmf/common/__init__.py | 5 ----- src/hdmf/container.py | 24 +++++++++++------------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index a44e45c2b..957001b26 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -739,8 +739,6 @@ def register_container_type(self, **kwargs): self.__container_types.setdefault(namespace, dict()) self.__container_types[namespace][data_type] = container_cls self.__data_types.setdefault(container_cls, (namespace, data_type)) - # from hdmf.common import get_type_map # circular import - # container_cls.type_map = get_type_map() if not isinstance(container_cls, TypeSource): setattr(container_cls, spec.type_key(), data_type) setattr(container_cls, 'namespace', namespace) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index f1ff98031..2b5cb1300 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -61,20 +61,15 @@ def register_class(**kwargs): def _dec(cls): _set_exp(cls) __TYPE_MAP.register_container_type(namespace, data_type, cls) - cls.type_map = __TYPE_MAP - # cls.namespace = namespace return cls else: def _dec(cls): __TYPE_MAP.register_container_type(namespace, data_type, cls) - cls.type_map = __TYPE_MAP - # cls.namespace = namespace return cls if container_cls is None: return _dec else: - # container_cls.type_map = get_type_map() _dec(container_cls) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 20ef886d3..2682f50b5 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -108,21 +108,19 @@ def _field_config(self, arg_name, val): The modifications are not written to file, avoiding permanent modifications. """ # load termset configuration file from global Config - try: - configurator = self.type_map.ts_config - if len(configurator.path)>0: - # The type_map has a config always set; however, when toggled off, the config path is empty. - CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) - termset_config = configurator.config - else: - return val - except AttributeError: - # These are for classes that have not been registered, e.g., some test containers. - # Without register_class, the type_map parameter is not defined. + from hdmf.common import get_type_map # circular import + type_map = get_type_map() + configurator = type_map.ts_config + + if len(configurator.path)>0: + # The type_map has a config always set; however, when toggled off, the config path is empty. + CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0])) + termset_config = configurator.config + else: return val # check to see that the namespace for the container is in the config - if self.namespace not in self.type_map.container_types: + if self.namespace not in type_map.container_types: msg = "%s not found within loaded configuration." % self.namespace warn(msg) return val @@ -137,7 +135,7 @@ def _field_config(self, arg_name, val): return val else: for attr in config_namespace['data_types'][data_type]: - obj_mapper = self.type_map.get_map(self) + obj_mapper = type_map.get_map(self) # get the spec according to attr name in schema # Note: this is the name for the field in the config From b2c902a7e904b083bec68c61e705dbb8ab1526b0 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 15:27:15 -0700 Subject: [PATCH 61/78] clean --- src/hdmf/build/manager.py | 2 +- src/hdmf/common/__init__.py | 8 ++++---- src/hdmf/container.py | 1 - src/hdmf/term_set.py | 6 +++--- tests/unit/test_term_set.py | 19 +++++++------------ 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 957001b26..6320dab17 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -398,7 +398,7 @@ class TypeMap: @docval({'name': 'namespaces', 'type': NamespaceCatalog, 'doc': 'the NamespaceCatalog to use', 'default': None}, {'name': 'mapper_cls', 'type': type, 'doc': 'the ObjectMapper class to use', 'default': None}, - {'name': 'type_config', 'type': TypeConfigurator, 'doc': 'The TypeConfigurator to use', + {'name': 'type_config', 'type': TypeConfigurator, 'doc': 'The TypeConfigurator to use.', 'default': None}) def __init__(self, **kwargs): namespaces, mapper_cls, type_config = getargs('namespaces', 'mapper_cls', 'type_config', kwargs) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 2b5cb1300..0e0b812ac 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -22,12 +22,12 @@ @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}, is_method=False) -def load_termset_config(**kwargs): +def load_type_config(**kwargs): """ This method will either load the default config or the config provided by the path. """ config_path = kwargs['config_path'] - __TYPE_MAP.ts_config.load_termset_config(config_path) + __TYPE_MAP.ts_config.load_type_config(config_path) def get_loaded_config(): """ @@ -39,11 +39,11 @@ def get_loaded_config(): else: return __TYPE_MAP.ts_config.config -def unload_termset_config(): +def unload_type_config(): """ Unload the configuration file. """ - return __TYPE_MAP.ts_config.unload_termset_config() + return __TYPE_MAP.ts_config.unload_type_config() # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 2682f50b5..20d18cfeb 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -162,7 +162,6 @@ def _field_config(self, arg_name, val): val = TermSetWrapper(value=val, termset=termset) return val - @classmethod def _getter(cls, field): """ diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index c082ffcff..f8a9c7b0b 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -313,7 +313,7 @@ def __init__(self, **kwargs): self.path = [] else: self.path = [kwargs['path']] - self.load_termset_config(config_path=self.path[0]) + self.load_type_config(config_path=self.path[0]) @docval({'name': 'data_type', 'type': str, 'doc': 'The desired data type within the configuration file.'}, @@ -337,7 +337,7 @@ def get_config(self, data_type, namespace): raise ValueError(msg) @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}) - def load_termset_config(self,config_path): + def load_type_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. """ @@ -375,7 +375,7 @@ def load_termset_config(self,config_path): # append path to self.path self.path.append(config_path) - def unload_termset_config(self): + def unload_type_config(self): """ Remove validation according to termset configuration file. """ diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index b4f7f0634..94915d060 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -4,8 +4,7 @@ from hdmf import Container from hdmf.term_set import TermSet, TermSetWrapper, TypeConfigurator from hdmf.testing import TestCase, remove_test_file -from hdmf.common import (VectorIndex, VectorData, unload_termset_config, get_loaded_config, load_termset_config, - get_type_map) +from hdmf.common import VectorIndex, VectorData, unload_type_config, get_loaded_config, load_type_config from hdmf.utils import popargs @@ -225,10 +224,10 @@ def setUp(self): self.skipTest("optional LinkML module is not installed") def tearDown(self): - unload_termset_config() + unload_type_config() def test_get_loaded_config(self): - load_termset_config(config_path='tests/unit/hdmf_config.yaml') + load_type_config(config_path='tests/unit/hdmf_config.yaml') loaded_config = get_loaded_config() config = {'namespaces': {'hdmf-common': {'version': '3.12.2', 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, @@ -267,13 +266,13 @@ def test_already_loaded_path_error(self): path = 'tests/unit/hdmf_config.yaml' tc = TypeConfigurator(path=path) with self.assertRaises(ValueError): - tc.load_termset_config(config_path=path) + tc.load_type_config(config_path=path) def test_load_two_unique_configs(self): path = 'tests/unit/hdmf_config.yaml' path2 = 'tests/unit/hdmf_config2.yaml' tc = TypeConfigurator(path=path) - tc.load_termset_config(config_path=path2) + tc.load_type_config(config_path=path2) config = {'namespaces': {'hdmf-common': {'version': '3.12.2', 'data_types': {'VectorData': {'description': '...'}, 'VectorIndex': {'data': '...'}, @@ -291,8 +290,6 @@ class ExtensionContainer(Container): def __init__(self, **kwargs): description, namespace = popargs('description', 'namespace', kwargs) super().__init__(**kwargs) - self.namespace = namespace - self.type_map = get_type_map() self.description = description @@ -300,10 +297,10 @@ class TestTermSetConfigVectorData(TestCase): def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") - load_termset_config(config_path='tests/unit/hdmf_config.yaml') + load_type_config(config_path='tests/unit/hdmf_config.yaml') def tearDown(self): - unload_termset_config() + unload_type_config() def test_load_config(self): config = get_loaded_config() @@ -335,8 +332,6 @@ def test_already_wrapped_warn(self): data=[0], description=TermSetWrapper(value='Homo sapiens', termset=terms)) - unload_termset_config() - def test_warn_field_not_in_spec(self): col1 = VectorData(name='col1', description='Homo sapiens', From 6d7369e3416b6d919b289a0fae6c31688678d538 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Mon, 25 Mar 2024 17:09:22 -0700 Subject: [PATCH 62/78] Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ecac8d01..27957d6c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # HDMF Changelog -## HDMF 3.13.0 (March 20, 2024) +## HDMF 3.14.0 (March 20, 2024) ### Enhancements - Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) + +## HDMF 3.13.0 (March 20, 2024) + +### Enhancements - Unwrap `TermSetWrapper` within the builder to support different backends more efficiently. @mavaylon1 [#1070](https://github.com/hdmf-dev/hdmf/pull/1070) - Added docs page that lists limitations of support for the HDMF specification language. @rly [#1069](https://github.com/hdmf-dev/hdmf/pull/1069) - Added warning when using `add_row` or `add_column` to add a ragged array to `DynamicTable` without an index parameter. @stephprince [#1066](https://github.com/hdmf-dev/hdmf/pull/1066) From 14c0e1b834aeda425d0e56e0778eae1c45078e43 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 17:11:56 -0700 Subject: [PATCH 63/78] clean up --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 20d18cfeb..6fb051d78 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -127,7 +127,7 @@ def _field_config(self, arg_name, val): else: # check to see that the container type is in the config under the namespace config_namespace = termset_config['namespaces'][self.namespace] - data_type = getattr(self, self._data_type_attr) + data_type = get_data_type() if data_type not in config_namespace['data_types']: msg = '%s not found within the configuration for %s' % (data_type, self.namespace) From e133b7f75684bb5b0d68eb4703d5f63375f2d382 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Mon, 25 Mar 2024 17:17:39 -0700 Subject: [PATCH 64/78] clean up --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 6fb051d78..f720379e7 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -127,7 +127,7 @@ def _field_config(self, arg_name, val): else: # check to see that the container type is in the config under the namespace config_namespace = termset_config['namespaces'][self.namespace] - data_type = get_data_type() + data_type = self.get_data_type() if data_type not in config_namespace['data_types']: msg = '%s not found within the configuration for %s' % (data_type, self.namespace) From 67d1c4f53d7ddcfa5e45ae71e99879e2e1420696 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 27 Mar 2024 07:01:07 -0700 Subject: [PATCH 65/78] test copy --- tests/unit/common/test_common.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/common/test_common.py b/tests/unit/common/test_common.py index 76c99d44a..6bf6c3766 100644 --- a/tests/unit/common/test_common.py +++ b/tests/unit/common/test_common.py @@ -1,5 +1,5 @@ from hdmf import Data, Container -from hdmf.common import get_type_map +from hdmf.common import get_type_map, load_type_config, unload_type_config from hdmf.testing import TestCase @@ -11,3 +11,15 @@ def test_base_types(self): self.assertIs(cls, Container) cls = tm.get_dt_container_cls('Data', 'hdmf-common') self.assertIs(cls, Data) + + def test_copy_ts_config(self): + path = 'tests/unit/hdmf_config.yaml' + load_type_config(config_path=path) + tm = get_type_map() + config = {'namespaces': {'hdmf-common': {'version': '3.12.2', + 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, + 'VectorIndex': {'data': '...'}}}}} + + self.assertEqual(tm.ts_config.config, config) + self.assertEqual(tm.ts_config.path, [path]) + unload_type_config() From 6ef126c33ba9a60e3d2cb026831ac3044b22beab Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Wed, 27 Mar 2024 14:50:57 -0700 Subject: [PATCH 66/78] name --- tests/unit/test_term_set.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 94915d060..10d4fddc2 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -218,7 +218,7 @@ def test_wrapper_extend_error(self): with self.assertRaises(ValueError): data_obj.extend(['bad_data']) -class TestTermSetConfig(TestCase): +class TestTypeConfig(TestCase): def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") @@ -226,15 +226,6 @@ def setUp(self): def tearDown(self): unload_type_config() - def test_get_loaded_config(self): - load_type_config(config_path='tests/unit/hdmf_config.yaml') - loaded_config = get_loaded_config() - config = {'namespaces': {'hdmf-common': {'version': '3.12.2', - 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, - 'VectorIndex': {'data': '...'}}}}} - - self.assertEqual(loaded_config, config) - def test_get_loaded_config_error(self): with self.assertRaises(ValueError): get_loaded_config() @@ -293,7 +284,7 @@ def __init__(self, **kwargs): self.description = description -class TestTermSetConfigVectorData(TestCase): +class TestGlobalTypeConfig(TestCase): def setUp(self): if not REQUIREMENTS_INSTALLED: self.skipTest("optional LinkML module is not installed") From 2a7a048eb283fddfa9a195443888ca4be8dcb297 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 27 Mar 2024 21:12:34 -0700 Subject: [PATCH 67/78] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27957d6c3..22c21b0e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.14.0 (March 20, 2024) +## HDMF 3.14.0 (Upcoming) ### Enhancements - Added `TermSetConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) From 6c28042af1e90c54fa41cdc3cd3b78ac2194f186 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 27 Mar 2024 21:13:07 -0700 Subject: [PATCH 68/78] Update requirements-opt.txt --- requirements-opt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-opt.txt b/requirements-opt.txt index 5ce083df5..a4c4c149b 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -1,5 +1,5 @@ # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF -tqdm==4.66.1 +tqdm==4.66.2 zarr==2.16.1 linkml-runtime==1.7.4; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" From c7b0d5f9ddff4c57fb94718768bf6412db34073b Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 27 Mar 2024 21:13:22 -0700 Subject: [PATCH 69/78] Update requirements-opt.txt --- requirements-opt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-opt.txt b/requirements-opt.txt index a4c4c149b..53fd11e3a 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -1,6 +1,6 @@ # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF tqdm==4.66.2 -zarr==2.16.1 +zarr==2.17.1 linkml-runtime==1.7.4; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" oaklib==0.5.32; python_version >= "3.9" From 54b68ae7ef25d4876780e294b7e797b729dae64d Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Wed, 27 Mar 2024 21:21:19 -0700 Subject: [PATCH 70/78] Update container.py Co-authored-by: Ryan Ly --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index f720379e7..1dfc155a7 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -149,7 +149,7 @@ def _field_config(self, arg_name, val): return val else: # If the val has been manually wrapped then skip checking the config for the attr - if type(val) == TermSetWrapper: + if isinstance(val, TermSetWrapper): msg = "Field value already wrapped with TermSetWrapper." warn(msg) return val From 2f3a51b9d2fbd3eb70a4930036815a0ecee4d101 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Wed, 27 Mar 2024 21:21:29 -0700 Subject: [PATCH 71/78] Update container.py Co-authored-by: Ryan Ly --- src/hdmf/container.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 1dfc155a7..2810e330c 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -90,9 +90,10 @@ def setter(self, val): return setter - def get_data_type(self): + @property + def data_type(self): """ - Method that return data type. + Return the spec data type associated with this container. """ return getattr(self, self._data_type_attr) From 89ab7fa889cf1978feda4ebf913920022e4b5dff Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Wed, 27 Mar 2024 21:21:42 -0700 Subject: [PATCH 72/78] Update __init__.py Co-authored-by: Ryan Ly --- src/hdmf/common/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 0e0b812ac..e7348cc56 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -29,7 +29,7 @@ def load_type_config(**kwargs): config_path = kwargs['config_path'] __TYPE_MAP.ts_config.load_type_config(config_path) -def get_loaded_config(): +def get_loaded_type_config(): """ This method returns the entire config file. """ From e5eff48daedf703f6af1045af24c3126f9f2e4c9 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Wed, 27 Mar 2024 21:21:47 -0700 Subject: [PATCH 73/78] Update manager.py Co-authored-by: Ryan Ly --- src/hdmf/build/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 6320dab17..2514181d2 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -416,7 +416,7 @@ def __init__(self, **kwargs): self.__data_types = dict() self.__default_mapper_cls = mapper_cls self.__class_generator = ClassGenerator() - self.ts_config = type_config + self.type_config = type_config self.register_generator(CustomClassGenerator) self.register_generator(MCIClassGenerator) From 5efb5e09717d03a75da360a9ae56f6a64f83e975 Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 28 Mar 2024 09:07:00 -0700 Subject: [PATCH 74/78] clean --- src/hdmf/build/manager.py | 2 +- src/hdmf/common/__init__.py | 8 ++++---- src/hdmf/container.py | 4 ++-- src/hdmf/term_set.py | 8 +------- tests/unit/common/test_common.py | 4 ++-- tests/unit/test_container.py | 12 ++++++------ tests/unit/test_term_set.py | 9 +++++---- 7 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index 2514181d2..a26de3279 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -430,7 +430,7 @@ def container_types(self): return self.__container_types def __copy__(self): - ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls, self.ts_config) + ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls, self.type_config) ret.merge(self) return ret diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index e7348cc56..6fbb5da58 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -27,23 +27,23 @@ def load_type_config(**kwargs): This method will either load the default config or the config provided by the path. """ config_path = kwargs['config_path'] - __TYPE_MAP.ts_config.load_type_config(config_path) + __TYPE_MAP.type_config.load_type_config(config_path) def get_loaded_type_config(): """ This method returns the entire config file. """ - if __TYPE_MAP.ts_config.config is None: + if __TYPE_MAP.type_config.config is None: msg = "No configuration is loaded." raise ValueError(msg) else: - return __TYPE_MAP.ts_config.config + return __TYPE_MAP.type_config.config def unload_type_config(): """ Unload the configuration file. """ - return __TYPE_MAP.ts_config.unload_type_config() + return __TYPE_MAP.type_config.unload_type_config() # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 2810e330c..cbd2dc000 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -111,7 +111,7 @@ def _field_config(self, arg_name, val): # load termset configuration file from global Config from hdmf.common import get_type_map # circular import type_map = get_type_map() - configurator = type_map.ts_config + configurator = type_map.type_config if len(configurator.path)>0: # The type_map has a config always set; however, when toggled off, the config path is empty. @@ -128,7 +128,7 @@ def _field_config(self, arg_name, val): else: # check to see that the container type is in the config under the namespace config_namespace = termset_config['namespaces'][self.namespace] - data_type = self.get_data_type() + data_type = self.data_type if data_type not in config_namespace['data_types']: msg = '%s not found within the configuration for %s' % (data_type, self.namespace) diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index f8a9c7b0b..1464f505c 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,6 +5,7 @@ import warnings import numpy as np from .data_utils import append_data, extend_data +from ruamel.yaml import YAML class TermSet: @@ -162,7 +163,6 @@ def __schemasheets_convert(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: - from ruamel.yaml import YAML from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover @@ -341,12 +341,6 @@ def load_type_config(self,config_path): """ Load the configuration file for validation on the fields defined for the objects within the file. """ - try: - from ruamel.yaml import YAML - except ImportError: # pragma: no cover - msg = "Install yaml." - raise ValueError(msg) - with open(config_path, 'r') as config: yaml=YAML(typ='safe') termset_config = yaml.load(config) diff --git a/tests/unit/common/test_common.py b/tests/unit/common/test_common.py index 6bf6c3766..e20614852 100644 --- a/tests/unit/common/test_common.py +++ b/tests/unit/common/test_common.py @@ -20,6 +20,6 @@ def test_copy_ts_config(self): 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, 'VectorIndex': {'data': '...'}}}}} - self.assertEqual(tm.ts_config.config, config) - self.assertEqual(tm.ts_config.path, [path]) + self.assertEqual(tm.type_config.config, config) + self.assertEqual(tm.type_config.path, [path]) unload_type_config() diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index 5c9948222..9ac81ba13 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -60,7 +60,7 @@ def test_new(self): def test_get_data_type(self): obj = Container('obj1') - dt = obj.get_data_type() + dt = obj.data_type self.assertEqual(dt, 'Container') def test_new_object_id_none(self): @@ -524,7 +524,7 @@ class EmptyFields(AbstractContainer): self.assertTupleEqual(EmptyFields.get_fields_conf(), tuple()) props = TestAbstractContainerFieldsConf.find_all_properties(EmptyFields) - expected = ['all_objects', 'children', 'container_source', 'fields', 'modified', + expected = ['all_objects', 'children', 'container_source', 'data_type', 'fields', 'modified', 'name', 'object_id', 'parent', 'read_io'] self.assertListEqual(props, expected) @@ -545,8 +545,8 @@ def __init__(self, **kwargs): self.assertTupleEqual(NamedFields.get_fields_conf(), expected) props = TestAbstractContainerFieldsConf.find_all_properties(NamedFields) - expected = ['all_objects', 'children', 'container_source', 'field1', 'field2', - 'fields', 'modified', 'name', 'object_id', + expected = ['all_objects', 'children', 'container_source', 'data_type', + 'field1', 'field2', 'fields', 'modified', 'name', 'object_id', 'parent', 'read_io'] self.assertListEqual(props, expected) @@ -627,8 +627,8 @@ class NamedFieldsChild(NamedFields): self.assertTupleEqual(NamedFieldsChild.get_fields_conf(), expected) props = TestAbstractContainerFieldsConf.find_all_properties(NamedFieldsChild) - expected = ['all_objects', 'children', 'container_source', 'field1', 'field2', - 'fields', 'modified', 'name', 'object_id', + expected = ['all_objects', 'children', 'container_source', 'data_type', + 'field1', 'field2', 'fields', 'modified', 'name', 'object_id', 'parent', 'read_io'] self.assertListEqual(props, expected) diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 10d4fddc2..ee0f20b06 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -4,7 +4,8 @@ from hdmf import Container from hdmf.term_set import TermSet, TermSetWrapper, TypeConfigurator from hdmf.testing import TestCase, remove_test_file -from hdmf.common import VectorIndex, VectorData, unload_type_config, get_loaded_config, load_type_config +from hdmf.common import (VectorIndex, VectorData, unload_type_config, + get_loaded_type_config, load_type_config) from hdmf.utils import popargs @@ -226,9 +227,9 @@ def setUp(self): def tearDown(self): unload_type_config() - def test_get_loaded_config_error(self): + def test_get_loaded_type_config_error(self): with self.assertRaises(ValueError): - get_loaded_config() + get_loaded_type_config() def test_config_path(self): path = 'tests/unit/hdmf_config.yaml' @@ -294,7 +295,7 @@ def tearDown(self): unload_type_config() def test_load_config(self): - config = get_loaded_config() + config = get_loaded_type_config() self.assertEqual(config, {'namespaces': {'hdmf-common': {'version': '3.12.2', 'data_types': {'VectorData': {'description': {'termset': 'example_test_term_set.yaml'}}, From 13ee1f4e6d5841c9b275441343cd5d6bb9d4929d Mon Sep 17 00:00:00 2001 From: mavaylon1 Date: Thu, 28 Mar 2024 09:24:27 -0700 Subject: [PATCH 75/78] namespace --- src/hdmf/container.py | 1 - tests/unit/test_term_set.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index cbd2dc000..f93c06199 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -119,7 +119,6 @@ def _field_config(self, arg_name, val): termset_config = configurator.config else: return val - # check to see that the namespace for the container is in the config if self.namespace not in type_map.container_types: msg = "%s not found within loaded configuration." % self.namespace diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index ee0f20b06..99bd6bf59 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -281,6 +281,7 @@ class ExtensionContainer(Container): def __init__(self, **kwargs): description, namespace = popargs('description', 'namespace', kwargs) + self.namespace = namespace super().__init__(**kwargs) self.description = description From bcd7f21c64e545da2a1e96eab4631eb1cb64e484 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 28 Mar 2024 10:02:19 -0700 Subject: [PATCH 76/78] Update src/hdmf/common/__init__.py --- src/hdmf/common/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 6fbb5da58..248ca1095 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -25,6 +25,7 @@ def load_type_config(**kwargs): """ This method will either load the default config or the config provided by the path. + NOTE: This config is global and shared across all type maps. """ config_path = kwargs['config_path'] __TYPE_MAP.type_config.load_type_config(config_path) From 87c18279d344317d38057081061581890d74b596 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Thu, 28 Mar 2024 10:03:04 -0700 Subject: [PATCH 77/78] Update src/hdmf/common/__init__.py Co-authored-by: Ryan Ly --- src/hdmf/common/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 248ca1095..006c2dcc2 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -26,6 +26,7 @@ def load_type_config(**kwargs): """ This method will either load the default config or the config provided by the path. NOTE: This config is global and shared across all type maps. + NOTE: This config is global and shared across all type maps. """ config_path = kwargs['config_path'] __TYPE_MAP.type_config.load_type_config(config_path) From 0efd22bd973c83da9f2947a4192aca9f140174c9 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 28 Mar 2024 10:04:43 -0700 Subject: [PATCH 78/78] Update __init__.py --- src/hdmf/common/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 006c2dcc2..248ca1095 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -26,7 +26,6 @@ def load_type_config(**kwargs): """ This method will either load the default config or the config provided by the path. NOTE: This config is global and shared across all type maps. - NOTE: This config is global and shared across all type maps. """ config_path = kwargs['config_path'] __TYPE_MAP.type_config.load_type_config(config_path)