From d93ee35bdf71ceb732653c1097b716edcc12e93a Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Sun, 14 Jan 2024 12:59:08 -0600 Subject: [PATCH 1/5] First pass on HedVersion to get read and write --- spec/ndx-hed.extensions.yaml | 15 --- src/pynwb/ndx_hed/__init__.py | 69 +++++++---- src/pynwb/ndx_hed/hed_annotations.py | 20 ++-- src/pynwb/tests/test_hed_annotations.py | 148 +++++++++++++----------- src/spec/create_extension_spec.py | 47 ++++---- 5 files changed, 165 insertions(+), 134 deletions(-) diff --git a/spec/ndx-hed.extensions.yaml b/spec/ndx-hed.extensions.yaml index 9f62e48..dcede8a 100644 --- a/spec/ndx-hed.extensions.yaml +++ b/spec/ndx-hed.extensions.yaml @@ -1,16 +1,3 @@ -datasets: -- neurodata_type_def: HedAnnotations - neurodata_type_inc: VectorData - dtype: text - doc: An extension of VectorData for Hierarchical Event Descriptor (HED) tags. If - HED tags are used, the HED schema version must be specified in the NWB file using - the HedVersion type. - attributes: - - name: sub_name - dtype: text - doc: The smallest possible difference between two event times. Usually 1 divided - by the event time sampling rate on the data acquisition system. - required: false groups: - neurodata_type_def: HedVersion neurodata_type_inc: LabMetaData @@ -21,7 +8,5 @@ groups: attributes: - name: version dtype: text - shape: - - null doc: The version of the HED schema used to validate the HED tags, e.g., '8.2.0'. Required if HED tags are used in the NWB file. diff --git a/src/pynwb/ndx_hed/__init__.py b/src/pynwb/ndx_hed/__init__.py index a5efd06..08462a3 100644 --- a/src/pynwb/ndx_hed/__init__.py +++ b/src/pynwb/ndx_hed/__init__.py @@ -1,30 +1,55 @@ import os from pynwb import load_namespaces, get_class -try: - from importlib.resources import files -except ImportError: - # TODO: Remove when python 3.9 becomes the new minimum - from importlib_resources import files +# Set path of the namespace.yaml file to the expected install location +ndx_hed_specpath = os.path.join( + os.path.dirname(__file__), + 'spec', + 'ndx-events.namespace.yaml' +) -# Get path to the namespace.yaml file with the expected location when installed not in editable mode -__location_of_this_file = files(__name__) -__spec_path = __location_of_this_file / "spec" / "ndx-hed.namespace.yaml" - -# If that path does not exist, we are likely running in editable mode. Use the local path instead -if not os.path.exists(__spec_path): - __spec_path = __location_of_this_file.parent.parent.parent / "spec" / "ndx-hed.namespace.yaml" +# If the extension has not been installed yet but we are running directly from +# the git repo +if not os.path.exists(ndx_hed_specpath): + ndx_hed_specpath = os.path.abspath(os.path.join( + os.path.dirname(__file__), + '..', '..', '..', + 'spec', + 'ndx-hed.namespace.yaml' + )) # Load the namespace -load_namespaces(str(__spec_path)) - -# TODO: Define your classes here to make them accessible at the package level. -# Either have PyNWB generate a class from the spec using `get_class` as shown -# below or write a custom class and register it using the class decorator -# `@register_class("TetrodeSeries", "ndx-hed")` -# HedAnnotations = get_class("HedAnnotations", "ndx-hed") +load_namespaces(ndx_hed_specpath) +# +# from . import io as __io # noqa: E402,F401 +# from .hed_models import NWBHedVersion +# +# try: +# from importlib.resources import files +# except ImportError: +# # TODO: Remove when python 3.9 becomes the new minimum +# from importlib_resources import files +# +# +# +# # Get path to the namespace.yaml file with the expected location when installed not in editable mode +# __location_of_this_file = files(__name__) +# __spec_path = __location_of_this_file / "spec" / "ndx-hed.namespace.yaml" +# +# # If that path does not exist, we are likely running in editable mode. Use the local path instead +# if not os.path.exists(__spec_path): +# __spec_path = __location_of_this_file.parent.parent.parent / "spec" / "ndx-hed.namespace.yaml" +# +# # Load the namespace +# load_namespaces(str(__spec_path)) +# +# # TODO: Define your classes here to make them accessible at the package level. +# # Either have PyNWB generate a class from the spec using `get_class` as shown +# # below or write a custom class and register it using the class decorator +# # `@register_class("TetrodeSeries", "ndx-hed")` +# # HedAnnotations = get_class("HedAnnotations", "ndx-hed") # HedVersion = get_class("HedVersion", "ndx-hed") - -# Remove these functions from the package +# +# # Remove these functions from the package del load_namespaces, get_class -from .hed_annotations import HedVersion, HedAnnotations +from .hed_annotations import HedVersion diff --git a/src/pynwb/ndx_hed/hed_annotations.py b/src/pynwb/ndx_hed/hed_annotations.py index 3d79778..f544285 100644 --- a/src/pynwb/ndx_hed/hed_annotations.py +++ b/src/pynwb/ndx_hed/hed_annotations.py @@ -1,7 +1,7 @@ from collections.abc import Iterable from hdmf.common import VectorData from hdmf.utils import docval, getargs, get_docval, popargs -from hed.schema import HedSchema, HedSchemaGroup, load_schema_version +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string from pynwb import register_class from pynwb.file import LabMetaData @@ -84,9 +84,10 @@ class HedVersion(LabMetaData): """ - __nwbfields__ = ('name', 'description', 'version') + __nwbfields__ = ('name', 'description', 'version', 'schema_string') - @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) + # @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) + @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) def __init__(self, version): kwargs = {'name': 'hed_version'} super().__init__(**kwargs) @@ -95,10 +96,15 @@ def __init__(self, version): def _init_internal(self): """ - Create a HedSchema or HedSchemaGroup object from the HED Versions + Create a HED schema string """ - self._hed_schema = load_schema_version(self.version) + hed_schema = load_schema_version(self.version) + self.schema_string = hed_schema.get_as_xml_string() @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) - def get_hed_schema(self): - return self._hed_schema + def get_version(self): + return self.version + + @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) + def get_schema(self): + return from_string(self.schema_string) diff --git a/src/pynwb/tests/test_hed_annotations.py b/src/pynwb/tests/test_hed_annotations.py index 723b5dd..70d23c7 100644 --- a/src/pynwb/tests/test_hed_annotations.py +++ b/src/pynwb/tests/test_hed_annotations.py @@ -1,11 +1,12 @@ """Unit and integration tests for ndx-hed.""" from datetime import datetime from dateutil.tz import tzlocal, tzutc +from uuid import uuid4, UUID from hed.schema import HedSchema, HedSchemaGroup -from pynwb import NWBFile, NWBHDF5IO, get_manager # , NWBFile +from pynwb import NWBHDF5IO, get_manager, NWBFile from pynwb.testing.mock.file import mock_NWBFile -from pynwb.testing import TestCase, remove_test_file # , NWBH5IOFlexMixin -from src.pynwb.ndx_hed import HedAnnotations, HedVersion +from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin +from src.pynwb.ndx_hed import HedVersion class TestHedVersion(TestCase): @@ -13,19 +14,32 @@ class TestHedVersion(TestCase): def test_constructor(self): """Test setting HedNWBFile values using the constructor.""" - hed_version1 = HedVersion(["8.2.0"]) + hed_version1 = HedVersion("8.2.0") + self.assertIsInstance(hed_version1, HedVersion) + self.assertIsInstance(hed_version1.version, str) + # hed_version2 = HedVersion(["8.2.0"]) + # self.assertIsInstance(hed_version2.version, list) print(f"HED version: {str(hed_version1.version)}") - print(f"{str(hed_version1)}") - self.assertIsInstance(hed_version1.version, list) - schema1 = hed_version1.get_hed_schema() - self.assertIsInstance(schema1, HedSchema) - hed_version2 = HedVersion(["8.2.0", "sc:score_1.0.0"]) - schema2 = hed_version2.get_hed_schema() - self.assertIsInstance(schema2, HedSchemaGroup) + + # schema1 = hed_version1.get_hed_schema() + # # self.assertIsInstance(schema1, HedSchema) + # hed_version2 = HedVersion(["8.2.0", "sc:score_1.0.0"]) + + def test_get_schema(self): + hed_version1 = HedVersion("8.2.0") + hed_schema = hed_version1.get_schema() + self.assertIsInstance(hed_schema, HedSchema) + + def test_get_version(self): + hed_version1 = HedVersion("8.2.0") + version = hed_version1.get_version() + self.assertIsInstance(version, str) + self.assertEqual(version, "8.2.0") def test_add_to_nwbfile(self): nwbfile = mock_NWBFile() - hed_version1 = HedVersion(["8.2.0"]) + #hed_version1 = HedVersion(["8.2.0"]) + hed_version1 = HedVersion("8.2.0") nwbfile.add_lab_meta_data(hed_version1) hed_version2 = nwbfile.get_lab_meta_data("hed_version") self.assertIsInstance(hed_version2, HedVersion) @@ -45,7 +59,8 @@ def test_roundtrip(self): Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. """ nwbfile = mock_NWBFile() - hed_version = HedVersion(["8.2.0"]) + # hed_version = HedVersion(["8.2.0"]) + hed_version = HedVersion("8.2.0") nwbfile.add_lab_meta_data(hed_version) print(f"nwb: {str(nwbfile)}") print(f"{str(hed_version)}") @@ -53,65 +68,64 @@ def test_roundtrip(self): with NWBHDF5IO(self.path, mode="w") as io: io.write(nwbfile) - # with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: - # read_nwbfile = io.read() - # print(f"nwb: {str(read_nwbfile)}") - # read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") - # self.assertIsInstance(read_hed_version, HedVersion) - # self.assertEqual(read_hed_version.hed_version, "8.2.0") + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") -# # class TestHedNWBFileRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): -# # """Complex, more complete roundtrip test for HedNWBFile using pynwb.testing infrastructure.""" +# class TestHedNWBFileRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): +# """Complex, more complete roundtrip test for HedNWBFile using pynwb.testing infrastructure.""" # -# # def getContainerType(self): -# # return "HedNWBFile" +# def getContainerType(self): +# return "NWBFile" # -# # def addContainer(self): -# # self.nwbfile = HedNWBFile( -# # session_description="session_description", -# # identifier=str(uuid4()), -# # session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), -# # hed_schema_version="8.2.0", -# # ) +# def addContainer(self): +# self.nwbfile = HedNWBFile( +# session_description="session_description", +# identifier=str(uuid4()), +# session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), +# hed_schema_version="8.2.0", +# ) # -# # def getContainer(self, nwbfile: NWBFile): -# # return nwbfile -# +# def getContainer(self, nwbfile: NWBFile): +# return nwbfile -class TestHedAnnotationsConstructor(TestCase): - """Simple unit test for creating a HedTags.""" - def test_constructor(self): - """Test setting HED values using the constructor.""" - hed_annotations = HedAnnotations( - name='HED', - description="description", - data=["animal_target, correct_response", "animal_target, incorrect_response"], - ) - print(f"{hed_annotations}") - self.assertEqual(hed_annotations.sub_name, "HED") - self.assertEqual(hed_annotations.description, "description") - self.assertEqual(hed_annotations.data,["animal_target, correct_response", "animal_target, incorrect_response"]) - - def test_add_to_trials_table(self): - """Test adding HED column and data to a trials table.""" - nwbfile = mock_NWBFile() - hed_version = HedVersion(["8.2.0"]) - nwbfile.add_lab_meta_data(hed_version) - #nwbfile.add_trial_column("HED", "HED annotations for each trial") - nwbfile.add_trial_column(name="HED", description="HED annotations for each trial", - col_cls=HedAnnotations, data=[]) - nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="animal_target, correct_response") - nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="animal_target, incorrect_response") - - self.assertIsInstance(nwbfile.trials["HED"], HedAnnotations) - print(f"{str(nwbfile)}") - print(f"{str(nwbfile.trials)}") - hed_col = nwbfile.trials["HED"] - print(f"hed_col: {str(hed_col.data)}") - self.assertEqual(nwbfile.trials["HED"].data[0], "animal_target, correct_response") - self.assertEqual(nwbfile.trials["HED"].data[1], "animal_target, incorrect_response") +# class TestHedAnnotationsConstructor(TestCase): +# """Simple unit test for creating a HedTags.""" +# +# def test_constructor(self): +# """Test setting HED values using the constructor.""" +# hed_annotations = HedAnnotations( +# name='HED', +# description="description", +# data=["animal_target, correct_response", "animal_target, incorrect_response"], +# ) +# print(f"{hed_annotations}") +# self.assertEqual(hed_annotations.sub_name, "HED") +# self.assertEqual(hed_annotations.description, "description") +# self.assertEqual(hed_annotations.data,["animal_target, correct_response", "animal_target, incorrect_response"]) +# +# def test_add_to_trials_table(self): +# """Test adding HED column and data to a trials table.""" +# nwbfile = mock_NWBFile() +# hed_version = HedVersion(["8.2.0"]) +# nwbfile.add_lab_meta_data(hed_version) +# #nwbfile.add_trial_column("HED", "HED annotations for each trial") +# nwbfile.add_trial_column(name="HED", description="HED annotations for each trial", +# col_cls=HedAnnotations, data=[]) +# nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="animal_target, correct_response") +# nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="animal_target, incorrect_response") +# +# self.assertIsInstance(nwbfile.trials["HED"], HedAnnotations) +# print(f"{str(nwbfile)}") +# print(f"{str(nwbfile.trials)}") +# hed_col = nwbfile.trials["HED"] +# print(f"hed_col: {str(hed_col.data)}") +# self.assertEqual(nwbfile.trials["HED"].data[0], "animal_target, correct_response") +# self.assertEqual(nwbfile.trials["HED"].data[1], "animal_target, incorrect_response") class TestHedTagsSimpleRoundtrip(TestCase): """Simple roundtrip test for HedTags.""" @@ -149,9 +163,9 @@ def tearDown(self): remove_test_file(self.path) def test_hed_version(self): - hed1 = HedVersion(["8.2.0"]) + hed1 = HedVersion("8.2.0") self.assertIsInstance(hed1, HedVersion) - schema = hed1.get_hed_schema() + schema = hed1.get_schema() self.assertIsInstance(schema, HedSchema) # def test_roundtrip(self): diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index 969e777..3e03fbf 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -29,25 +29,25 @@ def main(): ns_builder.include_namespace("core") - # TODO: define your new data types - # see https://pynwb.readthedocs.io/en/latest/extensions.html#extending-nwb - # for more information - hed_annotations = NWBDatasetSpec( - neurodata_type_def="HedAnnotations", - neurodata_type_inc="VectorData", - doc=("An extension of VectorData for Hierarchical Event Descriptor (HED) tags. If HED tags are used, " - "the HED schema version must be specified in the NWB file using the HedVersion type."), - dtype="text", - attributes=[ - NWBAttributeSpec( - name='sub_name', - dtype='text', - doc=('The smallest possible difference between two event times. Usually 1 divided by the event time ' - 'sampling rate on the data acquisition system.'), - required=False, - ), - ], - ) + # # TODO: define your new data types + # # see https://pynwb.readthedocs.io/en/latest/extensions.html#extending-nwb + # # for more information + # hed_annotations = NWBDatasetSpec( + # neurodata_type_def="HedAnnotations", + # neurodata_type_inc="VectorData", + # doc=("An extension of VectorData for Hierarchical Event Descriptor (HED) tags. If HED tags are used, " + # "the HED schema version must be specified in the NWB file using the HedVersion type."), + # dtype="text", + # attributes=[ + # NWBAttributeSpec( + # name='sub_name', + # dtype='text', + # doc=('The smallest possible difference between two event times. Usually 1 divided by the event time ' + # 'sampling rate on the data acquisition system.'), + # required=False, + # ), + # ], + # ) hed_version = NWBGroupSpec( neurodata_type_def="HedVersion", @@ -65,14 +65,15 @@ def main(): "Required if HED tags are used in the NWB file." ), dtype='text', - required=True, - shape=[None,] + required=True + #shape=[None,] ) ], ) # TODO: add all of your new data types to this list - new_data_types = [hed_annotations, hed_version] + # new_data_types = [hed_annotations, hed_version] + new_data_types = [hed_version] # export the spec to yaml files in the spec folder output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "spec")) @@ -82,4 +83,4 @@ def main(): if __name__ == "__main__": # usage: python create_extension_spec.py - main() + main() \ No newline at end of file From c8d132838051797b6f3430669ed15af069a30cc9 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:45:27 -0600 Subject: [PATCH 2/5] Another try at the roundtrip --- spec/ndx-hed.extensions.yaml | 18 ++ spec/ndx-hed.namespace.yaml | 3 + src/pynwb/ndx_hed/__init__.py | 5 +- .../{hed_annotations.py => hed_tags.py} | 88 ++++--- src/pynwb/ndx_hed/hed_version.py | 47 ++++ src/pynwb/ndx_hed/hed_version1.py | 47 ++++ src/pynwb/tests/test_hed_annotations.py | 222 ------------------ src/pynwb/tests/test_hed_tags.py | 190 +++++++++++++++ src/pynwb/tests/test_hed_version.py | 140 +++++++++++ src/pynwb/tests/test_hed_version1.py | 140 +++++++++++ src/spec/create_extension_spec.py | 55 +++-- 11 files changed, 662 insertions(+), 293 deletions(-) rename src/pynwb/ndx_hed/{hed_annotations.py => hed_tags.py} (54%) create mode 100644 src/pynwb/ndx_hed/hed_version.py create mode 100644 src/pynwb/ndx_hed/hed_version1.py delete mode 100644 src/pynwb/tests/test_hed_annotations.py create mode 100644 src/pynwb/tests/test_hed_tags.py create mode 100644 src/pynwb/tests/test_hed_version.py create mode 100644 src/pynwb/tests/test_hed_version1.py diff --git a/spec/ndx-hed.extensions.yaml b/spec/ndx-hed.extensions.yaml index dcede8a..fa094f3 100644 --- a/spec/ndx-hed.extensions.yaml +++ b/spec/ndx-hed.extensions.yaml @@ -1,3 +1,10 @@ +datasets: +- neurodata_type_def: HedTags + neurodata_type_inc: VectorData + dtype: text + doc: An extension of VectorData for Hierarchical Event Descriptor (HED) tags. If + HED tags are used, the HED schema version must be specified in the NWB file using + the HedMetadata type. groups: - neurodata_type_def: HedVersion neurodata_type_inc: LabMetaData @@ -10,3 +17,14 @@ groups: dtype: text doc: The version of the HED schema used to validate the HED tags, e.g., '8.2.0'. Required if HED tags are used in the NWB file. +- neurodata_type_def: HedVersion1 + neurodata_type_inc: LabMetaData + name: hed_version1 + doc: An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) + schema version. TODO When merged with core, this will no longer inherit from LabMetaData + but from NWBContainer and be placed optionally in /general. + datasets: + - name: version + dtype: text + doc: HED scheam version to use for this dataset + quantity: '?' diff --git a/spec/ndx-hed.namespace.yaml b/spec/ndx-hed.namespace.yaml index 722403a..a4ad483 100644 --- a/spec/ndx-hed.namespace.yaml +++ b/spec/ndx-hed.namespace.yaml @@ -13,5 +13,8 @@ namespaces: name: ndx-hed schema: - namespace: core + neurodata_types: + - LabMetaData + - VectorData - source: ndx-hed.extensions.yaml version: 0.1.0 diff --git a/src/pynwb/ndx_hed/__init__.py b/src/pynwb/ndx_hed/__init__.py index 08462a3..1c39594 100644 --- a/src/pynwb/ndx_hed/__init__.py +++ b/src/pynwb/ndx_hed/__init__.py @@ -52,4 +52,7 @@ # # # Remove these functions from the package del load_namespaces, get_class -from .hed_annotations import HedVersion + +from .hed_version import HedVersion +from .hed_tags import HedTags +from .hed_version1 import HedVersion1 diff --git a/src/pynwb/ndx_hed/hed_annotations.py b/src/pynwb/ndx_hed/hed_tags.py similarity index 54% rename from src/pynwb/ndx_hed/hed_annotations.py rename to src/pynwb/ndx_hed/hed_tags.py index f544285..690c415 100644 --- a/src/pynwb/ndx_hed/hed_annotations.py +++ b/src/pynwb/ndx_hed/hed_tags.py @@ -3,11 +3,12 @@ from hdmf.utils import docval, getargs, get_docval, popargs from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string from pynwb import register_class -from pynwb.file import LabMetaData +from pynwb.file import LabMetaData, NWBFile +from ndx_hed import HedVersion -@register_class('HedAnnotations', 'ndx-hed') -class HedAnnotations(VectorData): +@register_class('HedTags', 'ndx-hed') +class HedTags(VectorData): """ Column storing HED (Hierarchical Event Descriptors) annotations for a row. A HED string is a comma-separated, and possibly parenthesized list of HED tags selected from a valid HED vocabulary as specified by the @@ -15,12 +16,13 @@ class HedAnnotations(VectorData): """ - __nwbfields__ = ('sub_name') + __nwbfields__ = ('sub_name', '_hed_version') @docval(*get_docval(VectorData.__init__)) def __init__(self, **kwargs): # kwargs['name'] = 'HED' super().__init__(**kwargs) + self._hed_version = None self._init_internal() def _init_internal(self): @@ -31,14 +33,6 @@ def _init_internal(self): """ self.sub_name = "HED" - root = self - # parent = root.parent - # while parent is not None: - # root = parent - # parent = root.parent - # hed_version = parent.get_lab_meta_data("HedVersion") - # if hed_version: - # self.hed_schema = hed_version.get_schema() @docval({'name': 'val', 'type': str, 'doc': 'the value to add to this column. Should be a valid HED string.'}) @@ -69,42 +63,44 @@ def get(self, key): vals = super().get(key) return vals - @docval({'name': 'val', 'type': 'str', 'doc': 'the value to validate'}, - {'name': 'return', 'type': 'list', 'doc': 'list of issues or none'}) - def validate(self, **kwargs): - """Validate this HED string""" - val = getargs('val', kwargs) + @docval({'name': 'return', 'type': 'list', 'doc': 'list of issues or none'}) + def validate(self): + """Validate this VectorData. """ + hed_schema = self.get_hed_version() return True + def get_hed_version(self): + if not self._hed_version: + root = self._get_root() + if isinstance(root, NWBFile): + self._hed_version = root.get_lab_meta_data("HedVersion") + return self._hed_version -@register_class("HedVersion", "ndx-hed") -class HedVersion(LabMetaData): - """ - The class containing the HED versions and HED schema used in this data file. - - """ - - __nwbfields__ = ('name', 'description', 'version', 'schema_string') - - # @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) - @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) - def __init__(self, version): - kwargs = {'name': 'hed_version'} - super().__init__(**kwargs) - self.version = version - self._init_internal() - - def _init_internal(self): - """ - Create a HED schema string - """ - hed_schema = load_schema_version(self.version) - self.schema_string = hed_schema.get_as_xml_string() + def _get_root(self): + root = self + while hasattr(root, 'parent') and root.parent: + root = root.parent + return root + + + # root = parent + # parent = root.parent + # if parent: + # hed_version = parent.get_lab_meta_data("HedVersion") + # else: + # hed_version = None + # if hed_version: + # self.hed_schema = hed_version.get_schema() - @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) - def get_version(self): - return self.version + # root = self + # parent = root.parent + # while parent is not None: + # root = parent + # parent = root.parent + # if parent: + # hed_version = parent.get_lab_meta_data("HedVersion") + # else: + # hed_version = None + # if hed_version: + # self.hed_schema = hed_version.get_schema() - @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) - def get_schema(self): - return from_string(self.schema_string) diff --git a/src/pynwb/ndx_hed/hed_version.py b/src/pynwb/ndx_hed/hed_version.py new file mode 100644 index 0000000..ad7908e --- /dev/null +++ b/src/pynwb/ndx_hed/hed_version.py @@ -0,0 +1,47 @@ +from hdmf.utils import docval, getargs, get_docval, popargs +from pynwb import register_class +from pynwb.file import LabMetaData +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string + + +@register_class("HedVersion", "ndx-hed") +class HedVersion(LabMetaData): + """ + The class containing the HED versions and HED schema used in this data file. + + """ + + __nwbfields__ = ('version', 'schema_string') + + # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}, + # *get_docval(LabMetaData.__init__)) + @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) + def __init__(self, **kwargs): + version = popargs('version', kwargs) + kwargs['name'] = 'hed_version' + super().__init__(**kwargs) + self.version = version + self._init_internal() + + # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) + # def __init__(self, version): + # kwargs = {'name': 'hed_version'} + # super().__init__(**kwargs) + # self.name = 'hed_version' + # self.version = version + # self._init_internal() + + def _init_internal(self): + """ + Create a HED schema string + """ + hed_schema = load_schema_version(self.version) + self.schema_string = hed_schema.get_as_xml_string() + + @docval(returns='The HED schema version', rtype=str) + def get_version(self): + return self.version + + @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) + def get_schema(self): + return from_string(self.schema_string) diff --git a/src/pynwb/ndx_hed/hed_version1.py b/src/pynwb/ndx_hed/hed_version1.py new file mode 100644 index 0000000..dcaa242 --- /dev/null +++ b/src/pynwb/ndx_hed/hed_version1.py @@ -0,0 +1,47 @@ +from hdmf.utils import docval, getargs, get_docval, popargs +from pynwb import register_class +from pynwb.file import LabMetaData +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string + + +@register_class("HedVersion1", "ndx-hed") +class HedVersion1(LabMetaData): + """ + The class containing the HED versions and HED schema used in this data file. + + """ + + __nwbfields__ = ('name', 'version', 'schema_string') + + # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}, + # *get_docval(LabMetaData.__init__)) + # @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) + # def __init__(self, **kwargs): + # version = popargs('version', kwargs) + # kwargs['name'] = 'hed_version' + # super().__init__(**kwargs) + # self.version = version + # self._init_internal() + + @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) + def __init__(self, version): + kwargs = {'name': 'hed_version1'} + super().__init__(**kwargs) + self.name = 'hed_version1' + self.version = version + self._init_internal() + + def _init_internal(self): + """ + Create a HED schema string + """ + hed_schema = load_schema_version(self.version) + self.schema_string = hed_schema.get_as_xml_string() + + @docval(returns='The HED schema version', rtype=str) + def get_version(self): + return self.version + + @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) + def get_schema(self): + return from_string(self.schema_string) diff --git a/src/pynwb/tests/test_hed_annotations.py b/src/pynwb/tests/test_hed_annotations.py deleted file mode 100644 index 70d23c7..0000000 --- a/src/pynwb/tests/test_hed_annotations.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Unit and integration tests for ndx-hed.""" -from datetime import datetime -from dateutil.tz import tzlocal, tzutc -from uuid import uuid4, UUID -from hed.schema import HedSchema, HedSchemaGroup -from pynwb import NWBHDF5IO, get_manager, NWBFile -from pynwb.testing.mock.file import mock_NWBFile -from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin -from src.pynwb.ndx_hed import HedVersion - - -class TestHedVersion(TestCase): - """Simple unit test for creating a HedMetadata.""" - - def test_constructor(self): - """Test setting HedNWBFile values using the constructor.""" - hed_version1 = HedVersion("8.2.0") - self.assertIsInstance(hed_version1, HedVersion) - self.assertIsInstance(hed_version1.version, str) - # hed_version2 = HedVersion(["8.2.0"]) - # self.assertIsInstance(hed_version2.version, list) - print(f"HED version: {str(hed_version1.version)}") - - # schema1 = hed_version1.get_hed_schema() - # # self.assertIsInstance(schema1, HedSchema) - # hed_version2 = HedVersion(["8.2.0", "sc:score_1.0.0"]) - - def test_get_schema(self): - hed_version1 = HedVersion("8.2.0") - hed_schema = hed_version1.get_schema() - self.assertIsInstance(hed_schema, HedSchema) - - def test_get_version(self): - hed_version1 = HedVersion("8.2.0") - version = hed_version1.get_version() - self.assertIsInstance(version, str) - self.assertEqual(version, "8.2.0") - - def test_add_to_nwbfile(self): - nwbfile = mock_NWBFile() - #hed_version1 = HedVersion(["8.2.0"]) - hed_version1 = HedVersion("8.2.0") - nwbfile.add_lab_meta_data(hed_version1) - hed_version2 = nwbfile.get_lab_meta_data("hed_version") - self.assertIsInstance(hed_version2, HedVersion) - - -class TestHedNWBFileSimpleRoundtrip(TestCase): - """Simple roundtrip test for HedNWBFile.""" - - def setUp(self): - self.path = "test.nwb" - - def tearDown(self): - remove_test_file(self.path) - - def test_roundtrip(self): - """ - Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. - """ - nwbfile = mock_NWBFile() - # hed_version = HedVersion(["8.2.0"]) - hed_version = HedVersion("8.2.0") - nwbfile.add_lab_meta_data(hed_version) - print(f"nwb: {str(nwbfile)}") - print(f"{str(hed_version)}") - - with NWBHDF5IO(self.path, mode="w") as io: - io.write(nwbfile) - - with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: - read_nwbfile = io.read() - read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") - self.assertIsInstance(read_hed_version, HedVersion) - self.assertEqual(read_hed_version.version, "8.2.0") - - -# class TestHedNWBFileRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): -# """Complex, more complete roundtrip test for HedNWBFile using pynwb.testing infrastructure.""" -# -# def getContainerType(self): -# return "NWBFile" -# -# def addContainer(self): -# self.nwbfile = HedNWBFile( -# session_description="session_description", -# identifier=str(uuid4()), -# session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), -# hed_schema_version="8.2.0", -# ) -# -# def getContainer(self, nwbfile: NWBFile): -# return nwbfile - - -# class TestHedAnnotationsConstructor(TestCase): -# """Simple unit test for creating a HedTags.""" -# -# def test_constructor(self): -# """Test setting HED values using the constructor.""" -# hed_annotations = HedAnnotations( -# name='HED', -# description="description", -# data=["animal_target, correct_response", "animal_target, incorrect_response"], -# ) -# print(f"{hed_annotations}") -# self.assertEqual(hed_annotations.sub_name, "HED") -# self.assertEqual(hed_annotations.description, "description") -# self.assertEqual(hed_annotations.data,["animal_target, correct_response", "animal_target, incorrect_response"]) -# -# def test_add_to_trials_table(self): -# """Test adding HED column and data to a trials table.""" -# nwbfile = mock_NWBFile() -# hed_version = HedVersion(["8.2.0"]) -# nwbfile.add_lab_meta_data(hed_version) -# #nwbfile.add_trial_column("HED", "HED annotations for each trial") -# nwbfile.add_trial_column(name="HED", description="HED annotations for each trial", -# col_cls=HedAnnotations, data=[]) -# nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="animal_target, correct_response") -# nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="animal_target, incorrect_response") -# -# self.assertIsInstance(nwbfile.trials["HED"], HedAnnotations) -# print(f"{str(nwbfile)}") -# print(f"{str(nwbfile.trials)}") -# hed_col = nwbfile.trials["HED"] -# print(f"hed_col: {str(hed_col.data)}") -# self.assertEqual(nwbfile.trials["HED"].data[0], "animal_target, correct_response") -# self.assertEqual(nwbfile.trials["HED"].data[1], "animal_target, incorrect_response") - -class TestHedTagsSimpleRoundtrip(TestCase): - """Simple roundtrip test for HedTags.""" - - def setUp(self): - self.path = "test.nwb" - self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) - self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) - self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) - self.manager = get_manager() - self.filename = 'test_nwbfileio.h5' - self.nwbfile = NWBFile(session_description='a test NWB File', - identifier='TEST123', - session_start_time=self.start_time, - timestamps_reference_time=self.ref_time, - file_create_date=self.create_date, - experimenter='test experimenter', - stimulus_notes='test stimulus notes', - data_collection='test data collection notes', - experiment_description='test experiment description', - institution='nomad', - lab='nolab', - notes='nonotes', - pharmacology='nopharmacology', - protocol='noprotocol', - related_publications='nopubs', - session_id='007', - slices='noslices', - source_script='nosources', - surgery='nosurgery', - virus='novirus', - source_script_file_name='nofilename') - - def tearDown(self): - remove_test_file(self.path) - - def test_hed_version(self): - hed1 = HedVersion("8.2.0") - self.assertIsInstance(hed1, HedVersion) - schema = hed1.get_schema() - self.assertIsInstance(schema, HedSchema) - -# def test_roundtrip(self): -# """ -# Add a HedTags to an NWBFile, write it to file, read the file, and test that the HedTags from the -# file matches the original HedTags. -# """ -# hed_version = HedVersion(["8.2.0"]) -# self.nwbfile.add_lab_meta_data(hed_version) -# -# self.nwbfile.add_trial_column("HED", "HED annotations for each trial") -# self.nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="animal_target, correct_response") -# self.nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="animal_target, incorrect_response") -# print("to here") -# # nwbfile = mock_NWBFile() -# # hed_version = HedVersion(hed_schema_version="8.2.0") -# # nwbfile.add_lab_meta_data(hed_version) -# # -# # nwbfile.add_trial_column("HED", "HED annotations for each trial", col_cls=HedAnnotations) -# # nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="animal_target, correct_response") -# # nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="animal_target, incorrect_response") -# -# # with NWBHDF5IO(self.path, mode="w") as io: -# # io.write(nwbfile) -# # -# # with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: -# # read_nwbfile = io.read() -# # read_hed_annotations = read_nwbfile.trials["HED"] -# # assert isinstance(read_hed_annotations, HedAnnotations) -# # # read_nwbfile.trials["hed_tags"][0] is read as a numpy array -# # assert read_hed_annotations[0] == "animal_target, correct_response" -# # assert read_hed_annotations[1] == "animal_target, incorrect_response" -# -# -# # class TestHedTagsRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): -# # """Complex, more complete roundtrip test for HedTags using pynwb.testing infrastructure.""" -# -# # def getContainerType(self): -# # return "HedTags" -# -# # def addContainer(self): -# # self.nwbfile = HedNWBFile( -# # session_description="session_description", -# # identifier=str(uuid4()), -# # session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), -# # hed_schema_version="8.2.0", -# # ) -# -# # self.nwbfile.add_trial_column("hed_tags", "HED tags for each trial", col_cls=HedTags, index=True) -# # self.nwbfile.add_trial(start_time=0.0, stop_time=1.0, hed_tags=["animal_target", "correct_response"]) -# # self.nwbfile.add_trial(start_time=2.0, stop_time=3.0, hed_tags=["animal_target", "incorrect_response"]) -# -# # def getContainer(self, nwbfile: NWBFile): -# # return nwbfile.trials["hed_tags"].target diff --git a/src/pynwb/tests/test_hed_tags.py b/src/pynwb/tests/test_hed_tags.py new file mode 100644 index 0000000..743fdd8 --- /dev/null +++ b/src/pynwb/tests/test_hed_tags.py @@ -0,0 +1,190 @@ +"""Unit and integration tests for ndx-hed.""" +from datetime import datetime, timezone +from dateutil.tz import tzlocal, tzutc +from uuid import uuid4, UUID +from hed.schema import HedSchema, HedSchemaGroup +from pynwb import NWBHDF5IO, get_manager, NWBFile +from pynwb.testing.mock.file import mock_NWBFile +from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin +from src.pynwb.ndx_hed import HedVersion, HedTags + + +class TestHedAnnotationsConstructor(TestCase): + """Simple unit test for creating a HedTags.""" + + def test_constructor(self): + """Test setting HED values using the constructor.""" + hed_annotations = HedTags( + name='HED', + description="description", + data=["Correct-action", "Incorrect-action"], + ) + self.assertEqual(hed_annotations.sub_name, "HED") + self.assertEqual(hed_annotations.description, "description") + self.assertEqual(hed_annotations.data, ["Correct-action", "Incorrect-action"]) + + def test_add_to_trials_table(self): + """Test adding HED column and data to a trials table.""" + nwbfile = mock_NWBFile() + hed_version = HedVersion("8.2.0") + nwbfile.add_lab_meta_data(hed_version) + nwbfile.add_trial_column(name="HED", description="HED annotations for each trial", + col_cls=HedTags, data=[]) + nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="Correct-action") + nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="Incorrect-action") + self.assertIsInstance(nwbfile.trials["HED"], HedTags) + hed_col = nwbfile.trials["HED"] + self.assertEqual(nwbfile.trials["HED"].data[0], "Correct-action") + self.assertEqual(nwbfile.trials["HED"].data[1], "Incorrect-action") + + +# class TestHedTagsSimpleRoundtrip(TestCase): +# """Simple roundtrip test for HedNWBFile.""" +# +# def setUp(self): +# self.path = "test.nwb" +# +# def tearDown(self): +# remove_test_file(self.path) +# +# def test_roundtrip(self): +# """ +# Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. +# """ +# nwbfile = mock_NWBFile() +# # hed_version = HedVersion(["8.2.0"]) +# hed_version = HedVersion("8.2.0") +# nwbfile.add_lab_meta_data(hed_version) +# +# with NWBHDF5IO(self.path, mode="w") as io: +# io.write(nwbfile) +# +# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: +# read_nwbfile = io.read() +# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") +# self.assertIsInstance(read_hed_version, HedVersion) +# self.assertEqual(read_hed_version.version, "8.2.0") + + +class TestHedTagsNWBFileRoundtrip(TestCase): + """Simple roundtrip test for HedTags.""" + + def setUp(self): + self.path = "test.nwb" + self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) + self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) + self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) + self.manager = get_manager() + self.filename = 'test_nwbfileio.h5' + self.nwbfile = NWBFile(session_description='a test NWB File', + identifier='TEST123', + session_start_time=self.start_time, + timestamps_reference_time=self.ref_time, + file_create_date=self.create_date, + experimenter='test experimenter', + stimulus_notes='test stimulus notes', + data_collection='test data collection notes', + experiment_description='test experiment description', + institution='nomad', + lab='nolab', + notes='nonotes', + pharmacology='nopharmacology', + protocol='noprotocol', + related_publications='nopubs', + session_id='007', + slices='noslices', + source_script='nosources', + surgery='nosurgery', + virus='novirus', + source_script_file_name='nofilename') + hed_version = HedVersion("8.2.0") + self.nwbfile.add_lab_meta_data(hed_version) + self.nwbfile.add_trial_column(name="HED", description="HED annotations for each trial", + col_cls=HedTags, data=[]) + self.nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="Correct-action") + self.nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="Incorrect-action") + + def tearDown(self): + remove_test_file(self.path) + + def test_roundtrip(self): + """ + Add a HedTags to an NWBFile, write it to file, read the file, and test that the HedTags from the + file matches the original HedTags. + """ + + with NWBHDF5IO(self.path, mode="w") as io: + io.write(self.nwbfile) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + lab_metadata = read_nwbfile.get_lab_meta_data() + print(f"lab: {str(lab_metadata)}") + read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") + print(f"hed: {read_hed_version}") + # self.assertIsInstance(read_hed_version, HedVersion) + # self.assertEqual(read_hed_version.version, "8.2.0") + # self.assertEqual(read_nwbfile.trials["HED"].data[0], "Correct-action") + # self.assertEqual(read_nwbfile.trials["HED"].data[1], "Incorrect-action") + +# +# # class TestHedTagsRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): +# # """Complex, more complete roundtrip test for HedTags using pynwb.testing infrastructure.""" +# +# # def getContainerType(self): +# # return "HedTags" +# +# # def addContainer(self): +# # self.nwbfile = HedNWBFile( +# # session_description="session_description", +# # identifier=str(uuid4()), +# # session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), +# # hed_schema_version="8.2.0", +# # ) +# +# # self.nwbfile.add_trial_column("hed_tags", "HED tags for each trial", col_cls=HedTags, index=True) +# # self.nwbfile.add_trial(start_time=0.0, stop_time=1.0, hed_tags=["animal_target", "correct_response"]) +# # self.nwbfile.add_trial(start_time=2.0, stop_time=3.0, hed_tags=["animal_target", "incorrect_response"]) +# +# # def getContainer(self, nwbfile: NWBFile): +# # return nwbfile.trials["hed_tags"].target + + + +# class TestHedNWBFileRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): +# """Complex, more complete roundtrip test for HedNWBFile using pynwb.testing infrastructure.""" +# +# def setUp(self): +# self.nwbfile = NWBFile( +# session_description='session_description', +# identifier='identifier', +# session_start_time=datetime.now(timezone.utc) +# ) +# self.filename = "test.nwb" +# self.export_filename = "test_export.nwb" +# +# def tearDown(self): +# remove_test_file(self.filename) +# remove_test_file(self.export_filename) +# +# def addContainer(self): +# """ Add the test ElectricalSeries and related objects to the given NWBFile """ +# pass +# +# def getContainer(self, nwbfile: NWBFile): +# # return nwbfile.acquisition['test_eS'] +# return None +# +# def test_roundtrip(self): +# hed_version = HedVersion("8.2.0") +# self.nwbfile.add_lab_meta_data(hed_version) +# +# with NWBHDF5IO(self.filename, mode='w') as io: +# io.write(self.nwbfile) +# +# with NWBHDF5IO(self.filename, mode='r', load_namespaces=True) as io: +# read_nwbfile = io.read() +# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") +# self.assertIsInstance(read_hed_version, HedVersion) +# self.assertEqual(read_hed_version.version, "8.2.0") + diff --git a/src/pynwb/tests/test_hed_version.py b/src/pynwb/tests/test_hed_version.py new file mode 100644 index 0000000..7c17083 --- /dev/null +++ b/src/pynwb/tests/test_hed_version.py @@ -0,0 +1,140 @@ +"""Unit and integration tests for ndx-hed.""" +from datetime import datetime, timezone +from dateutil.tz import tzlocal, tzutc +from uuid import uuid4, UUID +from pynwb.file import LabMetaData +from pynwb import NWBHDF5IO, get_manager, NWBFile +from pynwb.testing.mock.file import mock_NWBFile +from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin +from src.pynwb.ndx_hed import HedVersion +from hed.schema import HedSchema, HedSchemaGroup + + +class TestHedVersion(TestCase): + """Simple unit test for creating a HedMetadata.""" + + def test_constructor(self): + """Test setting HedNWBFile values using the constructor.""" + hed_version1 = HedVersion("8.2.0") + self.assertIsInstance(hed_version1, HedVersion) + self.assertIsInstance(hed_version1, LabMetaData) + self.assertEqual(hed_version1.name, "hed_version") + self.assertIsInstance(hed_version1.version, str) + self.assertEqual(hed_version1.version, "8.2.0") + self.assertIsInstance(hed_version1.schema_string, str) + # hed_version2 = HedVersion(["8.2.0"]) + # self.assertIsInstance(hed_version2.version, list) + # hed_version3 = HedVersion(["8.2.0", "sc:score_1.0.0"]) + + def test_get_schema(self): + hed_version1 = HedVersion(version="8.2.0") + hed_schema = hed_version1.get_schema() + self.assertIsInstance(hed_schema, HedSchema) + + def test_get_version(self): + hed_version1 = HedVersion(version="8.2.0") + version = hed_version1.get_version() + self.assertIsInstance(version, str) + self.assertEqual(version, "8.2.0") + + def test_add_to_nwbfile(self): + nwbfile = mock_NWBFile() + hed_version1 = HedVersion(version="8.2.0") + nwbfile.add_lab_meta_data(hed_version1) + hed_version2 = nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(hed_version2, HedVersion) + # hed_version2 = HedVersion(["8.2.0"]) + + +class TestHedVersionSimpleRoundtrip(TestCase): + """Simple roundtrip test for HedNWBFile.""" + + def setUp(self): + self.path = "test.nwb" + + def tearDown(self): + remove_test_file(self.path) + + def test_roundtrip(self): + """ + Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. + """ + nwbfile = mock_NWBFile() + # hed_version = HedVersion(["8.2.0"]) + hed_version = HedVersion(version="8.2.0") + self.assertIsInstance(hed_version, HedVersion) + self.assertIsInstance(hed_version, LabMetaData) + nwbfile.add_lab_meta_data(lab_meta_data=hed_version) + meta = nwbfile.get_lab_meta_data("hed_version") + self.assertEqual(meta.name, "hed_version") + self.assertEqual(meta.version, "8.2.0") + self.assertIsInstance(meta, HedVersion) + self.assertIsInstance(meta, LabMetaData) + schema = meta.get_schema() + self.assertIsInstance(schema, HedSchema) + + with NWBHDF5IO(self.path, mode="w") as io: + io.write(nwbfile) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + read_hed_version = read_nwbfile.get_lab_meta_data(hed_version.name) + self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") + + +# class TestHedVersionNWBFileSimpleRoundtrip(TestCase): +# """Simple roundtrip test for HedTags.""" +# +# def setUp(self): +# self.path = "test.nwb" +# self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) +# self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) +# self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) +# self.manager = get_manager() +# self.filename = 'test_nwbfileio.h5' +# self.nwbfile = NWBFile(session_description='a test NWB File', +# identifier='TEST123', +# session_start_time=self.start_time, +# timestamps_reference_time=self.ref_time, +# file_create_date=self.create_date, +# experimenter='test experimenter', +# stimulus_notes='test stimulus notes', +# data_collection='test data collection notes', +# experiment_description='test experiment description', +# institution='nomad', +# lab='nolab', +# notes='nonotes', +# pharmacology='nopharmacology', +# protocol='noprotocol', +# related_publications='nopubs', +# session_id='007', +# slices='noslices', +# source_script='nosources', +# surgery='nosurgery', +# virus='novirus', +# source_script_file_name='nofilename') +# hed_version = HedVersion(version="8.2.0") +# self.nwbfile.add_lab_meta_data(hed_version) +# print(f"Name:{hed_version.name}") +# +# def tearDown(self): +# remove_test_file(self.path) +# +# def test_version(self): +# hed_version = self.nwbfile.get_lab_meta_data("hed_version") +# self.assertIsInstance(hed_version, HedVersion) +# schema = hed_version.get_schema() +# self.assertIsInstance(schema, HedSchema) +# +# def test_roundtrip(self): +# print(self.nwbfile) +# with NWBHDF5IO(self.path, mode="w") as io: +# io.write(self.nwbfile) +# +# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: +# read_nwbfile = io.read() +# print(read_nwbfile) +# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") +# #self.assertIsInstance(read_hed_version, HedVersion) +# self.assertEqual(read_hed_version.version, "8.2.0") diff --git a/src/pynwb/tests/test_hed_version1.py b/src/pynwb/tests/test_hed_version1.py new file mode 100644 index 0000000..2b6413b --- /dev/null +++ b/src/pynwb/tests/test_hed_version1.py @@ -0,0 +1,140 @@ +"""Unit and integration tests for ndx-hed.""" +from datetime import datetime, timezone +from dateutil.tz import tzlocal, tzutc +from uuid import uuid4, UUID +from pynwb.file import LabMetaData +from pynwb import NWBHDF5IO, get_manager, NWBFile +from pynwb.testing.mock.file import mock_NWBFile +from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin +from src.pynwb.ndx_hed import HedVersion1 +from hed.schema import HedSchema, HedSchemaGroup + + +class TestHedVersion1(TestCase): + """Simple unit test for creating a HedMetadata.""" + + def test_constructor(self): + """Test setting HedNWBFile values using the constructor.""" + hed_version1 = HedVersion1("8.2.0") + self.assertIsInstance(hed_version1, HedVersion1) + self.assertIsInstance(hed_version1, LabMetaData) + self.assertEqual(hed_version1.name, "hed_version1") + self.assertIsInstance(hed_version1.version, str) + self.assertEqual(hed_version1.version, "8.2.0") + self.assertIsInstance(hed_version1.schema_string, str) + # hed_version2 = HedVersion(["8.2.0"]) + # self.assertIsInstance(hed_version2.version, list) + # hed_version3 = HedVersion(["8.2.0", "sc:score_1.0.0"]) + + def test_get_schema(self): + hed_version1 = HedVersion1("8.2.0") + hed_schema = hed_version1.get_schema() + self.assertIsInstance(hed_schema, HedSchema) + + def test_get_version(self): + hed_version1 = HedVersion1("8.2.0") + version = hed_version1.get_version() + self.assertIsInstance(version, str) + self.assertEqual(version, "8.2.0") + + def test_add_to_nwbfile(self): + nwbfile = mock_NWBFile() + hed_version1 = HedVersion1("8.2.0") + nwbfile.add_lab_meta_data(hed_version1) + hed_version2 = nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(hed_version2, HedVersion1) + # hed_version2 = HedVersion(["8.2.0"]) + + +class TestHedVersion1SimpleRoundtrip(TestCase): + """Simple roundtrip test for HedNWBFile.""" + + def setUp(self): + self.path = "test.nwb" + + def tearDown(self): + remove_test_file(self.path) + + def test_roundtrip(self): + """ + Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. + """ + nwbfile = mock_NWBFile() + # hed_version = HedVersion(["8.2.0"]) + hed_version = HedVersion(version="8.2.0") + self.assertIsInstance(hed_version, HedVersion) + self.assertIsInstance(hed_version, LabMetaData) + nwbfile.add_lab_meta_data(lab_meta_data=hed_version) + meta = nwbfile.get_lab_meta_data("hed_version") + self.assertEqual(meta.name, "hed_version") + self.assertEqual(meta.version, "8.2.0") + self.assertIsInstance(meta, HedVersion) + self.assertIsInstance(meta, LabMetaData) + schema = meta.get_schema() + self.assertIsInstance(schema, HedSchema) + + with NWBHDF5IO(self.path, mode="w") as io: + io.write(nwbfile) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + read_hed_version = read_nwbfile.get_lab_meta_data(hed_version.name) + self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") + + +# class TestHedVersionNWBFileSimpleRoundtrip(TestCase): +# """Simple roundtrip test for HedTags.""" +# +# def setUp(self): +# self.path = "test.nwb" +# self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) +# self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) +# self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) +# self.manager = get_manager() +# self.filename = 'test_nwbfileio.h5' +# self.nwbfile = NWBFile(session_description='a test NWB File', +# identifier='TEST123', +# session_start_time=self.start_time, +# timestamps_reference_time=self.ref_time, +# file_create_date=self.create_date, +# experimenter='test experimenter', +# stimulus_notes='test stimulus notes', +# data_collection='test data collection notes', +# experiment_description='test experiment description', +# institution='nomad', +# lab='nolab', +# notes='nonotes', +# pharmacology='nopharmacology', +# protocol='noprotocol', +# related_publications='nopubs', +# session_id='007', +# slices='noslices', +# source_script='nosources', +# surgery='nosurgery', +# virus='novirus', +# source_script_file_name='nofilename') +# hed_version = HedVersion(version="8.2.0") +# self.nwbfile.add_lab_meta_data(hed_version) +# print(f"Name:{hed_version.name}") +# +# def tearDown(self): +# remove_test_file(self.path) +# +# def test_version(self): +# hed_version = self.nwbfile.get_lab_meta_data("hed_version") +# self.assertIsInstance(hed_version, HedVersion) +# schema = hed_version.get_schema() +# self.assertIsInstance(schema, HedSchema) +# +# def test_roundtrip(self): +# print(self.nwbfile) +# with NWBHDF5IO(self.path, mode="w") as io: +# io.write(self.nwbfile) +# +# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: +# read_nwbfile = io.read() +# print(read_nwbfile) +# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") +# #self.assertIsInstance(read_hed_version, HedVersion) +# self.assertEqual(read_hed_version.version, "8.2.0") diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index 3e03fbf..d088d02 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -28,31 +28,21 @@ def main(): ) ns_builder.include_namespace("core") + ns_builder.include_type('LabMetaData', namespace='core') + ns_builder.include_type('VectorData', namespace='core') - # # TODO: define your new data types - # # see https://pynwb.readthedocs.io/en/latest/extensions.html#extending-nwb - # # for more information - # hed_annotations = NWBDatasetSpec( - # neurodata_type_def="HedAnnotations", - # neurodata_type_inc="VectorData", - # doc=("An extension of VectorData for Hierarchical Event Descriptor (HED) tags. If HED tags are used, " - # "the HED schema version must be specified in the NWB file using the HedVersion type."), - # dtype="text", - # attributes=[ - # NWBAttributeSpec( - # name='sub_name', - # dtype='text', - # doc=('The smallest possible difference between two event times. Usually 1 divided by the event time ' - # 'sampling rate on the data acquisition system.'), - # required=False, - # ), - # ], - # ) + hed_tags = NWBDatasetSpec( + neurodata_type_def="HedTags", + neurodata_type_inc="VectorData", + doc=("An extension of VectorData for Hierarchical Event Descriptor (HED) tags. If HED tags are used, " + "the HED schema version must be specified in the NWB file using the HedMetadata type."), + dtype="text", + ) - hed_version = NWBGroupSpec( + hed_version_ext = NWBGroupSpec( + name="hed_version", # fixed name neurodata_type_def="HedVersion", neurodata_type_inc="LabMetaData", - name="hed_version", # fixed name doc=("An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) schema version. " "TODO When merged with core, " "this will no longer inherit from LabMetaData but from NWBContainer and be placed " @@ -68,12 +58,29 @@ def main(): required=True #shape=[None,] ) - ], + ] + ) + + + hed_version1_ext = NWBGroupSpec( + name="hed_version1", # fixed name + neurodata_type_def="HedVersion1", + neurodata_type_inc="LabMetaData", + doc=("An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) schema version. " + "TODO When merged with core, " + "this will no longer inherit from LabMetaData but from NWBContainer and be placed " + "optionally in /general."), ) + hed_version1_ext.add_dataset( + name="version", + doc="HED scheam version to use for this dataset", + dtype='text', + quantity='?' + ) + # TODO: add all of your new data types to this list - # new_data_types = [hed_annotations, hed_version] - new_data_types = [hed_version] + new_data_types = [hed_version_ext, hed_tags, hed_version1_ext] # export the spec to yaml files in the spec folder output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "spec")) From 8a4e974261c14d86cac2743cb1e973c8c55c5b50 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Wed, 17 Jan 2024 08:46:01 -0600 Subject: [PATCH 3/5] Started the readme --- src/pynwb/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pynwb/README.md b/src/pynwb/README.md index e69de29..1b7b10e 100644 --- a/src/pynwb/README.md +++ b/src/pynwb/README.md @@ -0,0 +1 @@ +## ndx_hed NWB extension \ No newline at end of file From 5aab6b6cd477d3c0b48f2a829a90706e3531a0cb Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:30:51 -0600 Subject: [PATCH 4/5] Worked with adding dataset rather than attributes --- src/pynwb/ndx_hed/hed_version.py | 2 +- src/pynwb/ndx_hed/hed_version1.py | 24 ++--- src/pynwb/tests/test_hed_version1.py | 127 ++++++++++++++------------- src/spec/create_extension_spec.py | 3 +- 4 files changed, 78 insertions(+), 78 deletions(-) diff --git a/src/pynwb/ndx_hed/hed_version.py b/src/pynwb/ndx_hed/hed_version.py index ad7908e..c88444e 100644 --- a/src/pynwb/ndx_hed/hed_version.py +++ b/src/pynwb/ndx_hed/hed_version.py @@ -15,7 +15,7 @@ class HedVersion(LabMetaData): # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}, # *get_docval(LabMetaData.__init__)) - @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) + @docval({'name': 'version', 'type': str, 'doc': 'HED strings of type str'}) def __init__(self, **kwargs): version = popargs('version', kwargs) kwargs['name'] = 'hed_version' diff --git a/src/pynwb/ndx_hed/hed_version1.py b/src/pynwb/ndx_hed/hed_version1.py index dcaa242..d5d7a3f 100644 --- a/src/pynwb/ndx_hed/hed_version1.py +++ b/src/pynwb/ndx_hed/hed_version1.py @@ -15,22 +15,22 @@ class HedVersion1(LabMetaData): # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}, # *get_docval(LabMetaData.__init__)) - # @docval({'name': 'version', 'type': (str, list), 'doc': 'HED strings of type str'}) - # def __init__(self, **kwargs): - # version = popargs('version', kwargs) - # kwargs['name'] = 'hed_version' - # super().__init__(**kwargs) - # self.version = version - # self._init_internal() - - @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) - def __init__(self, version): - kwargs = {'name': 'hed_version1'} + @docval({'name': 'version', 'type': str, 'doc': 'HED strings of type str'}) + def __init__(self, **kwargs): + version = popargs('version', kwargs) + kwargs['name'] = 'hed_version1_ext' super().__init__(**kwargs) - self.name = 'hed_version1' self.version = version self._init_internal() + # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) + # def __init__(self, version): + # kwargs = {'name': 'hed_version1'} + # super().__init__(**kwargs) + # self.name = 'hed_version1' + # self.version = version + # self._init_internal() + def _init_internal(self): """ Create a HED schema string diff --git a/src/pynwb/tests/test_hed_version1.py b/src/pynwb/tests/test_hed_version1.py index 2b6413b..d16178c 100644 --- a/src/pynwb/tests/test_hed_version1.py +++ b/src/pynwb/tests/test_hed_version1.py @@ -18,7 +18,7 @@ def test_constructor(self): hed_version1 = HedVersion1("8.2.0") self.assertIsInstance(hed_version1, HedVersion1) self.assertIsInstance(hed_version1, LabMetaData) - self.assertEqual(hed_version1.name, "hed_version1") + self.assertEqual(hed_version1.name, "hed_version1_ext") self.assertIsInstance(hed_version1.version, str) self.assertEqual(hed_version1.version, "8.2.0") self.assertIsInstance(hed_version1.schema_string, str) @@ -41,7 +41,7 @@ def test_add_to_nwbfile(self): nwbfile = mock_NWBFile() hed_version1 = HedVersion1("8.2.0") nwbfile.add_lab_meta_data(hed_version1) - hed_version2 = nwbfile.get_lab_meta_data("hed_version") + hed_version2 = nwbfile.get_lab_meta_data("hed_version1_ext") self.assertIsInstance(hed_version2, HedVersion1) # hed_version2 = HedVersion(["8.2.0"]) @@ -61,14 +61,15 @@ def test_roundtrip(self): """ nwbfile = mock_NWBFile() # hed_version = HedVersion(["8.2.0"]) - hed_version = HedVersion(version="8.2.0") - self.assertIsInstance(hed_version, HedVersion) + hed_version = HedVersion1(version="8.2.0") + print(f"name={hed_version.name}") + self.assertIsInstance(hed_version, HedVersion1) self.assertIsInstance(hed_version, LabMetaData) nwbfile.add_lab_meta_data(lab_meta_data=hed_version) - meta = nwbfile.get_lab_meta_data("hed_version") - self.assertEqual(meta.name, "hed_version") + meta = nwbfile.get_lab_meta_data("hed_version1_ext") + self.assertEqual(meta.name, "hed_version1_ext") self.assertEqual(meta.version, "8.2.0") - self.assertIsInstance(meta, HedVersion) + self.assertIsInstance(meta, HedVersion1) self.assertIsInstance(meta, LabMetaData) schema = meta.get_schema() self.assertIsInstance(schema, HedSchema) @@ -79,62 +80,62 @@ def test_roundtrip(self): with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: read_nwbfile = io.read() read_hed_version = read_nwbfile.get_lab_meta_data(hed_version.name) - self.assertIsInstance(read_hed_version, HedVersion) + self.assertIsInstance(read_hed_version, HedVersion1) self.assertEqual(read_hed_version.version, "8.2.0") -# class TestHedVersionNWBFileSimpleRoundtrip(TestCase): -# """Simple roundtrip test for HedTags.""" -# -# def setUp(self): -# self.path = "test.nwb" -# self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) -# self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) -# self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) -# self.manager = get_manager() -# self.filename = 'test_nwbfileio.h5' -# self.nwbfile = NWBFile(session_description='a test NWB File', -# identifier='TEST123', -# session_start_time=self.start_time, -# timestamps_reference_time=self.ref_time, -# file_create_date=self.create_date, -# experimenter='test experimenter', -# stimulus_notes='test stimulus notes', -# data_collection='test data collection notes', -# experiment_description='test experiment description', -# institution='nomad', -# lab='nolab', -# notes='nonotes', -# pharmacology='nopharmacology', -# protocol='noprotocol', -# related_publications='nopubs', -# session_id='007', -# slices='noslices', -# source_script='nosources', -# surgery='nosurgery', -# virus='novirus', -# source_script_file_name='nofilename') -# hed_version = HedVersion(version="8.2.0") -# self.nwbfile.add_lab_meta_data(hed_version) -# print(f"Name:{hed_version.name}") -# -# def tearDown(self): -# remove_test_file(self.path) -# -# def test_version(self): -# hed_version = self.nwbfile.get_lab_meta_data("hed_version") -# self.assertIsInstance(hed_version, HedVersion) -# schema = hed_version.get_schema() -# self.assertIsInstance(schema, HedSchema) -# -# def test_roundtrip(self): -# print(self.nwbfile) -# with NWBHDF5IO(self.path, mode="w") as io: -# io.write(self.nwbfile) -# -# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: -# read_nwbfile = io.read() -# print(read_nwbfile) -# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") -# #self.assertIsInstance(read_hed_version, HedVersion) -# self.assertEqual(read_hed_version.version, "8.2.0") +class TestHedVersionNWBFileSimpleRoundtrip(TestCase): + """Simple roundtrip test for HedTags.""" + + def setUp(self): + self.path = "test.nwb" + self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) + self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) + self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) + self.manager = get_manager() + self.filename = 'test_nwbfileio.h5' + self.nwbfile = NWBFile(session_description='a test NWB File', + identifier='TEST123', + session_start_time=self.start_time, + timestamps_reference_time=self.ref_time, + file_create_date=self.create_date, + experimenter='test experimenter', + stimulus_notes='test stimulus notes', + data_collection='test data collection notes', + experiment_description='test experiment description', + institution='nomad', + lab='nolab', + notes='nonotes', + pharmacology='nopharmacology', + protocol='noprotocol', + related_publications='nopubs', + session_id='007', + slices='noslices', + source_script='nosources', + surgery='nosurgery', + virus='novirus', + source_script_file_name='nofilename') + hed_version = HedVersion1(version="8.2.0") + self.nwbfile.add_lab_meta_data(hed_version) + print(f"Name:{hed_version.name}") + + def tearDown(self): + remove_test_file(self.path) + + def test_version(self): + hed_version = self.nwbfile.get_lab_meta_data("hed_version1_ext") + self.assertIsInstance(hed_version, HedVersion1) + schema = hed_version.get_schema() + self.assertIsInstance(schema, HedSchema) + + def test_roundtrip(self): + print(self.nwbfile) + with NWBHDF5IO(self.path, mode="w") as io: + io.write(self.nwbfile) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + print(read_nwbfile) + read_hed_version = read_nwbfile.get_lab_meta_data("hed_version1_ext") + #self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index d088d02..d749541 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -61,7 +61,6 @@ def main(): ] ) - hed_version1_ext = NWBGroupSpec( name="hed_version1", # fixed name neurodata_type_def="HedVersion1", @@ -74,7 +73,7 @@ def main(): hed_version1_ext.add_dataset( name="version", - doc="HED scheam version to use for this dataset", + doc="HED schema version to use for this dataset", dtype='text', quantity='?' ) From 5fa6597bb374ddf25310c879f84685e7f874c0d7 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Thu, 18 Jan 2024 15:40:21 -0600 Subject: [PATCH 5/5] Round trip unit tests are working --- spec/ndx-hed.extensions.yaml | 8 +- src/pynwb/ndx_hed/__init__.py | 4 +- src/pynwb/ndx_hed/hed_tags.py | 72 +++-- src/pynwb/ndx_hed/hed_version.py | 81 +++--- .../{hed_version1.py => hed_version_attr.py} | 84 +++--- src/pynwb/tests/test_hed_tags.py | 257 ++++++++++-------- src/pynwb/tests/test_hed_version.py | 131 +++++---- src/pynwb/tests/test_hed_version1.py | 141 ---------- src/pynwb/tests/test_hed_version_attr.py | 139 ++++++++++ src/spec/create_extension_spec.py | 14 +- 10 files changed, 467 insertions(+), 464 deletions(-) rename src/pynwb/ndx_hed/{hed_version1.py => hed_version_attr.py} (65%) delete mode 100644 src/pynwb/tests/test_hed_version1.py create mode 100644 src/pynwb/tests/test_hed_version_attr.py diff --git a/spec/ndx-hed.extensions.yaml b/spec/ndx-hed.extensions.yaml index fa094f3..a5c1954 100644 --- a/spec/ndx-hed.extensions.yaml +++ b/spec/ndx-hed.extensions.yaml @@ -6,7 +6,7 @@ datasets: HED tags are used, the HED schema version must be specified in the NWB file using the HedMetadata type. groups: -- neurodata_type_def: HedVersion +- neurodata_type_def: HedVersionAttr neurodata_type_inc: LabMetaData name: hed_version doc: An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) @@ -17,14 +17,14 @@ groups: dtype: text doc: The version of the HED schema used to validate the HED tags, e.g., '8.2.0'. Required if HED tags are used in the NWB file. -- neurodata_type_def: HedVersion1 +- neurodata_type_def: HedVersion neurodata_type_inc: LabMetaData - name: hed_version1 + name: hed_version doc: An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) schema version. TODO When merged with core, this will no longer inherit from LabMetaData but from NWBContainer and be placed optionally in /general. datasets: - name: version dtype: text - doc: HED scheam version to use for this dataset + doc: HED schema version to use for this dataset quantity: '?' diff --git a/src/pynwb/ndx_hed/__init__.py b/src/pynwb/ndx_hed/__init__.py index 1c39594..365e445 100644 --- a/src/pynwb/ndx_hed/__init__.py +++ b/src/pynwb/ndx_hed/__init__.py @@ -53,6 +53,6 @@ # # Remove these functions from the package del load_namespaces, get_class -from .hed_version import HedVersion +from .hed_version_attr import HedVersionAttr from .hed_tags import HedTags -from .hed_version1 import HedVersion1 +from .hed_version import HedVersion diff --git a/src/pynwb/ndx_hed/hed_tags.py b/src/pynwb/ndx_hed/hed_tags.py index 690c415..5d0c4e6 100644 --- a/src/pynwb/ndx_hed/hed_tags.py +++ b/src/pynwb/ndx_hed/hed_tags.py @@ -1,10 +1,12 @@ from collections.abc import Iterable from hdmf.common import VectorData from hdmf.utils import docval, getargs, get_docval, popargs +from hed.errors import HedFileError, get_printable_issue_string from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string +from hed.models import HedString from pynwb import register_class from pynwb.file import LabMetaData, NWBFile -from ndx_hed import HedVersion +from ndx_hed import HedVersionAttr @register_class('HedTags', 'ndx-hed') @@ -12,17 +14,19 @@ class HedTags(VectorData): """ Column storing HED (Hierarchical Event Descriptors) annotations for a row. A HED string is a comma-separated, and possibly parenthesized list of HED tags selected from a valid HED vocabulary as specified by the - NWBFile field HEDVersion. + NWBFile field HedVersion. """ - __nwbfields__ = ('sub_name', '_hed_version') + __nwbfields__ = ('_hed_schema') - @docval(*get_docval(VectorData.__init__)) + @docval({'name': 'name', 'type': 'str', 'doc': 'Must be HED', 'default': 'HED'}, + {'name': 'description', 'type': 'str', 'doc': 'Description of the HED annotations', + 'default': 'Hierarchical Event Descriptors (HED) annotations'}, + *get_docval(VectorData.__init__, 'data')) def __init__(self, **kwargs): - # kwargs['name'] = 'HED' + kwargs['name'] = 'HED' super().__init__(**kwargs) - self._hed_version = None self._init_internal() def _init_internal(self): @@ -32,54 +36,42 @@ def _init_internal(self): TODO: How should errors be handled if this file doesn't have a HedVersion object in the LabMetaData? """ - self.sub_name = "HED" + self._hed_schema = None @docval({'name': 'val', 'type': str, - 'doc': 'the value to add to this column. Should be a valid HED string.'}) + 'doc': 'the value to add to this column. Should be a valid HED string -- just forces string.'}) def add_row(self, **kwargs): """Append a data value to this column.""" val = getargs('val', kwargs) - # val.check_types() - # TODO how to validate - # - # if val is not None and self.validate(val): - # if self.term_set.validate(term=val): - # self.append(val) - # else: - # msg = ("%s is not in the term set." % val) - # raise ValueError(msg) - # - # else: - # self.append(val) super().append(val) - @docval({'name': 'key', 'type': 'str', 'doc': 'the value to add to this column'}) - def get(self, key): - """ - Retrieve elements from this object. - - """ - # TODO: Can key be more than a single value? Do we need to check validity of anything? - vals = super().get(key) - return vals - - @docval({'name': 'return', 'type': 'list', 'doc': 'list of issues or none'}) - def validate(self): - """Validate this VectorData. """ - hed_schema = self.get_hed_version() - return True - - def get_hed_version(self): - if not self._hed_version: + # @docval({'name': 'schema', 'type': (HedSchema, None), 'doc': 'HedSchema to use to validate.', 'default': None}, + # {'name': 'return', 'type': 'list', 'doc': 'list of issues or none'}) + def validate(self, schema): + """Validate this VectorData. This is assuming a list --- where is the general iterator.""" + if not schema: + raise HedFileError('HedSchemaMissing', "Must provide a valid HedSchema", "") + issues = [] + for index in range(len(self.data)): + hed_obj = HedString(self.get(index), schema) + these_issues = hed_obj.validate() + if these_issues: + issues.append(f"line {str(index)}: {get_printable_issue_string(these_issues)}") + return "\n".join(issues) + + def get_hed_schema(self): + if not self._hed_schema: root = self._get_root() if isinstance(root, NWBFile): - self._hed_version = root.get_lab_meta_data("HedVersion") - return self._hed_version + self._hed_schema = root.get_lab_meta_data("hed_version").get_schema() + return self._hed_schema def _get_root(self): root = self while hasattr(root, 'parent') and root.parent: root = root.parent + if root == self: + return None return root diff --git a/src/pynwb/ndx_hed/hed_version.py b/src/pynwb/ndx_hed/hed_version.py index c88444e..006c822 100644 --- a/src/pynwb/ndx_hed/hed_version.py +++ b/src/pynwb/ndx_hed/hed_version.py @@ -1,47 +1,34 @@ -from hdmf.utils import docval, getargs, get_docval, popargs -from pynwb import register_class -from pynwb.file import LabMetaData -from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string - - -@register_class("HedVersion", "ndx-hed") -class HedVersion(LabMetaData): - """ - The class containing the HED versions and HED schema used in this data file. - - """ - - __nwbfields__ = ('version', 'schema_string') - - # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}, - # *get_docval(LabMetaData.__init__)) - @docval({'name': 'version', 'type': str, 'doc': 'HED strings of type str'}) - def __init__(self, **kwargs): - version = popargs('version', kwargs) - kwargs['name'] = 'hed_version' - super().__init__(**kwargs) - self.version = version - self._init_internal() - - # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) - # def __init__(self, version): - # kwargs = {'name': 'hed_version'} - # super().__init__(**kwargs) - # self.name = 'hed_version' - # self.version = version - # self._init_internal() - - def _init_internal(self): - """ - Create a HED schema string - """ - hed_schema = load_schema_version(self.version) - self.schema_string = hed_schema.get_as_xml_string() - - @docval(returns='The HED schema version', rtype=str) - def get_version(self): - return self.version - - @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) - def get_schema(self): - return from_string(self.schema_string) +from hdmf.utils import docval, popargs +from pynwb import register_class +from pynwb.file import LabMetaData +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string + + +@register_class("HedVersion", "ndx-hed") +class HedVersion(LabMetaData): + """ The class containing the HED versions and HED schema used in this data file. """ + + __nwbfields__ = ('name', 'version', 'schema_string') + + @docval({'name': 'version', 'type': str, 'doc': 'HED strings of type str'}) + def __init__(self, **kwargs): + version = popargs('version', kwargs) + kwargs['name'] = 'hed_version' + super().__init__(**kwargs) + self.version = version + self._init_internal() + + def _init_internal(self): + """ Create a HED schema string """ + hed_schema = load_schema_version(self.version) + self.schema_string = hed_schema.get_as_xml_string() + + @docval(returns='The HED schema version', rtype=str) + def get_version(self): + """ Return the schema version. """ + return self.version + + @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) + def get_schema(self): + """ Return the HEDSchema object for this version.""" + return from_string(self.schema_string) diff --git a/src/pynwb/ndx_hed/hed_version1.py b/src/pynwb/ndx_hed/hed_version_attr.py similarity index 65% rename from src/pynwb/ndx_hed/hed_version1.py rename to src/pynwb/ndx_hed/hed_version_attr.py index d5d7a3f..bd9bb77 100644 --- a/src/pynwb/ndx_hed/hed_version1.py +++ b/src/pynwb/ndx_hed/hed_version_attr.py @@ -1,47 +1,37 @@ -from hdmf.utils import docval, getargs, get_docval, popargs -from pynwb import register_class -from pynwb.file import LabMetaData -from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string - - -@register_class("HedVersion1", "ndx-hed") -class HedVersion1(LabMetaData): - """ - The class containing the HED versions and HED schema used in this data file. - - """ - - __nwbfields__ = ('name', 'version', 'schema_string') - - # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}, - # *get_docval(LabMetaData.__init__)) - @docval({'name': 'version', 'type': str, 'doc': 'HED strings of type str'}) - def __init__(self, **kwargs): - version = popargs('version', kwargs) - kwargs['name'] = 'hed_version1_ext' - super().__init__(**kwargs) - self.version = version - self._init_internal() - - # @docval({'name': 'version', 'type': str, 'doc': 'HED version of type str'}) - # def __init__(self, version): - # kwargs = {'name': 'hed_version1'} - # super().__init__(**kwargs) - # self.name = 'hed_version1' - # self.version = version - # self._init_internal() - - def _init_internal(self): - """ - Create a HED schema string - """ - hed_schema = load_schema_version(self.version) - self.schema_string = hed_schema.get_as_xml_string() - - @docval(returns='The HED schema version', rtype=str) - def get_version(self): - return self.version - - @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) - def get_schema(self): - return from_string(self.schema_string) +from hdmf.utils import docval, getargs, get_docval, popargs +from pynwb import register_class +from pynwb.file import LabMetaData +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version, from_string + + +@register_class("HedVersionAttr", "ndx-hed") +class HedVersionAttr(LabMetaData): + """ + The class containing the HED versions and HED schema used in this data file. + + """ + + __nwbfields__ = ('name', 'version', 'schema_string') + + @docval({'name': 'version', 'type': str, 'doc': 'HED strings of type str'}) + def __init__(self, **kwargs): + version = popargs('version', kwargs) + kwargs['name'] = 'hed_version' + super().__init__(**kwargs) + self.version = version + self._init_internal() + + def _init_internal(self): + """ + Create a HED schema string + """ + hed_schema = load_schema_version(self.version) + self.schema_string = hed_schema.get_as_xml_string() + + @docval(returns='The HED schema version', rtype=str) + def get_version(self): + return self.version + + @docval(returns='The HED schema or schema group object for this version', rtype=(HedSchema, HedSchemaGroup)) + def get_schema(self): + return from_string(self.schema_string) diff --git a/src/pynwb/tests/test_hed_tags.py b/src/pynwb/tests/test_hed_tags.py index 743fdd8..1c38c30 100644 --- a/src/pynwb/tests/test_hed_tags.py +++ b/src/pynwb/tests/test_hed_tags.py @@ -1,69 +1,147 @@ """Unit and integration tests for ndx-hed.""" +from pandas import DataFrame from datetime import datetime, timezone from dateutil.tz import tzlocal, tzutc from uuid import uuid4, UUID -from hed.schema import HedSchema, HedSchemaGroup +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version +from hdmf.common import VectorData +from hdmf.utils import docval, getargs, get_docval, popargs from pynwb import NWBHDF5IO, get_manager, NWBFile from pynwb.testing.mock.file import mock_NWBFile from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin from src.pynwb.ndx_hed import HedVersion, HedTags -class TestHedAnnotationsConstructor(TestCase): +class TestHedTagsConstructor(TestCase): """Simple unit test for creating a HedTags.""" + def setUp(self): + self.tags = HedTags(data=["Correct-action", "Incorrect-action"]) + + def tearDown(self): + pass + def test_constructor(self): """Test setting HED values using the constructor.""" - hed_annotations = HedTags( - name='HED', - description="description", - data=["Correct-action", "Incorrect-action"], - ) - self.assertEqual(hed_annotations.sub_name, "HED") - self.assertEqual(hed_annotations.description, "description") - self.assertEqual(hed_annotations.data, ["Correct-action", "Incorrect-action"]) + self.assertEqual(self.tags.name, "HED") + self.assertTrue(self.tags.description) + self.assertEqual(self.tags.data, ["Correct-action", "Incorrect-action"]) + + def test_add_row(self): + """Testing adding a row to the HedTags. """ + self.assertEqual(len(self.tags.data), 2) + self.tags.add_row("Correct-action") + self.assertEqual(len(self.tags.data), 3) + + def test_get(self): + """Testing getting slices. """ + self.assertEqual(self.tags.get(0), "Correct-action") + self.assertEqual(self.tags.get([0, 1]), ['Correct-action', 'Incorrect-action']) def test_add_to_trials_table(self): """Test adding HED column and data to a trials table.""" nwbfile = mock_NWBFile() - hed_version = HedVersion("8.2.0") - nwbfile.add_lab_meta_data(hed_version) - nwbfile.add_trial_column(name="HED", description="HED annotations for each trial", - col_cls=HedTags, data=[]) + nwbfile.add_trial_column(name="HED", description="temp", col_cls=HedTags, data=[]) nwbfile.add_trial(start_time=0.0, stop_time=1.0, HED="Correct-action") nwbfile.add_trial(start_time=2.0, stop_time=3.0, HED="Incorrect-action") self.assertIsInstance(nwbfile.trials["HED"], HedTags) hed_col = nwbfile.trials["HED"] + self.assertEqual(hed_col.name, "HED") + self.assertEqual(hed_col.description, "temp") self.assertEqual(nwbfile.trials["HED"].data[0], "Correct-action") self.assertEqual(nwbfile.trials["HED"].data[1], "Incorrect-action") + def test_add_to_trials_table_force_HED(self): + """Trial table does not allow the forcing of the column name to be HED.""" + nwbfile = mock_NWBFile() + nwbfile.add_trial_column(name="Blech", description="temp", col_cls=HedTags, data=[]) + nwbfile.add_trial(start_time=0.0, stop_time=1.0, Blech="Correct-action") + nwbfile.add_trial(start_time=2.0, stop_time=3.0, Blech="Incorrect-action") + self.assertIsInstance(nwbfile.trials["Blech"], HedTags) + hed_col = nwbfile.trials["Blech"] + self.assertEqual(hed_col.name, "HED") + self.assertEqual(nwbfile.trials["Blech"].data[0], "Correct-action") + self.assertEqual(nwbfile.trials["Blech"].data[1], "Incorrect-action") + + def test_validate_hed_tags(self): + """Test the validate_hed_tags""" + schema = load_schema_version("8.2.0") + issues = self.tags.validate(schema) + self.assertFalse(issues) + self.tags.add_row("Blech") + self.tags.add_row("Sensory-event") + self.tags.add_row("") + self.tags.add_row("Agent-action") + self.tags.add_row("Red, (Blue, Green") + issues = self.tags.validate(schema) + self.assertEqual(7, len(self.tags.data)) + self.assertTrue(issues) + + def test_get_root(self): + root = self.tags._get_root() + self.assertFalse(root) + + def test_get_hed_version(self): + schema = self.tags.get_hed_schema() + self.assertFalse(schema) + + +class TestHedTagsSimpleRoundtrip(TestCase): + """Simple roundtrip test for HedNWBFile.""" -# class TestHedTagsSimpleRoundtrip(TestCase): -# """Simple roundtrip test for HedNWBFile.""" -# -# def setUp(self): -# self.path = "test.nwb" -# -# def tearDown(self): -# remove_test_file(self.path) -# -# def test_roundtrip(self): -# """ -# Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. -# """ -# nwbfile = mock_NWBFile() -# # hed_version = HedVersion(["8.2.0"]) -# hed_version = HedVersion("8.2.0") -# nwbfile.add_lab_meta_data(hed_version) -# -# with NWBHDF5IO(self.path, mode="w") as io: -# io.write(nwbfile) -# -# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: -# read_nwbfile = io.read() -# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") -# self.assertIsInstance(read_hed_version, HedVersion) -# self.assertEqual(read_hed_version.version, "8.2.0") + def setUp(self): + self.path = "test.nwb" + nwb_mock = mock_NWBFile() + nwb_mock.add_lab_meta_data(HedVersion("8.2.0")) + nwb_mock.add_trial_column(name="HED", description="HED annotations for each trial", + col_cls=HedTags, data=[]) + nwb_mock.add_trial(start_time=0.0, stop_time=1.0, HED="Correct-action") + nwb_mock.add_trial(start_time=2.0, stop_time=3.0, HED="Incorrect-action") + self.nwb_mock = nwb_mock + + def tearDown(self): + remove_test_file(self.path) + + def test_get_root(self): + tags = self.nwb_mock.trials["HED"] + self.assertIsInstance(tags, HedTags) + root = tags._get_root() + self.assertIsInstance(root, NWBFile) + + def test_get_hed_schema(self): + tags = self.nwb_mock.trials["HED"] + schema = tags.get_hed_schema() + self.assertIsInstance(schema, HedSchema) + + def test_validate_hed_tags(self): + """Test the validate_hed_tags""" + schema = load_schema_version("8.2.0") + tags = self.nwb_mock.trials["HED"] + issues = tags.validate(schema) + self.assertFalse(issues) + tags.add_row("Blech") + tags.add_row("Sensory-event") + tags.add_row("") + tags.add_row("Agent-action") + tags.add_row("Red, (Blue, Green") + self.assertEqual(7, len(self.nwb_mock.trials["HED"])) + issues = tags.validate(schema) + self.assertEqual(7, len(tags.data)) + self.assertTrue(issues) + + def test_roundtrip(self): + """ Create a HedMetadata, write it to mock file, read file, and test that it matches the original HedNWBFile.""" + + with NWBHDF5IO(self.path, mode="w") as io: + io.write(self.nwb_mock) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") + self.assertEqual(read_nwbfile.trials["HED"].data[0], "Correct-action") + self.assertEqual(read_nwbfile.trials["HED"].data[1], "Incorrect-action") class TestHedTagsNWBFileRoundtrip(TestCase): @@ -107,6 +185,32 @@ def setUp(self): def tearDown(self): remove_test_file(self.path) + def test_get_root(self): + tags = self.nwbfile.trials["HED"] + self.assertIsInstance(tags, HedTags) + root = tags._get_root() + self.assertIsInstance(root, NWBFile) + + def test_get_hed_schema(self): + tags = self.nwbfile.trials["HED"] + schema = tags.get_hed_schema() + self.assertIsInstance(schema, HedSchema) + + def test_validate_hed_tags(self): + """Test the validate_hed_tags""" + schema = load_schema_version("8.2.0") + tags = self.nwbfile.trials["HED"] + issues = tags.validate(schema) + self.assertFalse(issues) + tags.add_row("Blech") + tags.add_row("Sensory-event") + tags.add_row("") + tags.add_row("Agent-action") + tags.add_row("Red, (Blue, Green") + self.assertEqual(7, len(self.nwbfile.trials["HED"])) + issues = tags.validate(schema) + self.assertEqual(7, len(tags.data)) + self.assertTrue(issues) def test_roundtrip(self): """ Add a HedTags to an NWBFile, write it to file, read the file, and test that the HedTags from the @@ -118,73 +222,8 @@ def test_roundtrip(self): with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: read_nwbfile = io.read() - lab_metadata = read_nwbfile.get_lab_meta_data() - print(f"lab: {str(lab_metadata)}") read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") - print(f"hed: {read_hed_version}") - # self.assertIsInstance(read_hed_version, HedVersion) - # self.assertEqual(read_hed_version.version, "8.2.0") - # self.assertEqual(read_nwbfile.trials["HED"].data[0], "Correct-action") - # self.assertEqual(read_nwbfile.trials["HED"].data[1], "Incorrect-action") - -# -# # class TestHedTagsRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): -# # """Complex, more complete roundtrip test for HedTags using pynwb.testing infrastructure.""" -# -# # def getContainerType(self): -# # return "HedTags" -# -# # def addContainer(self): -# # self.nwbfile = HedNWBFile( -# # session_description="session_description", -# # identifier=str(uuid4()), -# # session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), -# # hed_schema_version="8.2.0", -# # ) -# -# # self.nwbfile.add_trial_column("hed_tags", "HED tags for each trial", col_cls=HedTags, index=True) -# # self.nwbfile.add_trial(start_time=0.0, stop_time=1.0, hed_tags=["animal_target", "correct_response"]) -# # self.nwbfile.add_trial(start_time=2.0, stop_time=3.0, hed_tags=["animal_target", "incorrect_response"]) -# -# # def getContainer(self, nwbfile: NWBFile): -# # return nwbfile.trials["hed_tags"].target - - - -# class TestHedNWBFileRoundtripPyNWB(NWBH5IOFlexMixin, TestCase): -# """Complex, more complete roundtrip test for HedNWBFile using pynwb.testing infrastructure.""" -# -# def setUp(self): -# self.nwbfile = NWBFile( -# session_description='session_description', -# identifier='identifier', -# session_start_time=datetime.now(timezone.utc) -# ) -# self.filename = "test.nwb" -# self.export_filename = "test_export.nwb" -# -# def tearDown(self): -# remove_test_file(self.filename) -# remove_test_file(self.export_filename) -# -# def addContainer(self): -# """ Add the test ElectricalSeries and related objects to the given NWBFile """ -# pass -# -# def getContainer(self, nwbfile: NWBFile): -# # return nwbfile.acquisition['test_eS'] -# return None -# -# def test_roundtrip(self): -# hed_version = HedVersion("8.2.0") -# self.nwbfile.add_lab_meta_data(hed_version) -# -# with NWBHDF5IO(self.filename, mode='w') as io: -# io.write(self.nwbfile) -# -# with NWBHDF5IO(self.filename, mode='r', load_namespaces=True) as io: -# read_nwbfile = io.read() -# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") -# self.assertIsInstance(read_hed_version, HedVersion) -# self.assertEqual(read_hed_version.version, "8.2.0") - + self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") + self.assertEqual(read_nwbfile.trials["HED"].data[0], "Correct-action") + self.assertEqual(read_nwbfile.trials["HED"].data[1], "Incorrect-action") diff --git a/src/pynwb/tests/test_hed_version.py b/src/pynwb/tests/test_hed_version.py index 7c17083..2f750ef 100644 --- a/src/pynwb/tests/test_hed_version.py +++ b/src/pynwb/tests/test_hed_version.py @@ -1,13 +1,13 @@ """Unit and integration tests for ndx-hed.""" -from datetime import datetime, timezone +from datetime import datetime from dateutil.tz import tzlocal, tzutc -from uuid import uuid4, UUID from pynwb.file import LabMetaData from pynwb import NWBHDF5IO, get_manager, NWBFile from pynwb.testing.mock.file import mock_NWBFile from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin from src.pynwb.ndx_hed import HedVersion -from hed.schema import HedSchema, HedSchemaGroup +from hed.schema import HedSchema, HedSchemaGroup, load_schema_version +from hed.errors import HedFileError class TestHedVersion(TestCase): @@ -26,27 +26,30 @@ def test_constructor(self): # self.assertIsInstance(hed_version2.version, list) # hed_version3 = HedVersion(["8.2.0", "sc:score_1.0.0"]) + def test_constructor_bad_version(self): + with self.assertRaises(HedFileError): + HedVersion("bad-version") + def test_get_schema(self): - hed_version1 = HedVersion(version="8.2.0") + hed_version1 = HedVersion("8.2.0") hed_schema = hed_version1.get_schema() self.assertIsInstance(hed_schema, HedSchema) def test_get_version(self): - hed_version1 = HedVersion(version="8.2.0") + hed_version1 = HedVersion("8.2.0") version = hed_version1.get_version() self.assertIsInstance(version, str) self.assertEqual(version, "8.2.0") def test_add_to_nwbfile(self): nwbfile = mock_NWBFile() - hed_version1 = HedVersion(version="8.2.0") + hed_version1 = HedVersion("8.2.0") nwbfile.add_lab_meta_data(hed_version1) hed_version2 = nwbfile.get_lab_meta_data("hed_version") self.assertIsInstance(hed_version2, HedVersion) - # hed_version2 = HedVersion(["8.2.0"]) -class TestHedVersionSimpleRoundtrip(TestCase): +class TestHedVersion1SimpleRoundtrip(TestCase): """Simple roundtrip test for HedNWBFile.""" def setUp(self): @@ -56,11 +59,8 @@ def tearDown(self): remove_test_file(self.path) def test_roundtrip(self): - """ - Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. - """ + """Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile.""" nwbfile = mock_NWBFile() - # hed_version = HedVersion(["8.2.0"]) hed_version = HedVersion(version="8.2.0") self.assertIsInstance(hed_version, HedVersion) self.assertIsInstance(hed_version, LabMetaData) @@ -83,58 +83,55 @@ def test_roundtrip(self): self.assertEqual(read_hed_version.version, "8.2.0") -# class TestHedVersionNWBFileSimpleRoundtrip(TestCase): -# """Simple roundtrip test for HedTags.""" -# -# def setUp(self): -# self.path = "test.nwb" -# self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) -# self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) -# self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) -# self.manager = get_manager() -# self.filename = 'test_nwbfileio.h5' -# self.nwbfile = NWBFile(session_description='a test NWB File', -# identifier='TEST123', -# session_start_time=self.start_time, -# timestamps_reference_time=self.ref_time, -# file_create_date=self.create_date, -# experimenter='test experimenter', -# stimulus_notes='test stimulus notes', -# data_collection='test data collection notes', -# experiment_description='test experiment description', -# institution='nomad', -# lab='nolab', -# notes='nonotes', -# pharmacology='nopharmacology', -# protocol='noprotocol', -# related_publications='nopubs', -# session_id='007', -# slices='noslices', -# source_script='nosources', -# surgery='nosurgery', -# virus='novirus', -# source_script_file_name='nofilename') -# hed_version = HedVersion(version="8.2.0") -# self.nwbfile.add_lab_meta_data(hed_version) -# print(f"Name:{hed_version.name}") -# -# def tearDown(self): -# remove_test_file(self.path) -# -# def test_version(self): -# hed_version = self.nwbfile.get_lab_meta_data("hed_version") -# self.assertIsInstance(hed_version, HedVersion) -# schema = hed_version.get_schema() -# self.assertIsInstance(schema, HedSchema) -# -# def test_roundtrip(self): -# print(self.nwbfile) -# with NWBHDF5IO(self.path, mode="w") as io: -# io.write(self.nwbfile) -# -# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: -# read_nwbfile = io.read() -# print(read_nwbfile) -# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") -# #self.assertIsInstance(read_hed_version, HedVersion) -# self.assertEqual(read_hed_version.version, "8.2.0") +class TestHedVersionNWBFileSimpleRoundtrip(TestCase): + """Simple roundtrip test for HedTags.""" + + def setUp(self): + self.path = "test.nwb" + self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) + self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) + self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) + self.manager = get_manager() + self.filename = 'test_nwbfileio.h5' + self.nwbfile = NWBFile(session_description='a test NWB File', + identifier='TEST123', + session_start_time=self.start_time, + timestamps_reference_time=self.ref_time, + file_create_date=self.create_date, + experimenter='test experimenter', + stimulus_notes='test stimulus notes', + data_collection='test data collection notes', + experiment_description='test experiment description', + institution='nomad', + lab='nolab', + notes='nonotes', + pharmacology='nopharmacology', + protocol='noprotocol', + related_publications='nopubs', + session_id='007', + slices='noslices', + source_script='nosources', + surgery='nosurgery', + virus='novirus', + source_script_file_name='nofilename') + hed_version = HedVersion(version="8.2.0") + self.nwbfile.add_lab_meta_data(hed_version) + + def tearDown(self): + remove_test_file(self.path) + + def test_version(self): + hed_version = self.nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(hed_version, HedVersion) + schema = hed_version.get_schema() + self.assertIsInstance(schema, HedSchema) + + def test_roundtrip(self): + with NWBHDF5IO(self.path, mode="w") as io: + io.write(self.nwbfile) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(read_hed_version, HedVersion) + self.assertEqual(read_hed_version.version, "8.2.0") diff --git a/src/pynwb/tests/test_hed_version1.py b/src/pynwb/tests/test_hed_version1.py deleted file mode 100644 index d16178c..0000000 --- a/src/pynwb/tests/test_hed_version1.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Unit and integration tests for ndx-hed.""" -from datetime import datetime, timezone -from dateutil.tz import tzlocal, tzutc -from uuid import uuid4, UUID -from pynwb.file import LabMetaData -from pynwb import NWBHDF5IO, get_manager, NWBFile -from pynwb.testing.mock.file import mock_NWBFile -from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin -from src.pynwb.ndx_hed import HedVersion1 -from hed.schema import HedSchema, HedSchemaGroup - - -class TestHedVersion1(TestCase): - """Simple unit test for creating a HedMetadata.""" - - def test_constructor(self): - """Test setting HedNWBFile values using the constructor.""" - hed_version1 = HedVersion1("8.2.0") - self.assertIsInstance(hed_version1, HedVersion1) - self.assertIsInstance(hed_version1, LabMetaData) - self.assertEqual(hed_version1.name, "hed_version1_ext") - self.assertIsInstance(hed_version1.version, str) - self.assertEqual(hed_version1.version, "8.2.0") - self.assertIsInstance(hed_version1.schema_string, str) - # hed_version2 = HedVersion(["8.2.0"]) - # self.assertIsInstance(hed_version2.version, list) - # hed_version3 = HedVersion(["8.2.0", "sc:score_1.0.0"]) - - def test_get_schema(self): - hed_version1 = HedVersion1("8.2.0") - hed_schema = hed_version1.get_schema() - self.assertIsInstance(hed_schema, HedSchema) - - def test_get_version(self): - hed_version1 = HedVersion1("8.2.0") - version = hed_version1.get_version() - self.assertIsInstance(version, str) - self.assertEqual(version, "8.2.0") - - def test_add_to_nwbfile(self): - nwbfile = mock_NWBFile() - hed_version1 = HedVersion1("8.2.0") - nwbfile.add_lab_meta_data(hed_version1) - hed_version2 = nwbfile.get_lab_meta_data("hed_version1_ext") - self.assertIsInstance(hed_version2, HedVersion1) - # hed_version2 = HedVersion(["8.2.0"]) - - -class TestHedVersion1SimpleRoundtrip(TestCase): - """Simple roundtrip test for HedNWBFile.""" - - def setUp(self): - self.path = "test.nwb" - - def tearDown(self): - remove_test_file(self.path) - - def test_roundtrip(self): - """ - Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. - """ - nwbfile = mock_NWBFile() - # hed_version = HedVersion(["8.2.0"]) - hed_version = HedVersion1(version="8.2.0") - print(f"name={hed_version.name}") - self.assertIsInstance(hed_version, HedVersion1) - self.assertIsInstance(hed_version, LabMetaData) - nwbfile.add_lab_meta_data(lab_meta_data=hed_version) - meta = nwbfile.get_lab_meta_data("hed_version1_ext") - self.assertEqual(meta.name, "hed_version1_ext") - self.assertEqual(meta.version, "8.2.0") - self.assertIsInstance(meta, HedVersion1) - self.assertIsInstance(meta, LabMetaData) - schema = meta.get_schema() - self.assertIsInstance(schema, HedSchema) - - with NWBHDF5IO(self.path, mode="w") as io: - io.write(nwbfile) - - with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: - read_nwbfile = io.read() - read_hed_version = read_nwbfile.get_lab_meta_data(hed_version.name) - self.assertIsInstance(read_hed_version, HedVersion1) - self.assertEqual(read_hed_version.version, "8.2.0") - - -class TestHedVersionNWBFileSimpleRoundtrip(TestCase): - """Simple roundtrip test for HedTags.""" - - def setUp(self): - self.path = "test.nwb" - self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) - self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) - self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) - self.manager = get_manager() - self.filename = 'test_nwbfileio.h5' - self.nwbfile = NWBFile(session_description='a test NWB File', - identifier='TEST123', - session_start_time=self.start_time, - timestamps_reference_time=self.ref_time, - file_create_date=self.create_date, - experimenter='test experimenter', - stimulus_notes='test stimulus notes', - data_collection='test data collection notes', - experiment_description='test experiment description', - institution='nomad', - lab='nolab', - notes='nonotes', - pharmacology='nopharmacology', - protocol='noprotocol', - related_publications='nopubs', - session_id='007', - slices='noslices', - source_script='nosources', - surgery='nosurgery', - virus='novirus', - source_script_file_name='nofilename') - hed_version = HedVersion1(version="8.2.0") - self.nwbfile.add_lab_meta_data(hed_version) - print(f"Name:{hed_version.name}") - - def tearDown(self): - remove_test_file(self.path) - - def test_version(self): - hed_version = self.nwbfile.get_lab_meta_data("hed_version1_ext") - self.assertIsInstance(hed_version, HedVersion1) - schema = hed_version.get_schema() - self.assertIsInstance(schema, HedSchema) - - def test_roundtrip(self): - print(self.nwbfile) - with NWBHDF5IO(self.path, mode="w") as io: - io.write(self.nwbfile) - - with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: - read_nwbfile = io.read() - print(read_nwbfile) - read_hed_version = read_nwbfile.get_lab_meta_data("hed_version1_ext") - #self.assertIsInstance(read_hed_version, HedVersion) - self.assertEqual(read_hed_version.version, "8.2.0") diff --git a/src/pynwb/tests/test_hed_version_attr.py b/src/pynwb/tests/test_hed_version_attr.py new file mode 100644 index 0000000..7fa7c54 --- /dev/null +++ b/src/pynwb/tests/test_hed_version_attr.py @@ -0,0 +1,139 @@ +"""Unit and integration tests for ndx-hed.""" +from datetime import datetime, timezone +from dateutil.tz import tzlocal, tzutc +from uuid import uuid4, UUID +from pynwb.file import LabMetaData +from pynwb import NWBHDF5IO, get_manager, NWBFile +from pynwb.testing.mock.file import mock_NWBFile +from pynwb.testing import TestCase, remove_test_file, NWBH5IOFlexMixin +from src.pynwb.ndx_hed import HedVersionAttr +from hed.schema import HedSchema, HedSchemaGroup + + +class TestHedVersionAttr(TestCase): + """Simple unit test for creating a HedMetadata.""" + + def test_constructor(self): + """Test setting HedNWBFile values using the constructor.""" + hed_version_example = HedVersionAttr("8.2.0") + self.assertIsInstance(hed_version_example, HedVersionAttr) + self.assertIsInstance(hed_version_example, LabMetaData) + self.assertEqual(hed_version_example.name, "hed_version") + self.assertIsInstance(hed_version_example.version, str) + self.assertEqual(hed_version_example.version, "8.2.0") + self.assertIsInstance(hed_version_example.schema_string, str) + # hed_version2 = HedVersion(["8.2.0"]) + # self.assertIsInstance(hed_version2.version, list) + # hed_version3 = HedVersion(["8.2.0", "sc:score_1.0.0"]) + + def test_get_schema(self): + hed_version_example = HedVersionAttr(version="8.2.0") + self.assertEqual(hed_version_example.name, "hed_version") + hed_schema = hed_version_example.get_schema() + self.assertIsInstance(hed_schema, HedSchema) + + def test_get_version(self): + hed_version_example = HedVersionAttr(version="8.2.0") + version = hed_version_example.get_version() + self.assertIsInstance(version, str) + self.assertEqual(version, "8.2.0") + + def test_add_to_nwbfile(self): + nwbfile = mock_NWBFile() + hed_version_example = HedVersionAttr(version="8.2.0") + nwbfile.add_lab_meta_data(lab_meta_data=hed_version_example) + hed_version_new = nwbfile.get_lab_meta_data(hed_version_example.name) + self.assertIsInstance(hed_version_new, HedVersionAttr) + # hed_version2 = HedVersion(["8.2.0"]) + + +class TestHedVersionSimpleRoundtrip(TestCase): + """Simple roundtrip test for HedNWBFile.""" + + def setUp(self): + self.path = "test.nwb" + + def tearDown(self): + remove_test_file(self.path) + + def test_roundtrip(self): + """ + Create a HedMetadata, write it to file, read the file, and test that it matches the original HedNWBFile. + """ + nwbfile = mock_NWBFile() + hed_version_example = HedVersionAttr(version="8.2.0") + nwbfile.add_lab_meta_data(lab_meta_data=hed_version_example) + hed_version_new = nwbfile.get_lab_meta_data("hed_version") + self.assertIsInstance(hed_version_new, HedVersionAttr) + self.assertEqual(hed_version_new.name, "hed_version") + self.assertEqual(hed_version_new.version, "8.2.0") + self.assertIsInstance(hed_version_new, HedVersionAttr) + self.assertIsInstance(hed_version_new, LabMetaData) + schema = hed_version_new.get_schema() + self.assertIsInstance(schema, HedSchema) + + with NWBHDF5IO(self.path, mode="w") as io: + io.write(nwbfile) + + with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: + read_nwbfile = io.read() + read_hed_version = read_nwbfile.get_lab_meta_data(hed_version_example.name) + # self.assertIsInstance(read_hed_version, HedVersionAttr) + # self.assertEqual(read_hed_version.version, "8.2.0") + + +# class TestHedVersionNWBFileSimpleRoundtrip(TestCase): +# """Simple roundtrip test for HedTags.""" +# +# def setUp(self): +# self.path = "test.nwb" +# self.start_time = datetime(1970, 1, 1, 12, tzinfo=tzutc()) +# self.ref_time = datetime(1979, 1, 1, 0, tzinfo=tzutc()) +# self.create_date = datetime(2017, 4, 15, 12, tzinfo=tzlocal()) +# self.manager = get_manager() +# self.filename = 'test_nwbfileio.h5' +# self.nwbfile = NWBFile(session_description='a test NWB File', +# identifier='TEST123', +# session_start_time=self.start_time, +# timestamps_reference_time=self.ref_time, +# file_create_date=self.create_date, +# experimenter='test experimenter', +# stimulus_notes='test stimulus notes', +# data_collection='test data collection notes', +# experiment_description='test experiment description', +# institution='nomad', +# lab='nolab', +# notes='nonotes', +# pharmacology='nopharmacology', +# protocol='noprotocol', +# related_publications='nopubs', +# session_id='007', +# slices='noslices', +# source_script='nosources', +# surgery='nosurgery', +# virus='novirus', +# source_script_file_name='nofilename') +# hed_version = HedVersion(version="8.2.0") +# self.nwbfile.add_lab_meta_data(hed_version) +# print(f"Name:{hed_version.name}") +# +# def tearDown(self): +# remove_test_file(self.path) +# +# def test_version(self): +# hed_version = self.nwbfile.get_lab_meta_data("hed_version") +# self.assertIsInstance(hed_version, HedVersion) +# schema = hed_version.get_schema() +# self.assertIsInstance(schema, HedSchema) +# +# def test_roundtrip(self): +# print(self.nwbfile) +# with NWBHDF5IO(self.path, mode="w") as io: +# io.write(self.nwbfile) +# +# with NWBHDF5IO(self.path, mode="r", load_namespaces=True) as io: +# read_nwbfile = io.read() +# print(read_nwbfile) +# read_hed_version = read_nwbfile.get_lab_meta_data("hed_version") +# #self.assertIsInstance(read_hed_version, HedVersion) +# self.assertEqual(read_hed_version.version, "8.2.0") diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index d749541..2d1fbb3 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -39,9 +39,9 @@ def main(): dtype="text", ) - hed_version_ext = NWBGroupSpec( + hed_version_attr = NWBGroupSpec( name="hed_version", # fixed name - neurodata_type_def="HedVersion", + neurodata_type_def="HedVersionAttr", neurodata_type_inc="LabMetaData", doc=("An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) schema version. " "TODO When merged with core, " @@ -61,9 +61,9 @@ def main(): ] ) - hed_version1_ext = NWBGroupSpec( - name="hed_version1", # fixed name - neurodata_type_def="HedVersion1", + hed_version = NWBGroupSpec( + name="hed_version", # fixed name + neurodata_type_def="HedVersion", neurodata_type_inc="LabMetaData", doc=("An extension of LabMetaData to store the Hierarchical Event Descriptor (HED) schema version. " "TODO When merged with core, " @@ -71,7 +71,7 @@ def main(): "optionally in /general."), ) - hed_version1_ext.add_dataset( + hed_version.add_dataset( name="version", doc="HED schema version to use for this dataset", dtype='text', @@ -79,7 +79,7 @@ def main(): ) # TODO: add all of your new data types to this list - new_data_types = [hed_version_ext, hed_tags, hed_version1_ext] + new_data_types = [hed_version_attr, hed_tags, hed_version] # export the spec to yaml files in the spec folder output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "spec"))