Integrate TypeConfig

.github/workflows/check_sphinx_links.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Install Sphinx dependencies and package
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements-doc.txt
+          python -m pip install -r requirements-doc.txt -r requirements-opt.txt
           python -m pip install .
 
       - name: Check Sphinx internal and external links

.github/workflows/run_all_tests.yml

@@ -26,20 +26,23 @@ jobs:
           - { name: linux-python3.9              , test-tox-env: py39            , build-tox-env: build-py39            , python-ver: "3.9" , os: ubuntu-latest }
           - { name: linux-python3.10             , test-tox-env: py310           , build-tox-env: build-py310           , python-ver: "3.10", os: ubuntu-latest }
           - { name: linux-python3.11             , test-tox-env: py311           , build-tox-env: build-py311           , python-ver: "3.11", os: ubuntu-latest }
+          - { name: linux-python3.11-opt         , test-tox-env: py311-optional  , build-tox-env: build-py311           , python-ver: "3.11", os: ubuntu-latest }
           - { name: linux-python3.12             , test-tox-env: py312           , build-tox-env: build-py312           , python-ver: "3.12", os: ubuntu-latest }
           - { name: linux-python3.12-upgraded    , test-tox-env: py312-upgraded  , build-tox-env: build-py312-upgraded  , python-ver: "3.12", os: ubuntu-latest }
           - { name: linux-python3.12-prerelease  , test-tox-env: py312-prerelease, build-tox-env: build-py312-prerelease, python-ver: "3.12", os: ubuntu-latest }
           - { name: windows-python3.8-minimum    , test-tox-env: py38-minimum    , build-tox-env: build-py38-minimum    , python-ver: "3.8" , os: windows-latest }
           - { name: windows-python3.9            , test-tox-env: py39            , build-tox-env: build-py39            , python-ver: "3.9" , os: windows-latest }
           - { name: windows-python3.10           , test-tox-env: py310           , build-tox-env: build-py310           , python-ver: "3.10", os: windows-latest }
           - { name: windows-python3.11           , test-tox-env: py311           , build-tox-env: build-py311           , python-ver: "3.11", os: windows-latest }
+          - { name: windows-python3.11-opt       , test-tox-env: py311-optional  , build-tox-env: build-py311           , python-ver: "3.11", os: windows-latest }
           - { name: windows-python3.12           , test-tox-env: py312           , build-tox-env: build-py312           , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.12-upgraded  , test-tox-env: py312-upgraded  , build-tox-env: build-py312-upgraded  , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.12-prerelease, test-tox-env: py312-prerelease, build-tox-env: build-py312-prerelease, python-ver: "3.11", os: windows-latest }
           - { name: macos-python3.8-minimum      , test-tox-env: py38-minimum    , build-tox-env: build-py38-minimum    , python-ver: "3.8" , os: macos-13 }
           - { name: macos-python3.9              , test-tox-env: py39            , build-tox-env: build-py39            , python-ver: "3.9" , os: macos-13 }
           - { name: macos-python3.10             , test-tox-env: py310           , build-tox-env: build-py310           , python-ver: "3.10", os: macos-latest }
           - { name: macos-python3.11             , test-tox-env: py311           , build-tox-env: build-py311           , python-ver: "3.11", os: macos-latest }
+          - { name: macos-python3.11-opt         , test-tox-env: py311-optional  , build-tox-env: build-py311           , python-ver: "3.11", os: macos-latest }
           - { name: macos-python3.12             , test-tox-env: py312           , build-tox-env: build-py312           , python-ver: "3.12", os: macos-latest }
           - { name: macos-python3.12-upgraded    , test-tox-env: py312-upgraded  , build-tox-env: build-py312-upgraded  , python-ver: "3.12", os: macos-latest }
           - { name: macos-python3.12-prerelease  , test-tox-env: py312-prerelease, build-tox-env: build-py312-prerelease, python-ver: "3.12", os: macos-latest }
@@ -198,7 +201,7 @@ jobs:
         include:
           - { name: conda-linux-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
           - { name: conda-windows-python3.12-ros3, python-ver: "3.12", os: windows-latest }
-          - { name: conda-macos-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes. 
+          - { name: conda-macos-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes.
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]
@@ -245,7 +248,7 @@ jobs:
         include:
           - { name: conda-linux-gallery-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
           - { name: conda-windows-gallery-python3.12-ros3, python-ver: "3.12", os: windows-latest }
-          - { name: conda-macos-gallery-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes. 
+          - { name: conda-macos-gallery-python3.12-ros3  , python-ver: "3.12", os: macos-13 } # This is due to DANDI not supporting osx-arm64. Will support macos-latest when this changes.
     steps:
       - name: Cancel non-latest runs
         uses: styfle/[email protected]

.github/workflows/run_coverage.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   run-coverage:
-    name: ${{ matrix.os }}
+    name: ${{ matrix.os }}, opt reqs ${{ matrix.opt_req }}
     runs-on: ${{ matrix.os }}
     # TODO handle forks
     # run pipeline on either a push event or a PR event on a fork
@@ -21,7 +21,11 @@ jobs:
         shell: bash
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - { os: ubuntu-latest , opt_req: true }
+          - { os: ubuntu-latest , opt_req: false }
+          - { os: windows-latest, opt_req: false }
+          - { os: macos-latest  , opt_req: false }
     env:
       OS: ${{ matrix.os }}
       PYTHON: '3.12'
@@ -47,9 +51,9 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements-dev.txt -r requirements.txt
 
-      # - name: Install optional dependencies
-      #   if: ${{ matrix.opt_req }}
-      #   run: python -m pip install -r requirements-opt.txt
+      - name: Install optional dependencies
+        if: ${{ matrix.opt_req }}
+        run: python -m pip install -r requirements-opt.txt
 
       - name: Install package
         run: |

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Enhancements and minor changes
 - Set rate default value inside `mock_ElectricalSeries` to avoid having to set `rate=None` explicitly when passing timestamps. @h-mayorquin [#1894](https://github.com/NeurodataWithoutBorders/pynwb/pull/1894)
+- Integrate validation through the `TypeConfigurator`. @mavaylon1 [#1829](https://github.com/NeurodataWithoutBorders/pynwb/pull/1829)
 
 ## PyNWB 2.7.0 (May 2, 2024)
 

docs/gallery/general/experimenter_termset.yaml

@@ -0,0 +1,17 @@
+id: termset/experimenter_example
+name: Experimenter
+version: 0.0.1
+prefixes:
+  ORC: https://orcid.org/
+imports:
+  - linkml:types
+default_range: string
+
+enums:
+  Experimenters:
+    permissible_values:
+      Bilbo Baggins:
+        description: He who must not be named.
+        meaning: ORC:111
+
+

docs/gallery/general/nwb_gallery_config.yaml

@@ -0,0 +1,10 @@
+namespaces:
+  core:
+    version: 2.7.0
+    data_types:
+      Subject:
+        species:
+          termset: nwb_subject_termset.yaml
+      NWBFile:
+        experimenter:
+          termset: experimenter_termset.yaml

docs/gallery/general/nwb_subject_termset.yaml

@@ -0,0 +1,27 @@
+id: termset/species_example
+name: Species
+version: 0.0.1
+prefixes:
+  NCBI_TAXON: https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=
+imports:
+  - linkml:types
+default_range: string
+
+enums:
+  Species:
+    permissible_values:
+      Homo sapiens:
+        description: the species is human
+        meaning: NCBI_TAXON:9606
+      Mus musculus:
+        description: the species is a house mouse
+        meaning: NCBI_TAXON:10090
+      Ursus arctos horribilis:
+        description: the species is a grizzly bear
+        meaning: NCBI_TAXON:116960
+      Myrmecophaga tridactyla:
+        description: the species is an anteater
+        meaning: NCBI_TAXON:71006
+      Ailuropoda melanoleuca:
+        description: the species is a panda
+        meaning: NCBI_TAXON:9646

docs/gallery/general/plot_configurator.py

@@ -0,0 +1,113 @@
+"""
+How to Configure Term Validations
+=================================
+
+This is a user guide for how to curate and take advantage of configuration files in
+order to more easily validate terms within datasets or attributes.
+
+Introduction
+-------------
+Users will create a configuration YAML file that outlines the fields (within a neurodata type)
+they want to be validated against a set of allowed terms. 
+After creating the configuration file, users will need to load the
+configuration file with the :py:func:`~pynwb.load_type_config` method.
+With the configuration loaded, every instance of the neurodata
+types defined in the configuration file will have the respective fields wrapped with a
+:py:class:`~hdmf.term_set.TermSetWrapper`.
+This automatic wrapping is what provides the term validation for the field value.
+For greater control on which datasets and attributes are validated
+against which sets of allowed terms, use the 
+:py:class:`~hdmf.term_set.TermSetWrapper` on individual datasets and attributes instead.
+You can follow the 
+`TermSet tutorial in the HDMF documentation 
+<https://hdmf.readthedocs.io/en/stable/tutorials/plot_term_set.html#sphx-glr-tutorials-plot-term-set-py>`_
+for more information.
+
+To unload a configuration, simply call :py:func:`~pynwb.unload_type_config`.
+We also provide a helper method to see the configuration that has been loaded:
+:py:func:`~pynwb.get_loaded_type_config`
+
+
+How to make a Configuration File
+--------------------------------
+To see an example of a configuration file, please refer to
+`<https://github.com/NeurodataWithoutBorders/pynwb/tree/dev/src/pynwb/config/nwb_config.yaml>`_.
+The configuration file uses YAML syntax. The
+user will construct a series of nested dictionaries to encompass all the necessary information.
+
+1. The user needs to define all the relevant namespaces. Recall that each neurodata type exists within
+   a namespace, whether that is the core namespace in PyNWB or a namespace in an extension. As namespaces grow,
+   we also require a version to be recorded in the configuration file to ensure proper functionality.
+2. Within a namespace dictionary, the user will have a list of data types they want to configure.
+3. Each data type will have a list of fields associated with a :py:class:`~hdmf.term_set.TermSet`.
+   The user can use the same or unique TermSet instances for each field.
+"""
+try:
+    import linkml_runtime  # noqa: F401
+except ImportError as e:
+    raise ImportError("Please install linkml-runtime to run this example: pip install linkml-runtime") from e
+
+from dateutil import tz
+from datetime import datetime
+from uuid import uuid4
+import os
+
+from pynwb import NWBFile, get_loaded_type_config, load_type_config, unload_type_config
+from pynwb.file import Subject
+
+# How to use a Configuration file
+# -------------------------------
+# As mentioned prior, the first step after creating a configuration file is
+# to load the file. In this configuration file, we have defined two fields
+# we want to always be validated: ``experimenter`` and ``species``. Each of these
+# are from a different neurodata type, :py:class:`~pynwb.file.NWBFile` and
+# :py:class:`~pynwb.file.Subject` respectively, and each
+# have a unique associated :py:class:`~hdmf.term_set.TermSet`.
+# It is important to remember that with the configuration loaded, the fields
+# are wrapped automatically, meaning the user should proceed with creating
+# the instances normally, i.e., without wrapping directly. Once instantiated,
+# the value of the fields are wrapped and then validated to see if it is a
+# permissible value in their respective :py:class:`~hdmf.term_set.TermSet`.
+
+dir_path = os.path.dirname(os.path.abspath("__file__"))
+yaml_file = os.path.join(dir_path, 'nwb_gallery_config.yaml')
+load_type_config(config_path=yaml_file)
+
+session_start_time = datetime(2018, 4, 25, hour=2, minute=30, second=3, tzinfo=tz.gettz("US/Pacific"))
+
+nwbfile = NWBFile(
+    session_description="Mouse exploring an open field",  # required
+    identifier=str(uuid4()),  # required
+    session_start_time=session_start_time,  # required
+    session_id="session_1234",  # optional
+    experimenter=[
+        "Bilbo Baggins",
+    ],  # optional
+    lab="Bag End Laboratory",  # optional
+    institution="University of My Institution",  # optional
+    experiment_description="I went on an adventure to reclaim vast treasures.",  # optional
+    related_publications="DOI:10.1016/j.neuron.2016.12.011",  # optional
+)
+
+subject = Subject(
+    subject_id="001",
+    age="P90D",
+    description="mouse 5",
+    species="Mus musculus",
+    sex="M",
+)
+
+nwbfile.subject = subject
+
+####################################
+# How to see the Configuration file
+# ---------------------------------
+# Call :py:class:`~pynwb.get_loaded_type_config` to get a dictionary containing the
+# current configuration.
+config = get_loaded_type_config()
+
+######################################
+# How to unload the Configuration file
+# ------------------------------------
+# Call :py:class:`~pynwb.unload_type_config` to toggle off the automatic validation.
+unload_type_config()

requirements-min.txt

@@ -1,6 +1,6 @@
 # minimum versions of package dependencies for installing PyNWB
 h5py==2.10  # support for selection of datasets with list of indices added in 2.10
-hdmf==3.12.2
+hdmf==3.14
 numpy==1.18
 pandas==1.1.5
 python-dateutil==2.7.3

requirements-opt.txt

@@ -0,0 +1,3 @@
+linkml-runtime==1.7.4; python_version >= "3.9"
+schemasheets==0.2.1; python_version >= "3.9"
+oaklib==0.5.32; python_version >= "3.9"

requirements.txt

@@ -1,6 +1,6 @@
 # pinned dependencies to reproduce an entire development environment to use PyNWB
 h5py==3.11.0
-hdmf==3.13.0
+hdmf==3.14.0
 numpy==1.26.4
 pandas==2.2.1
 python-dateutil==2.9.0.post0

src/pynwb/__init__.py

@@ -12,13 +12,44 @@
 from hdmf.backends.hdf5 import HDF5IO as _HDF5IO
 from hdmf.build import BuildManager, TypeMap
 import hdmf.common
+from hdmf.common import load_type_config as hdmf_load_type_config
+from hdmf.common import get_loaded_type_config as hdmf_get_loaded_type_config
+from hdmf.common import unload_type_config as hdmf_unload_type_config
+
 
 CORE_NAMESPACE = 'core'
 
 from .spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace  # noqa E402
 from .validate import validate  # noqa: F401, E402
 
 
+@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'},
+        {'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
+        is_method=False)
+def load_type_config(**kwargs):
+    """
+    This method will either load the default config or the config provided by the path.
+    """
+    config_path = kwargs['config_path']
+    type_map = kwargs['type_map'] or get_type_map()
+
+    hdmf_load_type_config(config_path=config_path, type_map=type_map)
+
+@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
+        is_method=False)
+def get_loaded_type_config(**kwargs):
+    type_map = kwargs['type_map'] or get_type_map()
+    return hdmf_get_loaded_type_config(type_map=type_map)
+
+@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
+        is_method=False)
+def unload_type_config(**kwargs):
+    """
+    Remove validation.
+    """
+    type_map = kwargs['type_map'] or get_type_map()
+    hdmf_unload_type_config(type_map=type_map)
+
 def __get_resources():
     try:
         from importlib.resources import files

src/pynwb/config/nwb_config.yaml

@@ -0,0 +1,7 @@
+namespaces:
+  core:
+    version: 2.7.0-alpha
+    data_types:
+      Subject:
+        species:
+          termset: nwb_subject_termset.yaml

src/pynwb/config/nwb_subject_termset.yaml

@@ -0,0 +1,27 @@
+id: termset/species_example
+name: Species
+version: 0.0.1
+prefixes:
+  NCBI_TAXON: https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=
+imports:
+  - linkml:types
+default_range: string
+
+enums:
+  Species:
+    permissible_values:
+      Homo sapiens:
+        description: the species is human
+        meaning: NCBI_TAXON:9606
+      Mus musculus:
+        description: the species is a house mouse
+        meaning: NCBI_TAXON:10090
+      Ursus arctos horribilis:
+        description: the species is a grizzly bear
+        meaning: NCBI_TAXON:116960
+      Myrmecophaga tridactyla:
+        description: the species is an anteater
+        meaning: NCBI_TAXON:71006
+      Ailuropoda melanoleuca:
+        description: the species is a panda
+        meaning: NCBI_TAXON:9646

src/pynwb/core.py

@@ -10,6 +10,7 @@
 from hdmf.utils import LabelledDict  # noqa: F401
 
 from . import CORE_NAMESPACE, register_class
+from pynwb import get_type_map
 
 
 def _not_parent(arg):
@@ -46,6 +47,18 @@ def _error_on_new_warn_on_construct(self, error_msg: str):
             raise ValueError(error_msg)
         warn(error_msg)
 
+    def _get_type_map(self):
+        return get_type_map()
+
+    @property
+    def data_type(self):
+        """
+        Return the spec data type associated with this container, i.e., the neurodata_type.
+        """
+        # we need this function here to use the correct _data_type_attr.
+        _type = getattr(self, self._data_type_attr)
+        return _type
+
 
 @register_class('NWBContainer', CORE_NAMESPACE)
 class NWBContainer(NWBMixin, Container):