Integrate TypeConfig

.github/workflows/check_sphinx_links.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Install Sphinx dependencies and package
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -r requirements-doc.txt
+          python -m pip install -r requirements-doc.txt -r requirements-opt.txt
           python -m pip install .
 
       - name: Check Sphinx internal and external links

.github/workflows/run_coverage.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   run-coverage:
-    name: ${{ matrix.os }}
+    name: ${{ matrix.os }}, opt reqs ${{ matrix.opt_req }}
     runs-on: ${{ matrix.os }}
     # TODO handle forks
     # run pipeline on either a push event or a PR event on a fork
@@ -21,7 +21,11 @@ jobs:
         shell: bash
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - { os: ubuntu-latest , opt_req: true }
+          - { os: ubuntu-latest , opt_req: false }
+          - { os: windows-latest, opt_req: false }
+          - { os: macos-latest  , opt_req: false }
     env:
       OS: ${{ matrix.os }}
       PYTHON: '3.12'
@@ -47,9 +51,9 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements-dev.txt -r requirements.txt
 
-      # - name: Install optional dependencies
-      #   if: ${{ matrix.opt_req }}
-      #   run: python -m pip install -r requirements-opt.txt
+      - name: Install optional dependencies
+        if: ${{ matrix.opt_req }}
+        run: python -m pip install -r requirements-opt.txt
 
       - name: Install package
         run: |

docs/gallery/general/experimenter_termset.yaml

@@ -0,0 +1,17 @@
+id: termset/experimenter_example
+name: Experimenter
+version: 0.0.1
+prefixes:
+  ORC: https://orcid.org/
+imports:
+  - linkml:types
+default_range: string
+
+enums:
+  Experimenters:
+    permissible_values:
+      Bilbo Baggins:
+        description: He who must not be named.
+        meaning: ORC:111
+
+

docs/gallery/general/nwb_gallery_config.yaml

@@ -0,0 +1,13 @@
+namespaces:
+  core:
+    version: 2.7.0-alpha
+    data_types:
+      Subject:
+        species:
+          termset: nwb_subject_termset.yaml
+      NWBFile:
+        experimenter:
+          termset: experimenter_termset.yaml
+      # ElectrodeGroup: This will return when electrode table is redefined
+      #   location:
+      #     termset: location_termset.yaml

docs/gallery/general/nwb_subject_termset.yaml

@@ -0,0 +1,27 @@
+id: termset/species_example
+name: Species
+version: 0.0.1
+prefixes:
+  NCBI_TAXON: https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=
+imports:
+  - linkml:types
+default_range: string
+
+enums:
+  Species:
+    permissible_values:
+      Homo sapiens:
+        description: the species is human
+        meaning: NCBI_TAXON:9606
+      Mus musculus:
+        description: the species is a house mouse
+        meaning: NCBI_TAXON:10090
+      Ursus arctos horribilis:
+        description: the species is a grizzly bear
+        meaning: NCBI_TAXON:116960
+      Myrmecophaga tridactyla:
+        description: the species is an anteater
+        meaning: NCBI_TAXON:71006
+      Ailuropoda melanoleuca:
+        description: the species is a panda
+        meaning: NCBI_TAXON:9646

docs/gallery/general/plot_configurator.py

@@ -0,0 +1,108 @@
+"""
+TypeConfigurator
+=======
+
+This is a user guide for how to take advantage of the
+:py:class:`~hdmf.term_set.TypeConfigurator` class by creating configuration files in
+order to more easily validate terms within datasets or attributes.
+
+Introduction
+-------------
+Users do not directly interact with the :py:class:`~hdmf.term_set.TypeConfigurator` class.
+Instead, users will create a configuration YAML file that outlines the fields (within a neurodata type)
+they want to be validated. After creating the configuration file, users will need to load the
+configuration with the :py:func:`~pynwb.load_type_config` method.
+With the configuration loaded, every instance of the neurodata
+types defined in the configuration file will have the respective fields wrapped with a
+:py:class:`~hdmf.term_set.TermSetWrapper`.
+This automatic wrapping is what provides the term validation for the the field value.
+If a user wants to have greater control on which instances have validated fields, the user cannot use the
+configurator, bur rather proceed with manually wrapping with a
+:py:class:`~hdmf.term_set.TermSetWrapper`.
+
+To unload a configuration, simply call :py:func:`~pynwb.unload_type_config`.
+We also provide a helper method to see the configuration that has been loaded:
+:py:func:`~pynwb.get_loaded_type_config`
+
+
+How to make a Configuration File
+--------------------------------
+Before taking advantage of the all the wonders that comes with using a configuration file,
+the user needs to create one following some simple guidelines. To follow along with an example,
+please refer to ``nwb_config.yaml`` under ``src/config``.
+The configuration file uses YAML syntax. The
+user will construct a series of nested dictionaries to encompass all the necessary information.
+
+1. The user needs to define all the relevant namespaces. Recall that each neurodata type exists within
+   a namespace, whether that is the core namespace in PyNWB or a namespace in an extension. As namespaces grow,
+   we also require a version to be recorded in the configuration file to ensure proper functionality.
+2. Within a namespace dictionary, the user will have a list of data types they want to configure.
+3. Each data type will have a list of fields associated with a :py:class:`~hdmf.term_set.TermSet`.
+   The user can use the same or unique TermSet instances for each field.
+"""
+try:
+    import linkml_runtime  # noqa: F401
+except ImportError as e:
+    raise ImportError("Please install linkml-runtime to run this example: pip install linkml-runtime") from e
+
+from dateutil import tz
+from datetime import datetime
+from uuid import uuid4
+import os
+
+from pynwb import NWBFile, get_loaded_type_config, load_type_config, unload_type_config
+from pynwb.file import Subject
+
+# How to use a Configuration file
+# -------------------------------
+# As mentioned prior, the first step after creating a configuration file is
+# to load the file.
+# It is important to remember that with the configuration loaded, the fields
+# are wrapped automatically, meaning the user should proceed with creating
+# the instances normally, i.e., without wrapping directly. In this example,
+# we load the a configuration file that associates a
+# :py:class:`~hdmf.term_set.TermSet` for the ``species`` field in
+# :py:class:`~pynwb.file.Subject`.
+
+dir_path = os.path.dirname(os.path.abspath("__file__"))
+yaml_file = os.path.join(dir_path, 'nwb_gallery_config.yaml')
+load_type_config(config_path=yaml_file)
+
+session_start_time = datetime(2018, 4, 25, hour=2, minute=30, second=3, tzinfo=tz.gettz("US/Pacific"))
+
+nwbfile = NWBFile(
+    session_description="Mouse exploring an open field",  # required
+    identifier=str(uuid4()),  # required
+    session_start_time=session_start_time,  # required
+    session_id="session_1234",  # optional
+    experimenter=[
+        "Bilbo Baggins",
+    ],  # optional
+    lab="Bag End Laboratory",  # optional
+    institution="University of My Institution",  # optional
+    experiment_description="I went on an adventure to reclaim vast treasures.",  # optional
+    related_publications="DOI:10.1016/j.neuron.2016.12.011",  # optional
+)
+
+subject = Subject(
+    subject_id="001",
+    age="P90D",
+    description="mouse 5",
+    species="Mus musculus",
+    sex="M",
+)
+
+nwbfile.subject = subject
+
+####################################
+# How to see the Configuration file
+# ---------------------------------
+# Users can retrieve the loaded configuration.
+config = get_loaded_type_config()
+
+######################################
+# How to unload the Configuration file
+# ------------------------------------
+# In order to toggle off the automatic validation, the user simple needs to unload
+# the configuration.
+unload_type_config()

requirements-min.txt

@@ -1,6 +1,6 @@
 # minimum versions of package dependencies for installing PyNWB
 h5py==2.10  # support for selection of datasets with list of indices added in 2.10
-hdmf==3.12.2
+hdmf==3.14
 numpy==1.18
 pandas==1.1.5
 python-dateutil==2.7.3

requirements-opt.txt

@@ -0,0 +1,3 @@
+linkml-runtime==1.7.4; python_version >= "3.9"
+schemasheets==0.2.1; python_version >= "3.9"
+oaklib==0.5.32; python_version >= "3.9"

requirements.txt

@@ -1,6 +1,6 @@
 # pinned dependencies to reproduce an entire development environment to use PyNWB
 h5py==3.11.0
-hdmf==3.13.0
+hdmf==3.14.0
 numpy==1.26.4
 pandas==2.2.1
 python-dateutil==2.9.0.post0

src/pynwb/__init__.py

@@ -12,13 +12,48 @@
 from hdmf.backends.hdf5 import HDF5IO as _HDF5IO
 from hdmf.build import BuildManager, TypeMap
 import hdmf.common
+from hdmf.common import load_type_config as hdmf_load_type_config
+from hdmf.common import get_loaded_type_config as hdmf_get_loaded_type_config
+from hdmf.common import unload_type_config as hdmf_unload_type_config
+
 
 CORE_NAMESPACE = 'core'
 
 from .spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace  # noqa E402
 from .validate import validate  # noqa: F401, E402
 
 
+CUR_DIR = os.path.dirname(os.path.realpath(__file__))
+path_to_config = os.path.join(CUR_DIR, 'config/nwb_config.yaml')
+
+@docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.',
+         'default': None},
+        {'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
+        is_method=False)
+def load_type_config(**kwargs):
+    """
+    This method will either load the default config or the config provided by the path.
+    """
+    config_path = kwargs['config_path'] or path_to_config
+    type_map = kwargs['type_map'] or get_type_map()
+
+    hdmf_load_type_config(config_path=config_path, type_map=type_map)
+
+@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
+        is_method=False)
+def get_loaded_type_config(**kwargs):
+    type_map = kwargs['type_map'] or get_type_map()
+    return hdmf_get_loaded_type_config(type_map=type_map)
+
+@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
+        is_method=False)
+def unload_type_config(**kwargs):
+    """
+    Remove validation.
+    """
+    type_map = kwargs['type_map'] or get_type_map()
+    hdmf_unload_type_config(type_map=type_map)
+
 def __get_resources():
     try:
         from importlib.resources import files

src/pynwb/config/nwb_config.yaml

@@ -0,0 +1,7 @@
+namespaces:
+  core:
+    version: 2.7.0-alpha
+    data_types:
+      Subject:
+        species:
+          termset: nwb_subject_termset.yaml

src/pynwb/config/nwb_subject_termset.yaml

@@ -0,0 +1,27 @@
+id: termset/species_example
+name: Species
+version: 0.0.1
+prefixes:
+  NCBI_TAXON: https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=
+imports:
+  - linkml:types
+default_range: string
+
+enums:
+  Species:
+    permissible_values:
+      Homo sapiens:
+        description: the species is human
+        meaning: NCBI_TAXON:9606
+      Mus musculus:
+        description: the species is a house mouse
+        meaning: NCBI_TAXON:10090
+      Ursus arctos horribilis:
+        description: the species is a grizzly bear
+        meaning: NCBI_TAXON:116960
+      Myrmecophaga tridactyla:
+        description: the species is an anteater
+        meaning: NCBI_TAXON:71006
+      Ailuropoda melanoleuca:
+        description: the species is a panda
+        meaning: NCBI_TAXON:9646

src/pynwb/core.py

@@ -10,6 +10,7 @@
 from hdmf.utils import LabelledDict  # noqa: F401
 
 from . import CORE_NAMESPACE, register_class
+from pynwb import get_type_map
 
 
 def _not_parent(arg):
@@ -46,6 +47,18 @@ def _error_on_new_warn_on_construct(self, error_msg: str):
             raise ValueError(error_msg)
         warn(error_msg)
 
+    def _get_type_map(self):
+        return get_type_map()
+
+    @property
+    def data_type(self):
+        """
+        Return the spec data type associated with this container, i.e., the neurodata_type.
+        """
+        # we need this function here to use the correct _data_type_attr.
+        _type = getattr(self, self._data_type_attr)
+        return _type
+
 
 @register_class('NWBContainer', CORE_NAMESPACE)
 class NWBContainer(NWBMixin, Container):

test.py

@@ -136,6 +136,11 @@ def __run_example_tests_helper(examples_scripts):
                         ws.append(w)
             for w in ws:
                 warnings.showwarning(w.message, w.category, w.filename, w.lineno, w.line)
+        except (ImportError, ValueError, ModuleNotFoundError) as e:
+            if "linkml" in str(e):
+                pass  # this is OK because linkml is not always installed
+            else:
+                raise e
         except Exception:
             print(traceback.format_exc())
             FAILURES += 1