From 3430b57c5a55a9f657910d8d2cf07aa408be9dec Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 4 Jan 2024 14:08:40 -0800 Subject: [PATCH 01/17] Update retinotopy.py --- src/pynwb/retinotopy.py | 293 ++++++++++++++++++++-------------------- 1 file changed, 148 insertions(+), 145 deletions(-) diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index a345177c0..55f92a3ff 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -1,145 +1,148 @@ -import warnings -from collections.abc import Iterable - -from hdmf.utils import docval, popargs, get_docval - -from . import register_class, CORE_NAMESPACE -from .core import NWBDataInterface, NWBData - - -class RetinotopyImage(NWBData): - """Gray-scale anatomical image of cortical surface. Array structure: [rows][columns] - """ - - __nwbfields__ = ('bits_per_pixel', - 'dimension', - 'format', - 'field_of_view') - - @docval({'name': 'name', 'type': str, 'doc': 'Name of this retinotopy image'}, - {'name': 'data', 'type': Iterable, 'doc': 'Data field.'}, - {'name': 'bits_per_pixel', 'type': int, - 'doc': 'Number of bits used to represent each value. This is necessary to determine maximum ' - '(white) pixel value.'}, - {'name': 'dimension', 'type': Iterable, 'shape': (2, ), 'doc': 'Number of rows and columns in the image.'}, - {'name': 'format', 'type': Iterable, 'doc': 'Format of image. Right now only "raw" supported.'}, - {'name': 'field_of_view', 'type': Iterable, 'shape': (2, ), 'doc': 'Size of viewing area, in meters.'}) - def __init__(self, **kwargs): - bits_per_pixel, dimension, format, field_of_view = popargs( - 'bits_per_pixel', 'dimension', 'format', 'field_of_view', kwargs) - super().__init__(**kwargs) - self.bits_per_pixel = bits_per_pixel - self.dimension = dimension - self.format = format - self.field_of_view = field_of_view - - -class FocalDepthImage(RetinotopyImage): - """Gray-scale image taken with same settings/parameters (e.g., focal depth, - wavelength) as data collection. Array format: [rows][columns]. - """ - - __nwbfields__ = ('focal_depth', ) - - @docval(*get_docval(RetinotopyImage.__init__), - {'name': 'focal_depth', 'type': float, 'doc': 'Focal depth offset, in meters.'}) - def __init__(self, **kwargs): - focal_depth = popargs('focal_depth', kwargs) - super().__init__(**kwargs) - self.focal_depth = focal_depth - - -class RetinotopyMap(NWBData): - """Abstract two-dimensional map of responses to stimuli along a single response axis (e.g., altitude) - """ - - __nwbfields__ = ('field_of_view', - 'dimension') - - @docval({'name': 'name', 'type': str, 'doc': 'the name of this axis map'}, - {'name': 'data', 'type': Iterable, 'shape': (None, None), 'doc': 'data field.'}, - {'name': 'field_of_view', 'type': Iterable, 'shape': (2, ), 'doc': 'Size of viewing area, in meters.'}, - {'name': 'dimension', 'type': Iterable, 'shape': (2, ), - 'doc': 'Number of rows and columns in the image'}) - def __init__(self, **kwargs): - field_of_view, dimension = popargs('field_of_view', 'dimension', kwargs) - super().__init__(**kwargs) - self.field_of_view = field_of_view - self.dimension = dimension - - -class AxisMap(RetinotopyMap): - """Abstract two-dimensional map of responses to stimuli along a single response axis (e.g., altitude) with unit - """ - - __nwbfields__ = ('unit', ) - - @docval(*get_docval(RetinotopyMap.__init__, 'name', 'data', 'field_of_view'), - {'name': 'unit', 'type': str, 'doc': 'Unit that axis data is stored in (e.g., degrees)'}, - *get_docval(RetinotopyMap.__init__, 'dimension')) - def __init__(self, **kwargs): - unit = popargs('unit', kwargs) - super().__init__(**kwargs) - self.unit = unit - - -@register_class('ImagingRetinotopy', CORE_NAMESPACE) -class ImagingRetinotopy(NWBDataInterface): - """ - Intrinsic signal optical imaging or widefield imaging for measuring retinotopy. Stores orthogonal - maps (e.g., altitude/azimuth; radius/theta) of responses to specific stimuli and a combined - polarity map from which to identify visual areas. - This group does not store the raw responses imaged during retinotopic mapping or the - stimuli presented, but rather the resulting phase and power maps after applying a Fourier - transform on the averaged responses. - Note: for data consistency, all images and arrays are stored in the format [row][column] and - [row, col], which equates to [y][x]. Field of view and dimension arrays may appear backward - (i.e., y before x). - """ - - __nwbfields__ = ({'name': 'sign_map', 'child': True}, - {'name': 'axis_1_phase_map', 'child': True}, - {'name': 'axis_1_power_map', 'child': True}, - {'name': 'axis_2_phase_map', 'child': True}, - {'name': 'axis_2_power_map', 'child': True}, - {'name': 'focal_depth_image', 'child': True}, - {'name': 'vasculature_image', 'child': True}, - 'axis_descriptions') - - @docval({'name': 'sign_map', 'type': RetinotopyMap, - 'doc': 'Sine of the angle between the direction of the gradient in axis_1 and axis_2.'}, - {'name': 'axis_1_phase_map', 'type': AxisMap, - 'doc': 'Phase response to stimulus on the first measured axis.'}, - {'name': 'axis_1_power_map', 'type': AxisMap, - 'doc': 'Power response on the first measured axis. Response is scaled so 0.0 is no power in ' - 'the response and 1.0 is maximum relative power.'}, - {'name': 'axis_2_phase_map', 'type': AxisMap, - 'doc': 'Phase response to stimulus on the second measured axis.'}, - {'name': 'axis_2_power_map', 'type': AxisMap, - 'doc': 'Power response on the second measured axis. Response is scaled so 0.0 is no ' - 'power in the response and 1.0 is maximum relative power.'}, - {'name': 'axis_descriptions', 'type': Iterable, 'shape': (2, ), - 'doc': 'Two-element array describing the contents of the two response axis fields. ' - 'Description should be something like ["altitude", "azimuth"] or ["radius", "theta"].'}, - {'name': 'focal_depth_image', 'type': FocalDepthImage, - 'doc': 'Gray-scale image taken with same settings/parameters (e.g., focal depth, wavelength) ' - 'as data collection. Array format: [rows][columns].'}, - {'name': 'vasculature_image', 'type': RetinotopyImage, - 'doc': 'Gray-scale anatomical image of cortical surface. Array structure: [rows][columns].'}, - {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'ImagingRetinotopy'}) - def __init__(self, **kwargs): - axis_1_phase_map, axis_1_power_map, axis_2_phase_map, axis_2_power_map, axis_descriptions, \ - focal_depth_image, sign_map, vasculature_image = popargs( - 'axis_1_phase_map', 'axis_1_power_map', 'axis_2_phase_map', 'axis_2_power_map', - 'axis_descriptions', 'focal_depth_image', 'sign_map', 'vasculature_image', kwargs) - super().__init__(**kwargs) - warnings.warn("The ImagingRetinotopy class currently cannot be written to or read from a file. " - "This is a known bug and will be fixed in a future release of PyNWB.") - self.axis_1_phase_map = axis_1_phase_map - self.axis_1_power_map = axis_1_power_map - self.axis_2_phase_map = axis_2_phase_map - self.axis_2_power_map = axis_2_power_map - self.axis_descriptions = axis_descriptions - self.focal_depth_image = focal_depth_image - self.sign_map = sign_map - self.vasculature_image = vasculature_image +raise ValueError( + "The pynwb.retinotopy module is deprecated. If you are interested in using these neurodata types, " + "please create an issue on https://github.com/NeurodataWithoutBorders/nwb-schema/issues." +) + +# import warnings +# from collections.abc import Iterable + +# from hdmf.utils import docval, popargs, get_docval + +# from . import register_class, CORE_NAMESPACE +# from .core import NWBDataInterface, NWBData + + +# class RetinotopyImage(NWBData): +# """Gray-scale anatomical image of cortical surface. Array structure: [rows][columns] +# """ + +# __nwbfields__ = ('bits_per_pixel', +# 'dimension', +# 'format', +# 'field_of_view') + +# @docval({'name': 'name', 'type': str, 'doc': 'Name of this retinotopy image'}, +# {'name': 'data', 'type': Iterable, 'doc': 'Data field.'}, +# {'name': 'bits_per_pixel', 'type': int, +# 'doc': 'Number of bits used to represent each value. This is necessary to determine maximum ' +# '(white) pixel value.'}, +# {'name': 'dimension', 'type': Iterable, 'shape': (2, ), 'doc': 'Number of rows and columns in the image.'}, +# {'name': 'format', 'type': Iterable, 'doc': 'Format of image. Right now only "raw" supported.'}, +# {'name': 'field_of_view', 'type': Iterable, 'shape': (2, ), 'doc': 'Size of viewing area, in meters.'}) +# def __init__(self, **kwargs): +# bits_per_pixel, dimension, format, field_of_view = popargs( +# 'bits_per_pixel', 'dimension', 'format', 'field_of_view', kwargs) +# super().__init__(**kwargs) +# self.bits_per_pixel = bits_per_pixel +# self.dimension = dimension +# self.format = format +# self.field_of_view = field_of_view + + +# class FocalDepthImage(RetinotopyImage): +# """Gray-scale image taken with same settings/parameters (e.g., focal depth, +# wavelength) as data collection. Array format: [rows][columns]. +# """ + +# __nwbfields__ = ('focal_depth', ) + +# @docval(*get_docval(RetinotopyImage.__init__), +# {'name': 'focal_depth', 'type': float, 'doc': 'Focal depth offset, in meters.'}) +# def __init__(self, **kwargs): +# focal_depth = popargs('focal_depth', kwargs) +# super().__init__(**kwargs) +# self.focal_depth = focal_depth + + +# class RetinotopyMap(NWBData): +# """Abstract two-dimensional map of responses to stimuli along a single response axis (e.g., altitude) +# """ + +# __nwbfields__ = ('field_of_view', +# 'dimension') + +# @docval({'name': 'name', 'type': str, 'doc': 'the name of this axis map'}, +# {'name': 'data', 'type': Iterable, 'shape': (None, None), 'doc': 'data field.'}, +# {'name': 'field_of_view', 'type': Iterable, 'shape': (2, ), 'doc': 'Size of viewing area, in meters.'}, +# {'name': 'dimension', 'type': Iterable, 'shape': (2, ), +# 'doc': 'Number of rows and columns in the image'}) +# def __init__(self, **kwargs): +# field_of_view, dimension = popargs('field_of_view', 'dimension', kwargs) +# super().__init__(**kwargs) +# self.field_of_view = field_of_view +# self.dimension = dimension + + +# class AxisMap(RetinotopyMap): +# """Abstract two-dimensional map of responses to stimuli along a single response axis (e.g., altitude) with unit +# """ + +# __nwbfields__ = ('unit', ) + +# @docval(*get_docval(RetinotopyMap.__init__, 'name', 'data', 'field_of_view'), +# {'name': 'unit', 'type': str, 'doc': 'Unit that axis data is stored in (e.g., degrees)'}, +# *get_docval(RetinotopyMap.__init__, 'dimension')) +# def __init__(self, **kwargs): +# unit = popargs('unit', kwargs) +# super().__init__(**kwargs) +# self.unit = unit + + +# @register_class('ImagingRetinotopy', CORE_NAMESPACE) +# class ImagingRetinotopy(NWBDataInterface): +# """ +# Intrinsic signal optical imaging or widefield imaging for measuring retinotopy. Stores orthogonal +# maps (e.g., altitude/azimuth; radius/theta) of responses to specific stimuli and a combined +# polarity map from which to identify visual areas. +# This group does not store the raw responses imaged during retinotopic mapping or the +# stimuli presented, but rather the resulting phase and power maps after applying a Fourier +# transform on the averaged responses. +# Note: for data consistency, all images and arrays are stored in the format [row][column] and +# [row, col], which equates to [y][x]. Field of view and dimension arrays may appear backward +# (i.e., y before x). +# """ + +# __nwbfields__ = ({'name': 'sign_map', 'child': True}, +# {'name': 'axis_1_phase_map', 'child': True}, +# {'name': 'axis_1_power_map', 'child': True}, +# {'name': 'axis_2_phase_map', 'child': True}, +# {'name': 'axis_2_power_map', 'child': True}, +# {'name': 'focal_depth_image', 'child': True}, +# {'name': 'vasculature_image', 'child': True}, +# 'axis_descriptions') + +# @docval({'name': 'sign_map', 'type': RetinotopyMap, +# 'doc': 'Sine of the angle between the direction of the gradient in axis_1 and axis_2.'}, +# {'name': 'axis_1_phase_map', 'type': AxisMap, +# 'doc': 'Phase response to stimulus on the first measured axis.'}, +# {'name': 'axis_1_power_map', 'type': AxisMap, +# 'doc': 'Power response on the first measured axis. Response is scaled so 0.0 is no power in ' +# 'the response and 1.0 is maximum relative power.'}, +# {'name': 'axis_2_phase_map', 'type': AxisMap, +# 'doc': 'Phase response to stimulus on the second measured axis.'}, +# {'name': 'axis_2_power_map', 'type': AxisMap, +# 'doc': 'Power response on the second measured axis. Response is scaled so 0.0 is no ' +# 'power in the response and 1.0 is maximum relative power.'}, +# {'name': 'axis_descriptions', 'type': Iterable, 'shape': (2, ), +# 'doc': 'Two-element array describing the contents of the two response axis fields. ' +# 'Description should be something like ["altitude", "azimuth"] or ["radius", "theta"].'}, +# {'name': 'focal_depth_image', 'type': FocalDepthImage, +# 'doc': 'Gray-scale image taken with same settings/parameters (e.g., focal depth, wavelength) ' +# 'as data collection. Array format: [rows][columns].'}, +# {'name': 'vasculature_image', 'type': RetinotopyImage, +# 'doc': 'Gray-scale anatomical image of cortical surface. Array structure: [rows][columns].'}, +# {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'ImagingRetinotopy'}) +# def __init__(self, **kwargs): +# axis_1_phase_map, axis_1_power_map, axis_2_phase_map, axis_2_power_map, axis_descriptions, \ +# focal_depth_image, sign_map, vasculature_image = popargs( +# 'axis_1_phase_map', 'axis_1_power_map', 'axis_2_phase_map', 'axis_2_power_map', +# 'axis_descriptions', 'focal_depth_image', 'sign_map', 'vasculature_image', kwargs) +# super().__init__(**kwargs) +# self.axis_1_phase_map = axis_1_phase_map +# self.axis_1_power_map = axis_1_power_map +# self.axis_2_phase_map = axis_2_phase_map +# self.axis_2_power_map = axis_2_power_map +# self.axis_descriptions = axis_descriptions +# self.focal_depth_image = focal_depth_image +# self.sign_map = sign_map +# self.vasculature_image = vasculature_image From efaebe03c484ad179c9cb91c3955bd0c38eaa1d1 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Thu, 4 Jan 2024 14:11:38 -0800 Subject: [PATCH 02/17] Remove references to retinotopy module --- docs/gallery/general/plot_file.py | 5 +- docs/source/api_docs.rst | 1 - src/pynwb/__init__.py | 1 - src/pynwb/io/__init__.py | 1 - src/pynwb/legacy/io/__init__.py | 1 - tests/unit/test_retinotopy.py | 348 +++++++++++++++--------------- 6 files changed, 176 insertions(+), 181 deletions(-) diff --git a/docs/gallery/general/plot_file.py b/docs/gallery/general/plot_file.py index beead22f6..2175f8e9e 100644 --- a/docs/gallery/general/plot_file.py +++ b/docs/gallery/general/plot_file.py @@ -108,8 +108,7 @@ :py:class:`~pynwb.ophys.ImageSegmentation`, :py:class:`~pynwb.ophys.MotionCorrection`. - * **Others:** :py:class:`~pynwb.retinotopy.ImagingRetinotopy`, - :py:class:`~pynwb.base.Images`. + * **Others:** :py:class:`~pynwb.base.Images`. * **TimeSeries:** Any :ref:`timeseries_overview` is also a subclass of :py:class:`~pynwb.core.NWBDataInterface` and can be used anywhere :py:class:`~pynwb.core.NWBDataInterface` is allowed. @@ -372,7 +371,7 @@ # Processing modules can be thought of as folders within the file for storing the related processed data. # # .. tip:: Use the NWB schema module names as processing module names where appropriate. -# These are: ``"behavior"``, ``"ecephys"``, ``"icephys"``, ``"ophys"``, ``"ogen"``, ``"retinotopy"``, and ``"misc"``. +# These are: ``"behavior"``, ``"ecephys"``, ``"icephys"``, ``"ophys"``, ``"ogen"``, and ``"misc"``. # # Let's assume that the subject's position was computed from a video tracking algorithm, # so it would be classified as processed data. diff --git a/docs/source/api_docs.rst b/docs/source/api_docs.rst index 3920ad316..94bf1957d 100644 --- a/docs/source/api_docs.rst +++ b/docs/source/api_docs.rst @@ -13,7 +13,6 @@ API Documentation Intracellular Electrophysiology Optophysiology Optogenetics - Retinotopy General Imaging Behavior NWB Base Classes diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 6e3b3104f..5d9bbc57b 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -381,7 +381,6 @@ def export(self, **kwargs): from . import misc # noqa: F401,E402 from . import ogen # noqa: F401,E402 from . import ophys # noqa: F401,E402 -from . import retinotopy # noqa: F401,E402 from . import legacy # noqa: F401,E402 from hdmf.data_utils import DataChunkIterator # noqa: F401,E402 from hdmf.backends.hdf5 import H5DataIO # noqa: F401,E402 diff --git a/src/pynwb/io/__init__.py b/src/pynwb/io/__init__.py index b7e9bea48..e0de46b87 100644 --- a/src/pynwb/io/__init__.py +++ b/src/pynwb/io/__init__.py @@ -9,4 +9,3 @@ from . import misc as __misc from . import ogen as __ogen from . import ophys as __ophys -from . import retinotopy as __retinotopy diff --git a/src/pynwb/legacy/io/__init__.py b/src/pynwb/legacy/io/__init__.py index 97dc7b9d2..70e1d7a77 100644 --- a/src/pynwb/legacy/io/__init__.py +++ b/src/pynwb/legacy/io/__init__.py @@ -8,4 +8,3 @@ from . import misc as __misc from . import ogen as __ogen from . import ophys as __ophys -from . import retinotopy as __retinotopy diff --git a/tests/unit/test_retinotopy.py b/tests/unit/test_retinotopy.py index 57942d274..e66ecfc32 100644 --- a/tests/unit/test_retinotopy.py +++ b/tests/unit/test_retinotopy.py @@ -1,174 +1,174 @@ -import numpy as np - -from pynwb.retinotopy import ImagingRetinotopy, AxisMap, RetinotopyImage, FocalDepthImage, RetinotopyMap -from pynwb.testing import TestCase - - -class ImageRetinotopyConstructor(TestCase): - - def setUp(self): - data = np.ones((2, 2)) - field_of_view = [1, 2] - dimension = [1, 2] - self.sign_map = RetinotopyMap('sign_map', data, field_of_view, dimension) - self.axis_1_phase_map = AxisMap('axis_1_phase_map', data, field_of_view, 'unit', dimension) - self.axis_1_power_map = AxisMap('axis_1_power_map', data, field_of_view, 'unit', dimension) - self.axis_2_phase_map = AxisMap('axis_2_phase_map', data, field_of_view, 'unit', dimension) - self.axis_2_power_map = AxisMap('axis_2_power_map', data, field_of_view, 'unit', dimension) - self.axis_descriptions = ['altitude', 'azimuth'] - - data = [[1, 1], [1, 1]] - bits_per_pixel = 8 - dimension = [3, 4] - format = 'raw' - field_of_view = [1, 2] - focal_depth = 1.0 - self.focal_depth_image = FocalDepthImage('focal_depth_image', data, bits_per_pixel, dimension, format, - field_of_view, focal_depth) - self.vasculature_image = RetinotopyImage('vasculature_image', np.uint16(data), bits_per_pixel, dimension, - format, field_of_view) - - def test_init(self): - """Test that ImagingRetinotopy constructor sets properties correctly.""" - msg = ('The ImagingRetinotopy class currently cannot be written to or read from a file. This is a known bug ' - 'and will be fixed in a future release of PyNWB.') - with self.assertWarnsWith(UserWarning, msg): - ir = ImagingRetinotopy(self.sign_map, self.axis_1_phase_map, self.axis_1_power_map, self.axis_2_phase_map, - self.axis_2_power_map, self.axis_descriptions, self.focal_depth_image, - self.vasculature_image) - self.assertEqual(ir.sign_map, self.sign_map) - self.assertEqual(ir.axis_1_phase_map, self.axis_1_phase_map) - self.assertEqual(ir.axis_1_power_map, self.axis_1_power_map) - self.assertEqual(ir.axis_2_phase_map, self.axis_2_phase_map) - self.assertEqual(ir.axis_2_power_map, self.axis_2_power_map) - self.assertEqual(ir.axis_descriptions, self.axis_descriptions) - self.assertEqual(ir.focal_depth_image, self.focal_depth_image) - self.assertEqual(ir.vasculature_image, self.vasculature_image) - - def test_init_axis_descriptions_wrong_shape(self): - """Test that creating a ImagingRetinotopy with a axis descriptions argument that is not 2 elements raises an - error. - """ - self.axis_descriptions = ['altitude', 'azimuth', 'extra'] - - msg = "ImagingRetinotopy.__init__: incorrect shape for 'axis_descriptions' (got '(3,)', expected '(2,)')" - with self.assertRaisesWith(ValueError, msg): - ImagingRetinotopy(self.sign_map, self.axis_1_phase_map, self.axis_1_power_map, self.axis_2_phase_map, - self.axis_2_power_map, self.axis_descriptions, self.focal_depth_image, - self.vasculature_image) - - -class RetinotopyImageConstructor(TestCase): - - def test_init(self): - """Test that RetinotopyImage constructor sets properties correctly.""" - data = [[1, 1], [1, 1]] - bits_per_pixel = 8 - dimension = [3, 4] - format = 'raw' - field_of_view = [1, 2] - image = RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) - - self.assertEqual(image.name, 'vasculature_image') - self.assertEqual(image.data, data) - self.assertEqual(image.bits_per_pixel, bits_per_pixel) - self.assertEqual(image.dimension, dimension) - self.assertEqual(image.format, format) - self.assertEqual(image.field_of_view, field_of_view) - - def test_init_dimension_wrong_shape(self): - """Test that creating a RetinotopyImage with a dimension argument that is not 2 elements raises an error.""" - data = [[1, 1], [1, 1]] - bits_per_pixel = 8 - dimension = [3, 4, 5] - format = 'raw' - field_of_view = [1, 2] - - msg = "RetinotopyImage.__init__: incorrect shape for 'dimension' (got '(3,)', expected '(2,)')" - with self.assertRaisesWith(ValueError, msg): - RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) - - def test_init_fov_wrong_shape(self): - """Test that creating a RetinotopyImage with a field of view argument that is not 2 elements raises an error.""" - data = [[1, 1], [1, 1]] - bits_per_pixel = 8 - dimension = [3, 4] - format = 'raw' - field_of_view = [1, 2, 3] - - msg = "RetinotopyImage.__init__: incorrect shape for 'field_of_view' (got '(3,)', expected '(2,)')" - with self.assertRaisesWith(ValueError, msg): - RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) - - -class RetinotopyMapConstructor(TestCase): - - def test_init(self): - """Test that RetinotopyMap constructor sets properties correctly.""" - data = np.ones((2, 2)) - field_of_view = [1, 2] - dimension = [1, 2] - map = RetinotopyMap('sign_map', data, field_of_view, dimension) - - self.assertEqual(map.name, 'sign_map') - np.testing.assert_array_equal(map.data, data) - self.assertEqual(map.field_of_view, field_of_view) - self.assertEqual(map.dimension, dimension) - - -class AxisMapConstructor(TestCase): - - def test_init(self): - """Test that AxisMap constructor sets properties correctly.""" - data = np.ones((2, 2)) - field_of_view = [1, 2] - dimension = [1, 2] - map = AxisMap('axis_1_phase', data, field_of_view, 'unit', dimension) - - self.assertEqual(map.name, 'axis_1_phase') - np.testing.assert_array_equal(map.data, data) - self.assertEqual(map.field_of_view, field_of_view) - self.assertEqual(map.dimension, dimension) - self.assertEqual(map.unit, 'unit') - - def test_init_dimension_wrong_shape(self): - """Test that creating an AxisMap with a dimension argument that is not 2 elements raises an error.""" - data = np.ones((2, 2)) - field_of_view = [1, 2] - dimension = [1, 2, 3] - - msg = "AxisMap.__init__: incorrect shape for 'dimension' (got '(3,)', expected '(2,)')" - with self.assertRaisesWith(ValueError, msg): - AxisMap('axis_1_phase', data, field_of_view, 'unit', dimension) - - def test_init_fov_wrong_shape(self): - """Test that creating an AxisMap with a dimension argument that is not 2 elements raises an error.""" - data = np.ones((2, 2)) - field_of_view = [1, 2, 3] - dimension = [1, 2] - - msg = "AxisMap.__init__: incorrect shape for 'field_of_view' (got '(3,)', expected '(2,)')" - with self.assertRaisesWith(ValueError, msg): - AxisMap('axis_1_phase', data, field_of_view, 'unit', dimension) - - -class FocalDepthImageConstructor(TestCase): - - def test_init(self): - """Test that FocalDepthImage constructor sets properties correctly.""" - data = [[1, 1], [1, 1]] - bits_per_pixel = 8 - dimension = [3, 4] - format = 'raw' - field_of_view = [1, 2] - focal_depth = 1.0 - image = FocalDepthImage('focal_depth_image', data, bits_per_pixel, dimension, format, field_of_view, - focal_depth) - - self.assertEqual(image.name, 'focal_depth_image') - self.assertEqual(image.data, data) - self.assertEqual(image.bits_per_pixel, bits_per_pixel) - self.assertEqual(image.dimension, dimension) - self.assertEqual(image.format, format) - self.assertEqual(image.field_of_view, field_of_view) - self.assertEqual(image.focal_depth, focal_depth) +# import numpy as np + +# from pynwb.retinotopy import ImagingRetinotopy, AxisMap, RetinotopyImage, FocalDepthImage, RetinotopyMap +# from pynwb.testing import TestCase + + +# class ImageRetinotopyConstructor(TestCase): + +# def setUp(self): +# data = np.ones((2, 2)) +# field_of_view = [1, 2] +# dimension = [1, 2] +# self.sign_map = RetinotopyMap('sign_map', data, field_of_view, dimension) +# self.axis_1_phase_map = AxisMap('axis_1_phase_map', data, field_of_view, 'unit', dimension) +# self.axis_1_power_map = AxisMap('axis_1_power_map', data, field_of_view, 'unit', dimension) +# self.axis_2_phase_map = AxisMap('axis_2_phase_map', data, field_of_view, 'unit', dimension) +# self.axis_2_power_map = AxisMap('axis_2_power_map', data, field_of_view, 'unit', dimension) +# self.axis_descriptions = ['altitude', 'azimuth'] + +# data = [[1, 1], [1, 1]] +# bits_per_pixel = 8 +# dimension = [3, 4] +# format = 'raw' +# field_of_view = [1, 2] +# focal_depth = 1.0 +# self.focal_depth_image = FocalDepthImage('focal_depth_image', data, bits_per_pixel, dimension, format, +# field_of_view, focal_depth) +# self.vasculature_image = RetinotopyImage('vasculature_image', np.uint16(data), bits_per_pixel, dimension, +# format, field_of_view) + +# def test_init(self): +# """Test that ImagingRetinotopy constructor sets properties correctly.""" +# msg = ('The ImagingRetinotopy class currently cannot be written to or read from a file. This is a known bug ' +# 'and will be fixed in a future release of PyNWB.') +# with self.assertWarnsWith(UserWarning, msg): +# ir = ImagingRetinotopy(self.sign_map, self.axis_1_phase_map, self.axis_1_power_map, self.axis_2_phase_map, +# self.axis_2_power_map, self.axis_descriptions, self.focal_depth_image, +# self.vasculature_image) +# self.assertEqual(ir.sign_map, self.sign_map) +# self.assertEqual(ir.axis_1_phase_map, self.axis_1_phase_map) +# self.assertEqual(ir.axis_1_power_map, self.axis_1_power_map) +# self.assertEqual(ir.axis_2_phase_map, self.axis_2_phase_map) +# self.assertEqual(ir.axis_2_power_map, self.axis_2_power_map) +# self.assertEqual(ir.axis_descriptions, self.axis_descriptions) +# self.assertEqual(ir.focal_depth_image, self.focal_depth_image) +# self.assertEqual(ir.vasculature_image, self.vasculature_image) + +# def test_init_axis_descriptions_wrong_shape(self): +# """Test that creating a ImagingRetinotopy with a axis descriptions argument that is not 2 elements raises an +# error. +# """ +# self.axis_descriptions = ['altitude', 'azimuth', 'extra'] + +# msg = "ImagingRetinotopy.__init__: incorrect shape for 'axis_descriptions' (got '(3,)', expected '(2,)')" +# with self.assertRaisesWith(ValueError, msg): +# ImagingRetinotopy(self.sign_map, self.axis_1_phase_map, self.axis_1_power_map, self.axis_2_phase_map, +# self.axis_2_power_map, self.axis_descriptions, self.focal_depth_image, +# self.vasculature_image) + + +# class RetinotopyImageConstructor(TestCase): + +# def test_init(self): +# """Test that RetinotopyImage constructor sets properties correctly.""" +# data = [[1, 1], [1, 1]] +# bits_per_pixel = 8 +# dimension = [3, 4] +# format = 'raw' +# field_of_view = [1, 2] +# image = RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) + +# self.assertEqual(image.name, 'vasculature_image') +# self.assertEqual(image.data, data) +# self.assertEqual(image.bits_per_pixel, bits_per_pixel) +# self.assertEqual(image.dimension, dimension) +# self.assertEqual(image.format, format) +# self.assertEqual(image.field_of_view, field_of_view) + +# def test_init_dimension_wrong_shape(self): +# """Test that creating a RetinotopyImage with a dimension argument that is not 2 elements raises an error.""" +# data = [[1, 1], [1, 1]] +# bits_per_pixel = 8 +# dimension = [3, 4, 5] +# format = 'raw' +# field_of_view = [1, 2] + +# msg = "RetinotopyImage.__init__: incorrect shape for 'dimension' (got '(3,)', expected '(2,)')" +# with self.assertRaisesWith(ValueError, msg): +# RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) + +# def test_init_fov_wrong_shape(self): +# """Test that creating a RetinotopyImage with a field of view argument that is not 2 elements raises an error.""" +# data = [[1, 1], [1, 1]] +# bits_per_pixel = 8 +# dimension = [3, 4] +# format = 'raw' +# field_of_view = [1, 2, 3] + +# msg = "RetinotopyImage.__init__: incorrect shape for 'field_of_view' (got '(3,)', expected '(2,)')" +# with self.assertRaisesWith(ValueError, msg): +# RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) + + +# class RetinotopyMapConstructor(TestCase): + +# def test_init(self): +# """Test that RetinotopyMap constructor sets properties correctly.""" +# data = np.ones((2, 2)) +# field_of_view = [1, 2] +# dimension = [1, 2] +# map = RetinotopyMap('sign_map', data, field_of_view, dimension) + +# self.assertEqual(map.name, 'sign_map') +# np.testing.assert_array_equal(map.data, data) +# self.assertEqual(map.field_of_view, field_of_view) +# self.assertEqual(map.dimension, dimension) + + +# class AxisMapConstructor(TestCase): + +# def test_init(self): +# """Test that AxisMap constructor sets properties correctly.""" +# data = np.ones((2, 2)) +# field_of_view = [1, 2] +# dimension = [1, 2] +# map = AxisMap('axis_1_phase', data, field_of_view, 'unit', dimension) + +# self.assertEqual(map.name, 'axis_1_phase') +# np.testing.assert_array_equal(map.data, data) +# self.assertEqual(map.field_of_view, field_of_view) +# self.assertEqual(map.dimension, dimension) +# self.assertEqual(map.unit, 'unit') + +# def test_init_dimension_wrong_shape(self): +# """Test that creating an AxisMap with a dimension argument that is not 2 elements raises an error.""" +# data = np.ones((2, 2)) +# field_of_view = [1, 2] +# dimension = [1, 2, 3] + +# msg = "AxisMap.__init__: incorrect shape for 'dimension' (got '(3,)', expected '(2,)')" +# with self.assertRaisesWith(ValueError, msg): +# AxisMap('axis_1_phase', data, field_of_view, 'unit', dimension) + +# def test_init_fov_wrong_shape(self): +# """Test that creating an AxisMap with a dimension argument that is not 2 elements raises an error.""" +# data = np.ones((2, 2)) +# field_of_view = [1, 2, 3] +# dimension = [1, 2] + +# msg = "AxisMap.__init__: incorrect shape for 'field_of_view' (got '(3,)', expected '(2,)')" +# with self.assertRaisesWith(ValueError, msg): +# AxisMap('axis_1_phase', data, field_of_view, 'unit', dimension) + + +# class FocalDepthImageConstructor(TestCase): + +# def test_init(self): +# """Test that FocalDepthImage constructor sets properties correctly.""" +# data = [[1, 1], [1, 1]] +# bits_per_pixel = 8 +# dimension = [3, 4] +# format = 'raw' +# field_of_view = [1, 2] +# focal_depth = 1.0 +# image = FocalDepthImage('focal_depth_image', data, bits_per_pixel, dimension, format, field_of_view, +# focal_depth) + +# self.assertEqual(image.name, 'focal_depth_image') +# self.assertEqual(image.data, data) +# self.assertEqual(image.bits_per_pixel, bits_per_pixel) +# self.assertEqual(image.dimension, dimension) +# self.assertEqual(image.format, format) +# self.assertEqual(image.field_of_view, field_of_view) +# self.assertEqual(image.focal_depth, focal_depth) From b697e40a1227dfb5228b3ea3ab850cda8954c0fd Mon Sep 17 00:00:00 2001 From: rly Date: Thu, 4 Jan 2024 14:14:12 -0800 Subject: [PATCH 03/17] Update schema --- src/pynwb/nwb-schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index b4f8838cb..596108f80 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit b4f8838cbfbb7f8a117bd7e0aad19133d26868b4 +Subproject commit 596108f801122ba8d4266b2f3f639634f44df1e4 From d46d2f9357d1497c15307b42db2a212d43404f57 Mon Sep 17 00:00:00 2001 From: rly Date: Thu, 4 Jan 2024 14:16:27 -0800 Subject: [PATCH 04/17] Fix test --- tests/back_compat/test_import_structure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/back_compat/test_import_structure.py b/tests/back_compat/test_import_structure.py index 79d4f6ad0..36831929d 100644 --- a/tests/back_compat/test_import_structure.py +++ b/tests/back_compat/test_import_structure.py @@ -78,7 +78,6 @@ def test_outer_import_structure(self): "popargs", "register_class", "register_map", - "retinotopy", "spec", "testing", "validate", From 0bf4e4c80f6f7e13e206700d9063c885d6bea51c Mon Sep 17 00:00:00 2001 From: rly Date: Thu, 4 Jan 2024 14:19:01 -0800 Subject: [PATCH 05/17] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5cfa5a9e..1c019309e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## PyNWB 2.6.0 (Upcoming) ### Enhancements and minor changes +- Added support for NWB Schema 2.7.0. + - Deprecated `ImagingRetinotopy` neurodata type. @rly - For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748) - Add `NWBHDF5IO.can_read()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703) - Add `pynwb.get_nwbfile_version()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703) From 7e92a2001b8bfa94d981ade35ff01efb0ad6cf46 Mon Sep 17 00:00:00 2001 From: rly Date: Fri, 12 Jan 2024 12:44:02 -0800 Subject: [PATCH 06/17] Update schema submodule --- src/pynwb/nwb-schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 596108f80..308c5d80d 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 596108f801122ba8d4266b2f3f639634f44df1e4 +Subproject commit 308c5d80d842f2e45cf25042cb12887ec8c7cbab From 35b831d2d1d2c7400668aa16a7a5bce332c64de8 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 12 Jan 2024 12:49:05 -0800 Subject: [PATCH 07/17] Update test_retinotopy.py --- tests/unit/test_retinotopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_retinotopy.py b/tests/unit/test_retinotopy.py index e66ecfc32..f2f5ab33d 100644 --- a/tests/unit/test_retinotopy.py +++ b/tests/unit/test_retinotopy.py @@ -89,7 +89,7 @@ # RetinotopyImage('vasculature_image', data, bits_per_pixel, dimension, format, field_of_view) # def test_init_fov_wrong_shape(self): -# """Test that creating a RetinotopyImage with a field of view argument that is not 2 elements raises an error.""" +# """Test that creating a RetinotopyImage with a field of view that is not 2 elements raises an error.""" # data = [[1, 1], [1, 1]] # bits_per_pixel = 8 # dimension = [3, 4] From 6e5b2b95687b1f02109a9a35ccb6758b8fce803d Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 12 Jan 2024 12:51:51 -0800 Subject: [PATCH 08/17] Update retinotopy.py --- src/pynwb/retinotopy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index 55f92a3ff..4382e81d6 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -26,7 +26,8 @@ # {'name': 'bits_per_pixel', 'type': int, # 'doc': 'Number of bits used to represent each value. This is necessary to determine maximum ' # '(white) pixel value.'}, -# {'name': 'dimension', 'type': Iterable, 'shape': (2, ), 'doc': 'Number of rows and columns in the image.'}, +# {'name': 'dimension', 'type': Iterable, 'shape': (2, ), +# 'doc': 'Number of rows and columns in the image.'}, # {'name': 'format', 'type': Iterable, 'doc': 'Format of image. Right now only "raw" supported.'}, # {'name': 'field_of_view', 'type': Iterable, 'shape': (2, ), 'doc': 'Size of viewing area, in meters.'}) # def __init__(self, **kwargs): From a03cfabbe5fe5f6f670aa5fc2d6949173e265ef4 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 12 Jan 2024 12:53:30 -0800 Subject: [PATCH 09/17] Update test_retinotopy.py --- tests/unit/test_retinotopy.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_retinotopy.py b/tests/unit/test_retinotopy.py index f2f5ab33d..1f23646df 100644 --- a/tests/unit/test_retinotopy.py +++ b/tests/unit/test_retinotopy.py @@ -1,3 +1,11 @@ +from pynwb.testing import TestCase + + +class TestRetinotopy(TestCase): + def test_retinotopy_deprecated(self): + with self.assertRaises(ValueError): + import pynwb.retinotopy + # import numpy as np # from pynwb.retinotopy import ImagingRetinotopy, AxisMap, RetinotopyImage, FocalDepthImage, RetinotopyMap From 3eebe3dc66233a58bf7af2e53201fb39862c2730 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 12 Jan 2024 12:54:43 -0800 Subject: [PATCH 10/17] Update retinotopy.py --- src/pynwb/retinotopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index 4382e81d6..e5773723b 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -26,7 +26,7 @@ # {'name': 'bits_per_pixel', 'type': int, # 'doc': 'Number of bits used to represent each value. This is necessary to determine maximum ' # '(white) pixel value.'}, -# {'name': 'dimension', 'type': Iterable, 'shape': (2, ), +# {'name': 'dimension', 'type': Iterable, 'shape': (2, ), # 'doc': 'Number of rows and columns in the image.'}, # {'name': 'format', 'type': Iterable, 'doc': 'Format of image. Right now only "raw" supported.'}, # {'name': 'field_of_view', 'type': Iterable, 'shape': (2, ), 'doc': 'Size of viewing area, in meters.'}) From 8661bc483b8f0b95a70777202c9fc36c53ffac34 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 12 Jan 2024 12:55:01 -0800 Subject: [PATCH 11/17] Update test_retinotopy.py --- tests/unit/test_retinotopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_retinotopy.py b/tests/unit/test_retinotopy.py index 1f23646df..fbb7a3c42 100644 --- a/tests/unit/test_retinotopy.py +++ b/tests/unit/test_retinotopy.py @@ -4,7 +4,7 @@ class TestRetinotopy(TestCase): def test_retinotopy_deprecated(self): with self.assertRaises(ValueError): - import pynwb.retinotopy + import pynwb.retinotopy # noqa: F401 # import numpy as np From 2da48bc7d78180d87783e7e20e99bc7ad9f997e5 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Sat, 13 Jan 2024 00:04:43 -0800 Subject: [PATCH 12/17] Update retinotopy.py --- src/pynwb/retinotopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index e5773723b..c0db417d8 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -1,4 +1,4 @@ -raise ValueError( +raise RuntimeError( "The pynwb.retinotopy module is deprecated. If you are interested in using these neurodata types, " "please create an issue on https://github.com/NeurodataWithoutBorders/nwb-schema/issues." ) From 9b19bffcd4966fed0ae70acbf62e02cca439df8d Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Sat, 13 Jan 2024 00:04:59 -0800 Subject: [PATCH 13/17] Update test_retinotopy.py --- tests/unit/test_retinotopy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_retinotopy.py b/tests/unit/test_retinotopy.py index fbb7a3c42..ce0a19e07 100644 --- a/tests/unit/test_retinotopy.py +++ b/tests/unit/test_retinotopy.py @@ -3,7 +3,7 @@ class TestRetinotopy(TestCase): def test_retinotopy_deprecated(self): - with self.assertRaises(ValueError): + with self.assertRaises(RuntimeError): import pynwb.retinotopy # noqa: F401 # import numpy as np From e42f9fa9047a7af42ff1903d427fef2baa2317ac Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 6 Feb 2024 16:51:45 -0800 Subject: [PATCH 14/17] RuntimeError -> ImportError --- src/pynwb/retinotopy.py | 2 +- tests/unit/test_retinotopy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynwb/retinotopy.py b/src/pynwb/retinotopy.py index c0db417d8..1dc0f365d 100644 --- a/src/pynwb/retinotopy.py +++ b/src/pynwb/retinotopy.py @@ -1,4 +1,4 @@ -raise RuntimeError( +raise ImportError( "The pynwb.retinotopy module is deprecated. If you are interested in using these neurodata types, " "please create an issue on https://github.com/NeurodataWithoutBorders/nwb-schema/issues." ) diff --git a/tests/unit/test_retinotopy.py b/tests/unit/test_retinotopy.py index ce0a19e07..9a9f67748 100644 --- a/tests/unit/test_retinotopy.py +++ b/tests/unit/test_retinotopy.py @@ -3,7 +3,7 @@ class TestRetinotopy(TestCase): def test_retinotopy_deprecated(self): - with self.assertRaises(RuntimeError): + with self.assertRaises(ImportError): import pynwb.retinotopy # noqa: F401 # import numpy as np From 08f3a885c84ec49ed52384aa01415c6d892301db Mon Sep 17 00:00:00 2001 From: rly Date: Tue, 6 Feb 2024 16:53:27 -0800 Subject: [PATCH 15/17] Update schema --- src/pynwb/nwb-schema | 2 +- src/pynwb/validation.py | 249 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 src/pynwb/validation.py diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 308c5d80d..f352a93c4 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 308c5d80d842f2e45cf25042cb12887ec8c7cbab +Subproject commit f352a93c4cbfb202b6a40210e998cab34b05a593 diff --git a/src/pynwb/validation.py b/src/pynwb/validation.py new file mode 100644 index 000000000..778f834fe --- /dev/null +++ b/src/pynwb/validation.py @@ -0,0 +1,249 @@ +"""Command line tool to Validate an NWB file against a namespace. + +As of PyNWB 3.0, the validate function and the validate CLI tool validate an NWB file or an open IO object +in the same way. + +""" +import sys +from argparse import ArgumentParser +from typing import Tuple, List, Dict, Optional +import warnings + +from hdmf.spec import NamespaceCatalog +from hdmf.build import BuildManager +from hdmf.build import TypeMap as TypeMap +from hdmf.utils import docval, getargs +from hdmf.backends.io import HDMFIO +from hdmf.validate import ValidatorMap + +from pynwb import CORE_NAMESPACE +from pynwb.spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace + + +def _print_errors(validation_errors: list): + if validation_errors: + print(" - found the following errors:", file=sys.stderr) + for err in validation_errors: + print(str(err), file=sys.stderr) + else: + print(" - no errors found.") + + +def _validate_helper(io: HDMFIO, namespace: str = CORE_NAMESPACE) -> list: + builder = io.read_builder() + validator = ValidatorMap(io.manager.namespace_catalog.get_namespace(name=namespace)) + return validator.validate(builder) + + +def _get_cached_namespaces_to_validate( + path: str, driver: Optional[str] = None +) -> Tuple[List[str], BuildManager, Dict[str, str]]: + """ + Determine the most specific namespace(s) that are cached in the given NWBFile that can be used for validation. + + Example + ------- + The following example illustrates how we can use this function to validate against namespaces + cached in a file. This is useful, e.g., when a file was created using an extension + >>> from pynwb import validate + >>> from pynwb.validate import _get_cached_namespaces_to_validate + >>> path = "my_nwb_file.nwb" + >>> validate_namespaces, manager, cached_namespaces = _get_cached_namespaces_to_validate(path) + >>> with NWBHDF5IO(path, "r", manager=manager) as reader: + >>> errors = [] + >>> for ns in validate_namespaces: + >>> errors += validate(io=reader, namespace=ns) + :param path: Path for the NWB file + :return: Tuple with: + - List of strings with the most specific namespace(s) to use for validation. + - BuildManager object for opening the file for validation + - Dict with the full result from NWBHDF5IO.load_namespaces + """ + from . import NWBHDF5IO # TODO: modularize to avoid circular import + + catalog = NamespaceCatalog( + group_spec_cls=NWBGroupSpec, dataset_spec_cls=NWBDatasetSpec, spec_namespace_cls=NWBNamespace + ) + namespace_dependencies = NWBHDF5IO.load_namespaces(namespace_catalog=catalog, path=path, driver=driver) + + # Determine which namespaces are the most specific (i.e. extensions) and validate against those + candidate_namespaces = set(namespace_dependencies.keys()) + for namespace_dependency in namespace_dependencies: + candidate_namespaces -= namespace_dependencies[namespace_dependency].keys() + + # TODO: remove this workaround for issue https://github.com/NeurodataWithoutBorders/pynwb/issues/1357 + candidate_namespaces.discard("hdmf-experimental") # remove validation of hdmf-experimental for now + cached_namespaces = sorted(candidate_namespaces) + + if len(cached_namespaces) > 0: + type_map = TypeMap(namespaces=catalog) + manager = BuildManager(type_map=type_map) + else: + manager = None + + return cached_namespaces, manager, namespace_dependencies + + +@docval( + { + "name": "paths", + "type": list, + "doc": "List of NWB file paths.", + "default": None, + }, + { + "name": "namespace", + "type": str, + "doc": ("A specific namespace to validate against. As of PyNWB 3.0, if None and use_cached_namespaces=True, " + "this function will validate against all namespaces in the file except for dependent namespaces."), + "default": None, + }, + { + "name": "use_cached_namespaces", + "type": bool, + "doc": "Whether to use namespaces cached within the file for validation.", + "default": True, + }, + { + "name": "verbose", + "type": bool, + "doc": "Whether or not to print messages to stdout.", + "default": False, + }, + { + "name": "driver", + "type": str, + "doc": "Driver for h5py to use when opening the HDF5 file.", + "default": None, + }, + { + "name": "io", + "type": HDMFIO, + "doc": "An open IO to an NWB file. Deprecated in PyNWB 3.0 in favor of 'paths'.", + "default": None, + }, + returns="Validation errors in the file.", + rtype=(list, (list, bool)), + is_method=False, +) +def validate(**kwargs): + """Validate NWB file(s) against a namespace or its cached namespaces.""" + from . import NWBHDF5IO # TODO: modularize to avoid circular import + + if "io" in kwargs: + warnings.warn("The 'io' argument is deprecated. Please use 'paths' instead.", DeprecationWarning, stacklevel=3) + + io, paths, use_cached_namespaces, namespace, verbose, driver = getargs( + "io", "paths", "use_cached_namespaces", "namespace", "verbose", "driver", kwargs + ) + assert io != paths, "Both 'io' and 'paths' were specified! Please choose only one." + + if io is not None: + validation_errors = _validate_helper(io=io, namespace=namespace or CORE_NAMESPACE) + return validation_errors + + status = 0 + validation_errors = list() + for path in paths: + namespaces_to_validate = [] + namespace_message = "PyNWB namespace information" + io_kwargs = dict(path=path, mode="r", driver=driver) + + if use_cached_namespaces: + cached_namespaces, manager, namespace_dependencies = _get_cached_namespaces_to_validate( + path=path, driver=driver + ) + io_kwargs.update(manager=manager) + + if any(cached_namespaces): + namespaces_to_validate = cached_namespaces + namespace_message = "cached namespace information" + else: + namespaces_to_validate = [CORE_NAMESPACE] + if verbose: + print( + f"The file {path} has no cached namespace information. Falling back to {namespace_message}.", + file=sys.stderr, + ) + else: + io_kwargs.update(load_namespaces=False) + namespaces_to_validate = [CORE_NAMESPACE] + + if namespace is not None: + if namespace in namespaces_to_validate: + namespaces_to_validate = [namespace] + elif use_cached_namespaces and namespace in namespace_dependencies: # validating against a dependency + for namespace_dependency in namespace_dependencies: + if namespace in namespace_dependencies[namespace_dependency]: + status = 1 + print( + f"The namespace '{namespace}' is included by the namespace " + f"'{namespace_dependency}'. Please validate against that namespace instead.", + file=sys.stderr, + ) + else: + status = 1 + print( + f"The namespace '{namespace}' could not be found in {namespace_message} as only " + f"{namespaces_to_validate} is present.", + file=sys.stderr, + ) + + if status == 1: + continue + + with NWBHDF5IO(**io_kwargs) as io: + for validation_namespace in namespaces_to_validate: + if verbose: + print(f"Validating {path} against {namespace_message} using namespace '{validation_namespace}'.") + validation_errors += _validate_helper(io=io, namespace=validation_namespace) + return validation_errors, status + + +def validate_cli(): + """CLI wrapper around pynwb.validate.""" + parser = ArgumentParser( + description="Validate an NWB file", + epilog="If --ns is not specified, validate against all namespaces in the NWB file.", + ) + + # Special arg specific to CLI + parser.add_argument( + "-lns", + "--list-namespaces", + dest="list_namespaces", + action="store_true", + help="List the available namespaces and exit.", + ) + + # Common args to the API validate + parser.add_argument("paths", type=str, nargs="+", help="NWB file paths") + parser.add_argument("-n", "--ns", type=str, help="the namespace to validate against") + feature_parser = parser.add_mutually_exclusive_group(required=False) + feature_parser.add_argument( + "--no-cached-namespace", + dest="no_cached_namespace", + action="store_true", + help="Use the PyNWB loaded namespace (true) or use the cached namespace (false; default).", + ) + parser.set_defaults(no_cached_namespace=False) + args = parser.parse_args() + status = 0 + + if args.list_namespaces: + for path in args.paths: + cached_namespaces, _, _ = _get_cached_namespaces_to_validate(path=path) + print("\n".join(cached_namespaces)) + else: + validation_errors, validation_status = validate( + paths=args.paths, use_cached_namespaces=not args.no_cached_namespace, namespace=args.ns, verbose=True + ) + if not validation_status: + _print_errors(validation_errors=validation_errors) + status = status or validation_status or (validation_errors is not None and len(validation_errors) > 0) + + sys.exit(status) + + +if __name__ == "__main__": # pragma: no cover + validate_cli() From a0eafa874a044ee3430d0cc787aa04d18c238d3b Mon Sep 17 00:00:00 2001 From: rly Date: Tue, 6 Feb 2024 16:54:05 -0800 Subject: [PATCH 16/17] Revert "Update schema" This reverts commit 08f3a885c84ec49ed52384aa01415c6d892301db. --- src/pynwb/nwb-schema | 2 +- src/pynwb/validation.py | 249 ---------------------------------------- 2 files changed, 1 insertion(+), 250 deletions(-) delete mode 100644 src/pynwb/validation.py diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index f352a93c4..308c5d80d 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit f352a93c4cbfb202b6a40210e998cab34b05a593 +Subproject commit 308c5d80d842f2e45cf25042cb12887ec8c7cbab diff --git a/src/pynwb/validation.py b/src/pynwb/validation.py deleted file mode 100644 index 778f834fe..000000000 --- a/src/pynwb/validation.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Command line tool to Validate an NWB file against a namespace. - -As of PyNWB 3.0, the validate function and the validate CLI tool validate an NWB file or an open IO object -in the same way. - -""" -import sys -from argparse import ArgumentParser -from typing import Tuple, List, Dict, Optional -import warnings - -from hdmf.spec import NamespaceCatalog -from hdmf.build import BuildManager -from hdmf.build import TypeMap as TypeMap -from hdmf.utils import docval, getargs -from hdmf.backends.io import HDMFIO -from hdmf.validate import ValidatorMap - -from pynwb import CORE_NAMESPACE -from pynwb.spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace - - -def _print_errors(validation_errors: list): - if validation_errors: - print(" - found the following errors:", file=sys.stderr) - for err in validation_errors: - print(str(err), file=sys.stderr) - else: - print(" - no errors found.") - - -def _validate_helper(io: HDMFIO, namespace: str = CORE_NAMESPACE) -> list: - builder = io.read_builder() - validator = ValidatorMap(io.manager.namespace_catalog.get_namespace(name=namespace)) - return validator.validate(builder) - - -def _get_cached_namespaces_to_validate( - path: str, driver: Optional[str] = None -) -> Tuple[List[str], BuildManager, Dict[str, str]]: - """ - Determine the most specific namespace(s) that are cached in the given NWBFile that can be used for validation. - - Example - ------- - The following example illustrates how we can use this function to validate against namespaces - cached in a file. This is useful, e.g., when a file was created using an extension - >>> from pynwb import validate - >>> from pynwb.validate import _get_cached_namespaces_to_validate - >>> path = "my_nwb_file.nwb" - >>> validate_namespaces, manager, cached_namespaces = _get_cached_namespaces_to_validate(path) - >>> with NWBHDF5IO(path, "r", manager=manager) as reader: - >>> errors = [] - >>> for ns in validate_namespaces: - >>> errors += validate(io=reader, namespace=ns) - :param path: Path for the NWB file - :return: Tuple with: - - List of strings with the most specific namespace(s) to use for validation. - - BuildManager object for opening the file for validation - - Dict with the full result from NWBHDF5IO.load_namespaces - """ - from . import NWBHDF5IO # TODO: modularize to avoid circular import - - catalog = NamespaceCatalog( - group_spec_cls=NWBGroupSpec, dataset_spec_cls=NWBDatasetSpec, spec_namespace_cls=NWBNamespace - ) - namespace_dependencies = NWBHDF5IO.load_namespaces(namespace_catalog=catalog, path=path, driver=driver) - - # Determine which namespaces are the most specific (i.e. extensions) and validate against those - candidate_namespaces = set(namespace_dependencies.keys()) - for namespace_dependency in namespace_dependencies: - candidate_namespaces -= namespace_dependencies[namespace_dependency].keys() - - # TODO: remove this workaround for issue https://github.com/NeurodataWithoutBorders/pynwb/issues/1357 - candidate_namespaces.discard("hdmf-experimental") # remove validation of hdmf-experimental for now - cached_namespaces = sorted(candidate_namespaces) - - if len(cached_namespaces) > 0: - type_map = TypeMap(namespaces=catalog) - manager = BuildManager(type_map=type_map) - else: - manager = None - - return cached_namespaces, manager, namespace_dependencies - - -@docval( - { - "name": "paths", - "type": list, - "doc": "List of NWB file paths.", - "default": None, - }, - { - "name": "namespace", - "type": str, - "doc": ("A specific namespace to validate against. As of PyNWB 3.0, if None and use_cached_namespaces=True, " - "this function will validate against all namespaces in the file except for dependent namespaces."), - "default": None, - }, - { - "name": "use_cached_namespaces", - "type": bool, - "doc": "Whether to use namespaces cached within the file for validation.", - "default": True, - }, - { - "name": "verbose", - "type": bool, - "doc": "Whether or not to print messages to stdout.", - "default": False, - }, - { - "name": "driver", - "type": str, - "doc": "Driver for h5py to use when opening the HDF5 file.", - "default": None, - }, - { - "name": "io", - "type": HDMFIO, - "doc": "An open IO to an NWB file. Deprecated in PyNWB 3.0 in favor of 'paths'.", - "default": None, - }, - returns="Validation errors in the file.", - rtype=(list, (list, bool)), - is_method=False, -) -def validate(**kwargs): - """Validate NWB file(s) against a namespace or its cached namespaces.""" - from . import NWBHDF5IO # TODO: modularize to avoid circular import - - if "io" in kwargs: - warnings.warn("The 'io' argument is deprecated. Please use 'paths' instead.", DeprecationWarning, stacklevel=3) - - io, paths, use_cached_namespaces, namespace, verbose, driver = getargs( - "io", "paths", "use_cached_namespaces", "namespace", "verbose", "driver", kwargs - ) - assert io != paths, "Both 'io' and 'paths' were specified! Please choose only one." - - if io is not None: - validation_errors = _validate_helper(io=io, namespace=namespace or CORE_NAMESPACE) - return validation_errors - - status = 0 - validation_errors = list() - for path in paths: - namespaces_to_validate = [] - namespace_message = "PyNWB namespace information" - io_kwargs = dict(path=path, mode="r", driver=driver) - - if use_cached_namespaces: - cached_namespaces, manager, namespace_dependencies = _get_cached_namespaces_to_validate( - path=path, driver=driver - ) - io_kwargs.update(manager=manager) - - if any(cached_namespaces): - namespaces_to_validate = cached_namespaces - namespace_message = "cached namespace information" - else: - namespaces_to_validate = [CORE_NAMESPACE] - if verbose: - print( - f"The file {path} has no cached namespace information. Falling back to {namespace_message}.", - file=sys.stderr, - ) - else: - io_kwargs.update(load_namespaces=False) - namespaces_to_validate = [CORE_NAMESPACE] - - if namespace is not None: - if namespace in namespaces_to_validate: - namespaces_to_validate = [namespace] - elif use_cached_namespaces and namespace in namespace_dependencies: # validating against a dependency - for namespace_dependency in namespace_dependencies: - if namespace in namespace_dependencies[namespace_dependency]: - status = 1 - print( - f"The namespace '{namespace}' is included by the namespace " - f"'{namespace_dependency}'. Please validate against that namespace instead.", - file=sys.stderr, - ) - else: - status = 1 - print( - f"The namespace '{namespace}' could not be found in {namespace_message} as only " - f"{namespaces_to_validate} is present.", - file=sys.stderr, - ) - - if status == 1: - continue - - with NWBHDF5IO(**io_kwargs) as io: - for validation_namespace in namespaces_to_validate: - if verbose: - print(f"Validating {path} against {namespace_message} using namespace '{validation_namespace}'.") - validation_errors += _validate_helper(io=io, namespace=validation_namespace) - return validation_errors, status - - -def validate_cli(): - """CLI wrapper around pynwb.validate.""" - parser = ArgumentParser( - description="Validate an NWB file", - epilog="If --ns is not specified, validate against all namespaces in the NWB file.", - ) - - # Special arg specific to CLI - parser.add_argument( - "-lns", - "--list-namespaces", - dest="list_namespaces", - action="store_true", - help="List the available namespaces and exit.", - ) - - # Common args to the API validate - parser.add_argument("paths", type=str, nargs="+", help="NWB file paths") - parser.add_argument("-n", "--ns", type=str, help="the namespace to validate against") - feature_parser = parser.add_mutually_exclusive_group(required=False) - feature_parser.add_argument( - "--no-cached-namespace", - dest="no_cached_namespace", - action="store_true", - help="Use the PyNWB loaded namespace (true) or use the cached namespace (false; default).", - ) - parser.set_defaults(no_cached_namespace=False) - args = parser.parse_args() - status = 0 - - if args.list_namespaces: - for path in args.paths: - cached_namespaces, _, _ = _get_cached_namespaces_to_validate(path=path) - print("\n".join(cached_namespaces)) - else: - validation_errors, validation_status = validate( - paths=args.paths, use_cached_namespaces=not args.no_cached_namespace, namespace=args.ns, verbose=True - ) - if not validation_status: - _print_errors(validation_errors=validation_errors) - status = status or validation_status or (validation_errors is not None and len(validation_errors) > 0) - - sys.exit(status) - - -if __name__ == "__main__": # pragma: no cover - validate_cli() From ee361ffef6d12478da0e99107bd4e5b3cf635c25 Mon Sep 17 00:00:00 2001 From: rly Date: Tue, 6 Feb 2024 16:54:53 -0800 Subject: [PATCH 17/17] Update schema --- src/pynwb/nwb-schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 308c5d80d..f352a93c4 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 308c5d80d842f2e45cf25042cb12887ec8c7cbab +Subproject commit f352a93c4cbfb202b6a40210e998cab34b05a593