[Pydantic IVa] Add validation to most interfaces and converters (#1022)

Co-authored-by: CodyCBakerPhD <[email protected]>

CHANGELOG.md

@@ -6,17 +6,19 @@
 ### Features
 * Added `get_stream_names` to `OpenEphysRecordingInterface`: [PR #1039](https://github.com/catalystneuro/neuroconv/pull/1039)
 
-### Bug fixes
-
 ### Improvements
 * Using ruff to enforce existence of public classes' docstrings [PR #1034](https://github.com/catalystneuro/neuroconv/pull/1034)
 * Separated tests that use external data by modality [PR #1049](https://github.com/catalystneuro/neuroconv/pull/1049)
 
+
+
 ## v0.6.1 (August 30, 2024)
 
 ### Bug fixes
 * Fixed the JSON schema inference warning on excluded fields; also improved error message reporting of which method triggered the error. [PR #1037](https://github.com/catalystneuro/neuroconv/pull/1037)
 
+
+
 ## v0.6.0 (August 27, 2024)
 
 ### Deprecations
@@ -38,6 +40,7 @@
 * Added helper function `neuroconv.tools.data_transfers.submit_aws_batch_job` for basic automated submission of AWS batch jobs. [PR #384](https://github.com/catalystneuro/neuroconv/pull/384)
 * Data interfaces `run_conversion` method now performs metadata validation before running the conversion. [PR #949](https://github.com/catalystneuro/neuroconv/pull/949)
 * Introduced `null_values_for_properties` to `add_units_table` to give user control over null values behavior [PR #989](https://github.com/catalystneuro/neuroconv/pull/989)
+* Most data interfaces and converters now use Pydantic to validate their inputs, including existence of file and folder paths. [PR #1022](https://github.com/catalystneuro/neuroconv/pull/1022)
 
 ### Bug fixes
 * Fixed the default naming of multiple electrical series in the `SpikeGLXConverterPipe`. [PR #957](https://github.com/catalystneuro/neuroconv/pull/957)

src/neuroconv/basedatainterface.py

@@ -6,7 +6,7 @@
 from typing import Literal, Optional, Union
 
 from jsonschema.validators import validate
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb import NWBFile
 
 from .tools.nwb_helpers import (
@@ -39,6 +39,7 @@ def get_source_schema(cls) -> dict:
         """Infer the JSON schema for the source_data from the method signature (annotation typing)."""
         return get_json_schema_from_method_signature(cls, exclude=["source_data"])
 
+    @validate_call
     def __init__(self, verbose: bool = False, **source_data):
         self.verbose = verbose
         self.source_data = source_data

src/neuroconv/datainterfaces/behavior/audio/audiointerface.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 import scipy
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
@@ -28,6 +28,7 @@ class AudioInterface(BaseTemporalAlignmentInterface):
     associated_suffixes = (".wav",)
     info = "Interface for writing audio recordings to an NWB file."
 
+    @validate_call
     def __init__(self, file_paths: list[FilePath], verbose: bool = False):
         """
         Data interface for writing acoustic recordings to an NWB file.

src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py

@@ -2,7 +2,7 @@
 from typing import Optional, Union
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb.file import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
@@ -25,6 +25,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["config_file_path"]["description"] = "Path to .yml config file"
         return source_schema
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py

@@ -6,7 +6,7 @@
 from typing import Optional, Union
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb.behavior import Position, SpatialSeries
 from pynwb.file import NWBFile
 
@@ -154,6 +154,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"]["description"] = "Path to the .dat file (the output of fictrac)"
         return source_schema
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/behavior/lightningpose/lightningposeconverter.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 from typing import Optional
 
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb import NWBFile
 
 from neuroconv import NWBConverter
@@ -26,6 +26,7 @@ class LightningPoseConverter(NWBConverter):
     def get_source_schema(cls):
         return get_schema_from_method_signature(cls)
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/behavior/lightningpose/lightningposedatainterface.py

@@ -5,7 +5,7 @@
 from typing import Optional
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb import NWBFile
 
 from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
@@ -58,6 +58,7 @@ def get_metadata_schema(self) -> dict:
 
         return metadata_schema
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/behavior/medpc/medpcdatainterface.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb.behavior import BehavioralEpochs, IntervalSeries
 from pynwb.file import NWBFile
 
@@ -41,6 +41,7 @@ class MedPCInterface(BaseTemporalAlignmentInterface):
     info = "Interface for handling MedPC output files."
     associated_suffixes = (".txt",)
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/behavior/miniscope/miniscopedatainterface.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from pydantic import DirectoryPath
+from pydantic import DirectoryPath, validate_call
 from pynwb import NWBFile
 
 from .... import BaseDataInterface
@@ -24,6 +24,7 @@ def get_source_schema(cls) -> dict:
         ] = "The main Miniscope folder. The movie files are expected to be in sub folders within the main folder."
         return source_schema
 
+    @validate_call
     def __init__(self, folder_path: DirectoryPath):
         """
         Initialize reading recordings from the Miniscope behavioral camera.

src/neuroconv/datainterfaces/behavior/neuralynx/neuralynx_nvt_interface.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb import NWBFile
 from pynwb.behavior import CompassDirection, Position, SpatialSeries
 
@@ -20,6 +20,7 @@ class NeuralynxNvtInterface(BaseTemporalAlignmentInterface):
     associated_suffixes = (".nvt",)
     info = "Interface for writing Neuralynx position tracking .nvt files to NWB."
 
+    @validate_call
     def __init__(self, file_path: FilePath, verbose: bool = True):
         """
         Interface for writing Neuralynx .nvt files to nwb.

src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb.file import NWBFile
 
 from .sleap_utils import extract_timestamps
@@ -27,6 +27,7 @@ def get_source_schema(cls) -> dict:
         ] = "Path of the video for extracting timestamps (optional)."
         return source_schema
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/behavior/video/videodatainterface.py

@@ -6,7 +6,7 @@
 import numpy as np
 import psutil
 from hdmf.data_utils import DataChunkIterator
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 from pynwb import NWBFile
 from pynwb.image import ImageSeries
 from tqdm import tqdm
@@ -28,6 +28,7 @@ class VideoInterface(BaseDataInterface):
     # Other suffixes, while they can be opened by OpenCV, are not supported by DANDI so should probably not list here
     info = "Interface for handling standard video file formats."
 
+    @validate_call
     def __init__(
         self,
         file_paths: list[FilePath],

src/neuroconv/datainterfaces/ecephys/openephys/openephyslegacydatainterface.py

@@ -2,7 +2,7 @@
 from typing import Optional
 from warnings import warn
 
-from pydantic import DirectoryPath
+from pydantic import DirectoryPath, validate_call
 
 from ..baserecordingextractorinterface import BaseRecordingExtractorInterface
 
@@ -35,6 +35,7 @@ def get_source_schema(cls):
 
         return source_schema
 
+    @validate_call
     def __init__(
         self,
         folder_path: DirectoryPath,

src/neuroconv/datainterfaces/ecephys/openephys/openephyssortingdatainterface.py

@@ -1,4 +1,4 @@
-from pydantic import DirectoryPath
+from pydantic import DirectoryPath, validate_call
 
 from ..basesortingextractorinterface import BaseSortingExtractorInterface
 from ....utils import get_schema_from_method_signature
@@ -23,6 +23,7 @@ def get_source_schema(cls) -> dict:
         metadata_schema["additionalProperties"] = False
         return metadata_schema
 
+    @validate_call
     def __init__(self, folder_path: DirectoryPath, experiment_id: int = 0, recording_id: int = 0):
         from spikeextractors import OpenEphysSortingExtractor
 

src/neuroconv/datainterfaces/ecephys/phy/phydatainterface.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import DirectoryPath
+from pydantic import DirectoryPath, validate_call
 
 from ..basesortingextractorinterface import BaseSortingExtractorInterface
 
@@ -24,6 +24,7 @@ def get_source_schema(cls) -> dict:
         ] = "Path to the output Phy folder (containing the params.py)."
         return source_schema
 
+    @validate_call
     def __init__(
         self,
         folder_path: DirectoryPath,

src/neuroconv/datainterfaces/ecephys/plexon/plexondatainterface.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 
 from ..baserecordingextractorinterface import BaseRecordingExtractorInterface
 from ..basesortingextractorinterface import BaseSortingExtractorInterface
@@ -24,6 +24,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"]["description"] = "Path to the .plx file."
         return source_schema
 
+    @validate_call
     def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"):
         """
         Load and prepare data for Plexon.
@@ -68,6 +69,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"]["description"] = "Path to the .pl2 file."
         return source_schema
 
+    @validate_call
     def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"):
         """
         Load and prepare data for Plexon.
@@ -119,6 +121,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"]["description"] = "Path to the plexon spiking data (.plx file)."
         return source_schema
 
+    @validate_call
     def __init__(self, file_path: FilePath, verbose: bool = True):
         """
         Load and prepare data for Plexon.

src/neuroconv/datainterfaces/ecephys/spike2/spike2datainterface.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 
 from ..baserecordingextractorinterface import BaseRecordingExtractorInterface
 from ....tools import get_package
@@ -40,6 +40,7 @@ def get_all_channels_info(cls, file_path: FilePath):
         _test_sonpy_installation()
         return cls.get_extractor().get_all_channels_info(file_path=file_path)
 
+    @validate_call
     def __init__(self, file_path: FilePath, verbose: bool = True, es_key: str = "ElectricalSeries"):
         """
         Initialize reading of Spike2 file.

src/neuroconv/datainterfaces/ecephys/spikegadgets/spikegadgetsdatainterface.py

@@ -22,6 +22,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"].update(description="Path to SpikeGadgets (.rec) file.")
         return source_schema
 
+    # @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxconverter.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Optional
 
-from pydantic import DirectoryPath
+from pydantic import DirectoryPath, validate_call
 
 from .spikeglxdatainterface import SpikeGLXRecordingInterface
 from .spikeglxnidqinterface import SpikeGLXNIDQInterface
@@ -33,6 +33,7 @@ def get_streams(cls, folder_path: DirectoryPath) -> list[str]:
 
         return SpikeGLXRecordingExtractor.get_streams(folder_path=folder_path)[0]
 
+    @validate_call
     def __init__(
         self,
         folder_path: DirectoryPath,

src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxdatainterface.py

@@ -4,7 +4,7 @@
 from typing import Optional
 
 import numpy as np
-from pydantic import FilePath
+from pydantic import FilePath, validate_call
 
 from .spikeglx_utils import (
     add_recording_extractor_properties,
@@ -42,6 +42,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX ap.bin or lf.bin file."
         return source_schema
 
+    @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/ecephys/spikeglx/spikeglxnidqinterface.py

@@ -25,6 +25,7 @@ def get_source_schema(cls) -> dict:
         source_schema["properties"]["file_path"]["description"] = "Path to SpikeGLX .nidq file."
         return source_schema
 
+    # @validate_call
     def __init__(
         self,
         file_path: FilePath,

src/neuroconv/datainterfaces/ecephys/tdt/tdtdatainterface.py

@@ -1,4 +1,4 @@
-from pydantic import DirectoryPath
+from pydantic import DirectoryPath, validate_call
 
 from ..baserecordingextractorinterface import BaseRecordingExtractorInterface
 
@@ -10,6 +10,7 @@ class TdtRecordingInterface(BaseRecordingExtractorInterface):
     associated_suffixes = (".tbk", ".tbx", ".tev", ".tsq")
     info = "Interface for TDT recording data."
 
+    @validate_call
     def __init__(
         self,
         folder_path: DirectoryPath,