Merge branch 'main' into python-3.11

.github/workflows/deploy-tests.yml

@@ -3,12 +3,11 @@ name: Deploy tests
 on:
   pull_request:
 
+concurrency:  # Cancel previous workflows on the same pull request
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  cancel-previous-runs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/[email protected]
 
   assess-file-changes:
     uses: catalystneuro/neuroconv/.github/workflows/assess-file-changes.yml@main

.github/workflows/testing.yml

@@ -156,7 +156,7 @@ jobs:
         run: pytest -rsx -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml
       - name: Upload full coverage to Codecov
         if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest' }}
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./codecov.xml

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.1.0
     hooks:
     -   id: black
         exclude: ^docs/

CHANGELOG.md

@@ -1,12 +1,20 @@
 # Upcoming
 
+### Features
+* The `OpenEphysRecordingInterface` is now a wrapper for `OpenEphysBinaryRecordingInterface`. [PR #294](https://github.com/catalystneuro/neuroconv/pull/294)
+
+
+# v0.2.4
+
 ### Deprecation
 * All usages of `use_times` have been removed from spikeinterface tools and interfaces. The function `add_electrical_series` now determines whether the timestamps of the spikeinterface recording extractor are uniform or not and automatically stores the data according to best practices [PR #40](https://github.com/catalystneuro/neuroconv/pull/40)
+* Dropped Python 3.7 support. [PR #237](https://github.com/catalystneuro/neuroconv/pull/237)
 
 ### Features
 * Added a tool for determining rising and falling frames from TTL signals (`parse_rising_frames_from_ttl` and `get_falling_frames_from_ttl`). [PR #244](https://github.com/catalystneuro/neuroconv/pull/244)
 * Added the `SpikeGLXNIDQInterface` for reading data from `.nidq.bin` files, as well as the ability to parse event times from specific channels via the `get_event_starting_times_from_ttl` method. Also included a `neuroconv.tools.testing.MockSpikeGLXNIDQInterface` for testing purposes. [PR #247](https://github.com/catalystneuro/neuroconv/pull/247)
 * Improved handling of writing multiple probes to the same `NWB` file [PR #255](https://github.com/catalystneuro/neuroconv/pull/255)
+* Added basic temporal alignment methods to all DataInterfaces. These are `get_timestamps`, `align_starting_time`, `align_timestamps`, and `align_by_interpolation`. Added tests that serve as a first demonstration of the intended uses in a variety of cases. [PR #237](https://github.com/catalystneuro/neuroconv/pull/237)
 
 ### Pending deprecation
 * Added `DeprecationWarnings` to all `spikeextractors` backends. [PR #265](https://github.com/catalystneuro/neuroconv/pull/265)
@@ -15,6 +23,7 @@
 ### Fixes
 * Temporarily hotfixed the `tensorflow` dependency after the release of `deeplabcut==2.3.0`. [PR #268](https://github.com/catalystneuro/neuroconv/pull/268)
 * Fixed cleanup of waveform tests in SI tools. [PR #277](https://github.com/catalystneuro/neuroconv/pull/277)
+* Fixed metadata structure for the CsvTimeIntervalsInterface, which was previously not passed validation in NWBConverters. [PR #237](https://github.com/catalystneuro/neuroconv/pull/237)
 * Added propagation of the `load_sync_channel` argument for the `SpikeGLXNIDQInterface`. [PR #282](https://github.com/catalystneuro/neuroconv/pull/282)
 * Fixed the default `es_key` used by stand-alone write using any `RecordingExtractorInterface` or `LFPExtractorInterface`. [PR #288](https://github.com/catalystneuro/neuroconv/pull/288)
 * Fixed the default `ExtractorName` used to load the spikeinterface extractor of the `SpikeGLXLFPInterface`. [PR #288](https://github.com/catalystneuro/neuroconv/pull/288)

setup.py

@@ -44,7 +44,7 @@
 
 setup(
     name="neuroconv",
-    version="0.2.4",
+    version="0.2.5",
     description="Convert data from proprietary formats to NWB format.",
     long_description=long_description,
     long_description_content_type="text/markdown",

src/neuroconv/basedatainterface.py

@@ -1,8 +1,9 @@
 """Authors: Cody Baker and Ben Dichter."""
-from abc import abstractmethod, ABC
 import uuid
+from abc import abstractmethod, ABC
 from typing import Optional
 
+import numpy as np
 from pynwb import NWBFile
 
 from .utils import get_base_schema, get_schema_from_method_signature
@@ -22,7 +23,7 @@ def get_conversion_options_schema(cls):
         return get_schema_from_method_signature(cls.run_conversion, exclude=["nwbfile", "metadata"])
 
     def __init__(self, **source_data):
-        self.source_data = source_data
+        self.source_data: dict = source_data
 
     def get_metadata_schema(self):
         """Retrieve JSON schema for metadata."""
@@ -46,6 +47,91 @@ def get_metadata(self):
 
         return metadata
 
+    @abstractmethod
+    def get_original_timestamps(self) -> np.ndarray:
+        """
+        Retrieve the original unaltered timestamps for the data in this interface.
+
+        This function should retrieve the data on-demand by re-initializing the IO.
+
+        Returns
+        -------
+        timestamps: numpy.ndarray
+            The timestamps for the data stream.
+        """
+        raise NotImplementedError(
+            "Unable to retrieve the original unaltered timestamps for this interface! "
+            "Define the `get_original_timestamps` method for this interface."
+        )
+
+    @abstractmethod
+    def get_timestamps(self) -> np.ndarray:
+        """
+        Retrieve the timestamps for the data in this interface.
+
+        Returns
+        -------
+        timestamps: numpy.ndarray
+            The timestamps for the data stream.
+        """
+        raise NotImplementedError(
+            "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface."
+        )
+
+    @abstractmethod
+    def align_timestamps(self, aligned_timestamps: np.ndarray):
+        """
+        Replace all timestamps for this interface with those aligned to the common session start time.
+
+        Must be in units seconds relative to the common 'session_start_time'.
+
+        Parameters
+        ----------
+        aligned_timestamps : numpy.ndarray
+            The synchronized timestamps for data in this interface.
+        """
+        raise NotImplementedError(
+            "The protocol for synchronizing the timestamps of this interface has not been specified!"
+        )
+
+    def align_starting_time(self, starting_time: float):
+        """
+        Align the starting time for this interface relative to the common session start time.
+
+        Must be in units seconds relative to the common 'session_start_time'.
+
+        Parameters
+        ----------
+        starting_time : float
+            The starting time for all temporal data in this interface.
+        """
+        self.align_timestamps(aligned_timestamps=self.get_timestamps() + starting_time)
+
+    def align_by_interpolation(self, unaligned_timestamps: np.ndarray, aligned_timestamps: np.ndarray):
+        """
+        Interpolate the timestamps of this interface using a mapping from some unaligned time basis to its aligned one.
+
+        Use this method if the unaligned timestamps of the data in this interface are not directly tracked by a primary
+        system, but are known to occur between timestamps that are tracked, then align the timestamps of this interface
+        by interpolating between the two.
+
+        An example could be a metronomic TTL pulse (e.g., every second) from a secondary data stream to the primary
+        timing system; if the time references of this interface are recorded within the relative time of the secondary
+        data stream, then their exact time in the primary system is inferred given the pulse times.
+
+        Must be in units seconds relative to the common 'session_start_time'.
+
+        Parameters
+        ----------
+        unaligned_timestamps : numpy.ndarray
+            The timestamps of the unaligned secondary time basis.
+        aligned_timestamps : numpy.ndarray
+            The timestamps aligned to the primary time basis.
+        """
+        self.align_timestamps(
+            aligned_timestamps=np.interp(x=self.get_timestamps(), xp=unaligned_timestamps, fp=aligned_timestamps)
+        )
+
     def get_conversion_options(self):
         """Child DataInterface classes should override this to match their conversion options."""
         return dict()
@@ -78,5 +164,4 @@ def run_conversion(
             If 'nwbfile_path' is specified, informs user after a successful write operation.
             The default is True.
         """
-
         raise NotImplementedError("The run_conversion method for this DataInterface has not been defined!")

src/neuroconv/datainterfaces/__init__.py

@@ -19,8 +19,8 @@
     BlackrockRecordingInterface,
     BlackrockSortingInterface,
 )
-from .ecephys.openephys.openephysdatainterface import (
-    OpenEphysRecordingInterface,
+from .ecephys.openephys.openephysbinarydatainterface import (
+    OpenEphysRecordingInterface,  # temporary import until transitioning to the renamed interface
     OpenEphysSortingInterface,
 )
 from .ecephys.axona.axonadatainterface import (

src/neuroconv/datainterfaces/behavior/audio/audiointerface.py

@@ -2,6 +2,7 @@
 from typing import Optional
 from warnings import warn
 
+import numpy as np
 from scipy.io.wavfile import read
 
 from neuroconv.basedatainterface import BaseDataInterface
@@ -79,7 +80,6 @@ def __init__(self, file_paths: list, verbose: bool = False):
         super().__init__(file_paths=file_paths)
 
     def get_metadata_schema(self):
-
         metadata_schema = super().get_metadata_schema()
         time_series_metadata_schema = get_schema_from_hdmf_class(TimeSeries)
         metadata_schema["properties"]["Behavior"] = get_base_schema(tag="Behavior")
@@ -112,6 +112,22 @@ def get_metadata(self) -> dict:
         metadata.update(Behavior=behavior_metadata)
         return metadata
 
+    def get_original_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve the original unaltered timestamps for this interface! "
+            "Define the `get_original_timestamps` method for this interface."
+        )
+
+    def get_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface."
+        )
+
+    def align_timestamps(self, aligned_timestamps: np.ndarray):
+        raise NotImplementedError(
+            "The protocol for synchronizing the timestamps of this interface has not been specified!"
+        )
+
     def run_conversion(
         self,
         nwbfile_path: Optional[FilePathType] = None,

src/neuroconv/datainterfaces/behavior/deeplabcut/deeplabcutdatainterface.py

@@ -2,6 +2,7 @@
 from typing import Optional
 from pathlib import Path
 
+import numpy as np
 from pynwb.file import NWBFile
 
 from ....basedatainterface import BaseDataInterface
@@ -81,6 +82,22 @@ def get_metadata(self):
         )
         return metadata
 
+    def get_original_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve the original unaltered timestamps for this interface! "
+            "Define the `get_original_timestamps` method for this interface."
+        )
+
+    def get_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface."
+        )
+
+    def align_timestamps(self, aligned_timestamps: np.ndarray):
+        raise NotImplementedError(
+            "The protocol for synchronizing the timestamps of this interface has not been specified!"
+        )
+
     def run_conversion(
         self,
         nwbfile_path: OptionalFilePathType = None,

src/neuroconv/datainterfaces/behavior/sleap/sleapdatainterface.py

@@ -2,6 +2,7 @@
 from typing import Optional
 from pathlib import Path
 
+import numpy as np
 from pynwb.file import NWBFile
 
 from ....basedatainterface import BaseDataInterface
@@ -44,6 +45,22 @@ def __init__(
         self.verbose = verbose
         super().__init__(file_path=file_path)
 
+    def get_original_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve the original unaltered timestamps for this interface! "
+            "Define the `get_original_timestamps` method for this interface."
+        )
+
+    def get_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface."
+        )
+
+    def align_timestamps(self, aligned_timestamps: np.ndarray):
+        raise NotImplementedError(
+            "The protocol for synchronizing the timestamps of this interface has not been specified!"
+        )
+
     def run_conversion(
         self,
         nwbfile_path: OptionalFilePathType = None,
@@ -81,7 +98,6 @@ def run_conversion(
         with make_or_load_nwbfile(
             nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=self.verbose
         ) as nwbfile_out:
-
             labels = self.sleap_io.load_slp(self.file_path)
             nwbfile_out = self.sleap_io.io.nwb.append_nwb_data(
                 labels=labels, nwbfile=nwbfile_out, pose_estimation_metadata=pose_estimation_metadata

src/neuroconv/datainterfaces/behavior/video/videodatainterface.py

@@ -91,7 +91,6 @@ def get_metadata_schema(self):
         return metadata_schema
 
     def get_metadata(self):
-
         metadata = super().get_metadata()
         behavior_metadata = dict(
             Movies=[
@@ -103,6 +102,22 @@ def get_metadata(self):
 
         return metadata
 
+    def get_original_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve the original unaltered timestamps for this interface! "
+            "Define the `get_original_timestamps` method for this interface."
+        )
+
+    def get_timestamps(self) -> np.ndarray:
+        raise NotImplementedError(
+            "Unable to retrieve timestamps for this interface! Define the `get_timestamps` method for this interface."
+        )
+
+    def align_timestamps(self, aligned_timestamps: np.ndarray):
+        raise NotImplementedError(
+            "The protocol for synchronizing the timestamps of this interface has not been specified!"
+        )
+
     def run_conversion(
         self,
         nwbfile_path: Optional[FilePathType] = None,
@@ -233,7 +248,6 @@ def run_conversion(
             nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=self.verbose
         ) as nwbfile_out:
             for j, (image_series_kwargs, file_list) in enumerate(zip(videos_metadata_unique, file_paths_list)):
-
                 with VideoCaptureContext(str(file_list[0])) as vc:
                     fps = vc.get_video_fps()
                     max_frames = stub_frames if stub_test else None

src/neuroconv/datainterfaces/ecephys/axona/axona_utils.py

@@ -111,7 +111,6 @@ def read_all_eeg_file_lfp_data(file_path: FilePathType) -> np.ndarray:
     eeg_memmaps = list()
     sampling_rates = set()
     for fname in file_path_list:
-
         sampling_rates.add(get_eeg_sampling_frequency(parent_path / fname))
 
         eeg_memmaps.append(read_eeg_file_lfp_data(parent_path / fname))
@@ -397,7 +396,6 @@ def get_position_object(file_path: FilePathType) -> Position:
     position_timestamps = position_data[:, 0]
 
     for ichan in range(position_data.shape[1]):
-
         spatial_series = SpatialSeries(
             name=position_channel_names[ichan],
             timestamps=position_timestamps,

src/neuroconv/datainterfaces/ecephys/axona/axonadatainterface.py

@@ -32,7 +32,6 @@ def __init__(self, file_path: FilePathType, verbose: bool = True):
         self.recording_extractor.set_channel_groups(tetrode_id)
 
     def extract_nwb_file_metadata(self):
-
         raw_annotations = self.recording_extractor.neo_reader.raw_annotations
         session_start_time = raw_annotations["blocks"][0]["segments"][0]["rec_datetime"]
         session_description = self.metadata_in_set_file["comments"]
@@ -76,7 +75,6 @@ def extract_ecephys_metadata(self):
         return ecephys_metadata
 
     def get_metadata(self):
-
         metadata = super().get_metadata()
 
         nwbfile_metadata = self.extract_nwb_file_metadata()
@@ -124,7 +122,6 @@ def get_source_schema(cls):
         )
 
     def __init__(self, file_path: FilePathType):
-
         data = read_all_eeg_file_lfp_data(file_path).T
         sampling_frequency = get_eeg_sampling_frequency(file_path)
         super().__init__(traces_list=[data], sampling_frequency=sampling_frequency)