Merge pull request #664 from catalystneuro/fix_yaml_on_new_dandi

Fix for latest DANDI release

CHANGELOG.md

@@ -7,10 +7,10 @@
 * Modify the filtering of traces to also filter out traces with empty values. [PR #649](https://github.com/catalystneuro/neuroconv/pull/649)
 * Added tool function `get_default_dataset_configurations` for identifying and collecting all fields of an in-memory `NWBFile` that could become datasets on disk; and return instances of the Pydantic dataset models filled with default values for chunking/buffering/compression. [PR #569](https://github.com/catalystneuro/neuroconv/pull/569)
 
-
 ### Fixes
 * Fixed GenericDataChunkIterator (in hdmf.py) in the case where the number of dimensions is 1 and the size in bytes is greater than the threshold of 1 GB. [PR #638](https://github.com/catalystneuro/neuroconv/pull/638)
 * Changed `np.floor` and `np.prod` usage to `math.floor` and `math.prod` in various files. [PR #638](https://github.com/catalystneuro/neuroconv/pull/638)
+* Updated minimal required version of DANDI CLI; updated `run_conversion_from_yaml` API function and tests to be compatible with naming changes. [PR #664](https://github.com/catalystneuro/neuroconv/pull/664)
 
 ### Improvements
 * Change metadata extraction library from `fparse` to `parse`. [PR #654](https://github.com/catalystneuro/neuroconv/pull/654)

setup.py

@@ -18,7 +18,7 @@
     testing_suite_dependencies = f.readlines()
 
 extras_require = defaultdict(list)
-extras_require["dandi"].append("dandi<0.58.1")  # TODO: fix >= 0.58.1
+extras_require["dandi"].append("dandi>=0.58.1")
 extras_require["full"].extend(extras_require["dandi"])
 
 extras_require.update(test=testing_suite_dependencies, docs=documentation_dependencies)

src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py

@@ -67,8 +67,8 @@ def run_conversion_from_yaml(
         If True, replaces any existing NWBFile at the nwbfile_path location, if save_to_file is True.
         If False, appends the existing NWBFile at the nwbfile_path location, if save_to_file is True.
     """
-    from dandi.metadata import _get_pynwb_metadata
     from dandi.organize import create_unique_filenames_from_metadata
+    from dandi.pynwb_utils import _get_pynwb_metadata
 
     if data_folder_path is None:
         data_folder_path = Path(specification_file_path).parent
@@ -125,18 +125,25 @@ def run_conversion_from_yaml(
             )
     # To properly mimic a true dandi organization, the full directory must be populated with NWBFiles.
     all_nwbfile_paths = [nwbfile_path for nwbfile_path in output_folder_path.iterdir() if nwbfile_path.suffix == ".nwb"]
-    if any(["temp_nwbfile_name_" in nwbfile_path.stem for nwbfile_path in all_nwbfile_paths]):
-        dandi_metadata_list = []
-        for nwbfile_path in all_nwbfile_paths:
-            dandi_metadata = _get_pynwb_metadata(path=nwbfile_path)
-            dandi_metadata.update(path=nwbfile_path)
+    nwbfile_paths_to_set = [
+        nwbfile_path for nwbfile_path in all_nwbfile_paths if "temp_nwbfile_name_" in nwbfile_path.stem
+    ]
+    if any(nwbfile_paths_to_set):
+        dandi_metadata_list = list()
+        for nwbfile_path_to_set in nwbfile_paths_to_set:
+            dandi_metadata = _get_pynwb_metadata(path=nwbfile_path_to_set)
+            dandi_metadata.update(path=nwbfile_path_to_set)
             dandi_metadata_list.append(dandi_metadata)
-        named_dandi_metadata_list = create_unique_filenames_from_metadata(metadata=dandi_metadata_list)
+        dandi_metadata_with_set_paths = create_unique_filenames_from_metadata(metadata=dandi_metadata_list)
 
-        for named_dandi_metadata in named_dandi_metadata_list:
-            if "temp_nwbfile_name_" in named_dandi_metadata["path"].stem:
-                dandi_filename = named_dandi_metadata["dandi_filename"].replace(" ", "_")
-                assert (
-                    dandi_filename != ".nwb"
-                ), f"Not enough metadata available to assign name to {str(named_dandi_metadata['path'])}!"
-                named_dandi_metadata["path"].rename(str(output_folder_path / dandi_filename))
+        for nwbfile_path_to_set, dandi_metadata_with_set_path in zip(
+            nwbfile_paths_to_set, dandi_metadata_with_set_paths
+        ):
+            dandi_filename = dandi_metadata_with_set_path["dandi_filename"]
+
+            assert (
+                dandi_filename != ".nwb"
+            ), f"Not enough metadata available to assign name to {str(nwbfile_path_to_set)}!"
+
+            # Rename file on system
+            nwbfile_path_to_set.rename(str(output_folder_path / dandi_filename))

tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml

@@ -28,6 +28,17 @@ experiments:
             sex: F
             age: P35D
             species: Mus musculus
+      - source_data:
+          ap:
+            file_path: spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin
+        metadata:
+          NWBFile:
+            session_start_time: "2020-11-09T21:19:09+00:00"
+          Subject:
+            subject_id: Mouse 1
+            sex: F
+            age: P35D
+            species: Mus musculus
       - nwbfile_name: example_defined_name
         metadata:
           NWBFile:

tests/test_on_data/test_yaml_conversion_specification.py

@@ -2,6 +2,7 @@
 import unittest
 from datetime import datetime
 from pathlib import Path
+from typing import Union
 
 import pytest
 from hdmf.testing import TestCase
@@ -51,7 +52,9 @@ def test_run_conversion_from_yaml():
         overwrite=True,
     )
 
-    with NWBHDF5IO(path=OUTPUT_PATH / "example_converter_spec_1.nwb", mode="r") as io:
+    nwbfile_path_1 = OUTPUT_PATH / "example_converter_spec_1.nwb"
+    assert nwbfile_path_1.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_1}'!"
+    with NWBHDF5IO(path=nwbfile_path_1, mode="r") as io:
         nwbfile = io.read()
         assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
         assert nwbfile.lab == "My Lab"
@@ -60,15 +63,19 @@ def test_run_conversion_from_yaml():
         assert nwbfile.subject.subject_id == "1"
         assert "ElectricalSeriesAP" in nwbfile.acquisition
 
-    with NWBHDF5IO(path=OUTPUT_PATH / "example_converter_spec_2.nwb", mode="r") as io:
+    nwbfile_path_2 = OUTPUT_PATH / "example_converter_spec_2.nwb"
+    assert nwbfile_path_2.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_2}'!"
+    with NWBHDF5IO(path=nwbfile_path_2, mode="r") as io:
         nwbfile = io.read()
         assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
         assert nwbfile.lab == "My Lab"
         assert nwbfile.institution == "My Institution"
         assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00")
         assert nwbfile.subject.subject_id == "002"
 
-    with NWBHDF5IO(path=OUTPUT_PATH / "example_converter_spec_3.nwb", mode="r") as io:
+    nwbfile_path_3 = OUTPUT_PATH / "example_converter_spec_3.nwb"
+    assert nwbfile_path_3.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_3}'!"
+    with NWBHDF5IO(path=nwbfile_path_3, mode="r") as io:
         nwbfile = io.read()
         assert nwbfile.session_description == "Auto-generated by neuroconv"
         assert nwbfile.lab == "My Lab"
@@ -93,22 +100,41 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self):
             overwrite=True,
         )
 
-        with NWBHDF5IO(path=self.test_folder / "sub-Mouse_1_ses-20201009T211909.nwb", mode="r") as io:
+        nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201009T211909.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
             assert nwbfile.lab == "My Lab"
             assert nwbfile.institution == "My Institution"
             assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00")
             assert nwbfile.subject.subject_id == "Mouse 1"
             assert "ElectricalSeriesAP" in nwbfile.acquisition
-        with NWBHDF5IO(path=self.test_folder / "example_defined_name.nwb", mode="r") as io:
+
+        nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
+            nwbfile = io.read()
+            assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
+            assert nwbfile.lab == "My Lab"
+            assert nwbfile.institution == "My Institution"
+            assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00")
+            assert nwbfile.subject.subject_id == "Mouse 1"
+            assert "ElectricalSeriesAP" in nwbfile.acquisition
+
+        nwbfile_path = self.test_folder / "example_defined_name.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
             assert nwbfile.lab == "My Lab"
             assert nwbfile.institution == "My Institution"
             assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00")
             assert nwbfile.subject.subject_id == "MyMouse002"
-        with NWBHDF5IO(path=self.test_folder / "sub-Subject_Name_ses-20201011T211909.nwb", mode="r") as io:
+
+        nwbfile_path = self.test_folder / "sub-Subject-Name_ses-20201011T211909.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Auto-generated by neuroconv"
             assert nwbfile.lab == "My Lab"
@@ -136,7 +162,7 @@ def test_run_conversion_from_yaml_no_nwbfile_name_or_other_metadata_assertion(se
                 overwrite=True,
             )
 
-    def test_run_conversion_from_yaml(self):
+    def test_run_conversion_from_yaml_on_behavior(self):
         path_to_test_yml_files = Path(__file__).parent / "conversion_specifications"
         yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_videos.yml"
         run_conversion_from_yaml(

tests/test_on_data/test_yaml_conversion_specification_cli.py

@@ -24,22 +24,30 @@ def test_run_conversion_from_yaml_cli(self):
             )
         )
 
-        with NWBHDF5IO(path=self.test_folder / "example_converter_spec_1.nwb", mode="r") as io:
+        nwbfile_path = self.test_folder / "example_converter_spec_1.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
             assert nwbfile.lab == "My Lab"
             assert nwbfile.institution == "My Institution"
             assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00")
             assert nwbfile.subject.subject_id == "1"
             assert "ElectricalSeriesAP" in nwbfile.acquisition
-        with NWBHDF5IO(path=self.test_folder / "example_converter_spec_2.nwb", mode="r") as io:
+
+        nwbfile_path = self.test_folder / "example_converter_spec_2.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
             assert nwbfile.lab == "My Lab"
             assert nwbfile.institution == "My Institution"
             assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00")
             assert nwbfile.subject.subject_id == "002"
-        with NWBHDF5IO(path=self.test_folder / "example_converter_spec_3.nwb", mode="r") as io:
+
+        nwbfile_path = self.test_folder / "example_converter_spec_3.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Auto-generated by neuroconv"
             assert nwbfile.lab == "My Lab"
@@ -60,22 +68,41 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self):
             )
         )
 
-        with NWBHDF5IO(path=self.test_folder / "sub-Mouse_1_ses-20201009T211909.nwb", mode="r") as io:
+        nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201009T211909.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
             assert nwbfile.lab == "My Lab"
             assert nwbfile.institution == "My Institution"
             assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00")
             assert nwbfile.subject.subject_id == "Mouse 1"
             assert "ElectricalSeriesAP" in nwbfile.acquisition
-        with NWBHDF5IO(path=self.test_folder / "example_defined_name.nwb", mode="r") as io:
+
+        nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
+            nwbfile = io.read()
+            assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
+            assert nwbfile.lab == "My Lab"
+            assert nwbfile.institution == "My Institution"
+            assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00")
+            assert nwbfile.subject.subject_id == "Mouse 1"
+            assert "ElectricalSeriesAP" in nwbfile.acquisition
+
+        nwbfile_path = self.test_folder / "example_defined_name.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Subject navigating a Y-shaped maze."
             assert nwbfile.lab == "My Lab"
             assert nwbfile.institution == "My Institution"
             assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00")
             assert nwbfile.subject.subject_id == "MyMouse002"
-        with NWBHDF5IO(path=self.test_folder / "sub-Subject_Name_ses-20201011T211909.nwb", mode="r") as io:
+
+        nwbfile_path = self.test_folder / "sub-Subject-Name_ses-20201011T211909.nwb"
+        assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! "
+        with NWBHDF5IO(path=nwbfile_path, mode="r") as io:
             nwbfile = io.read()
             assert nwbfile.session_description == "Auto-generated by neuroconv"
             assert nwbfile.lab == "My Lab"