diff --git a/CHANGELOG.md b/CHANGELOG.md index 10ab5b2e1..c46ac7994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,10 @@ * Modify the filtering of traces to also filter out traces with empty values. [PR #649](https://github.com/catalystneuro/neuroconv/pull/649) * Added tool function `get_default_dataset_configurations` for identifying and collecting all fields of an in-memory `NWBFile` that could become datasets on disk; and return instances of the Pydantic dataset models filled with default values for chunking/buffering/compression. [PR #569](https://github.com/catalystneuro/neuroconv/pull/569) - ### Fixes * Fixed GenericDataChunkIterator (in hdmf.py) in the case where the number of dimensions is 1 and the size in bytes is greater than the threshold of 1 GB. [PR #638](https://github.com/catalystneuro/neuroconv/pull/638) * Changed `np.floor` and `np.prod` usage to `math.floor` and `math.prod` in various files. [PR #638](https://github.com/catalystneuro/neuroconv/pull/638) +* Updated minimal required version of DANDI CLI; updated `run_conversion_from_yaml` API function and tests to be compatible with naming changes. [PR #664](https://github.com/catalystneuro/neuroconv/pull/664) ### Improvements * Change metadata extraction library from `fparse` to `parse`. [PR #654](https://github.com/catalystneuro/neuroconv/pull/654) diff --git a/setup.py b/setup.py index b6688a2bc..7ff18db27 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ testing_suite_dependencies = f.readlines() extras_require = defaultdict(list) -extras_require["dandi"].append("dandi<0.58.1") # TODO: fix >= 0.58.1 +extras_require["dandi"].append("dandi>=0.58.1") extras_require["full"].extend(extras_require["dandi"]) extras_require.update(test=testing_suite_dependencies, docs=documentation_dependencies) diff --git a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py index fe9537239..2c7e5b25c 100644 --- a/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py +++ b/src/neuroconv/tools/yaml_conversion_specification/_yaml_conversion_specification.py @@ -67,8 +67,8 @@ def run_conversion_from_yaml( If True, replaces any existing NWBFile at the nwbfile_path location, if save_to_file is True. If False, appends the existing NWBFile at the nwbfile_path location, if save_to_file is True. """ - from dandi.metadata import _get_pynwb_metadata from dandi.organize import create_unique_filenames_from_metadata + from dandi.pynwb_utils import _get_pynwb_metadata if data_folder_path is None: data_folder_path = Path(specification_file_path).parent @@ -125,18 +125,25 @@ def run_conversion_from_yaml( ) # To properly mimic a true dandi organization, the full directory must be populated with NWBFiles. all_nwbfile_paths = [nwbfile_path for nwbfile_path in output_folder_path.iterdir() if nwbfile_path.suffix == ".nwb"] - if any(["temp_nwbfile_name_" in nwbfile_path.stem for nwbfile_path in all_nwbfile_paths]): - dandi_metadata_list = [] - for nwbfile_path in all_nwbfile_paths: - dandi_metadata = _get_pynwb_metadata(path=nwbfile_path) - dandi_metadata.update(path=nwbfile_path) + nwbfile_paths_to_set = [ + nwbfile_path for nwbfile_path in all_nwbfile_paths if "temp_nwbfile_name_" in nwbfile_path.stem + ] + if any(nwbfile_paths_to_set): + dandi_metadata_list = list() + for nwbfile_path_to_set in nwbfile_paths_to_set: + dandi_metadata = _get_pynwb_metadata(path=nwbfile_path_to_set) + dandi_metadata.update(path=nwbfile_path_to_set) dandi_metadata_list.append(dandi_metadata) - named_dandi_metadata_list = create_unique_filenames_from_metadata(metadata=dandi_metadata_list) + dandi_metadata_with_set_paths = create_unique_filenames_from_metadata(metadata=dandi_metadata_list) - for named_dandi_metadata in named_dandi_metadata_list: - if "temp_nwbfile_name_" in named_dandi_metadata["path"].stem: - dandi_filename = named_dandi_metadata["dandi_filename"].replace(" ", "_") - assert ( - dandi_filename != ".nwb" - ), f"Not enough metadata available to assign name to {str(named_dandi_metadata['path'])}!" - named_dandi_metadata["path"].rename(str(output_folder_path / dandi_filename)) + for nwbfile_path_to_set, dandi_metadata_with_set_path in zip( + nwbfile_paths_to_set, dandi_metadata_with_set_paths + ): + dandi_filename = dandi_metadata_with_set_path["dandi_filename"] + + assert ( + dandi_filename != ".nwb" + ), f"Not enough metadata available to assign name to {str(nwbfile_path_to_set)}!" + + # Rename file on system + nwbfile_path_to_set.rename(str(output_folder_path / dandi_filename)) diff --git a/tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml b/tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml index 7ba295bcc..ff17b0f52 100644 --- a/tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml +++ b/tests/test_on_data/conversion_specifications/GIN_conversion_specification_missing_nwbfile_names.yml @@ -28,6 +28,17 @@ experiments: sex: F age: P35D species: Mus musculus + - source_data: + ap: + file_path: spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin + metadata: + NWBFile: + session_start_time: "2020-11-09T21:19:09+00:00" + Subject: + subject_id: Mouse 1 + sex: F + age: P35D + species: Mus musculus - nwbfile_name: example_defined_name metadata: NWBFile: diff --git a/tests/test_on_data/test_yaml_conversion_specification.py b/tests/test_on_data/test_yaml_conversion_specification.py index 5e44fac7c..f2abf210f 100644 --- a/tests/test_on_data/test_yaml_conversion_specification.py +++ b/tests/test_on_data/test_yaml_conversion_specification.py @@ -2,6 +2,7 @@ import unittest from datetime import datetime from pathlib import Path +from typing import Union import pytest from hdmf.testing import TestCase @@ -51,7 +52,9 @@ def test_run_conversion_from_yaml(): overwrite=True, ) - with NWBHDF5IO(path=OUTPUT_PATH / "example_converter_spec_1.nwb", mode="r") as io: + nwbfile_path_1 = OUTPUT_PATH / "example_converter_spec_1.nwb" + assert nwbfile_path_1.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_1}'!" + with NWBHDF5IO(path=nwbfile_path_1, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" @@ -60,7 +63,9 @@ def test_run_conversion_from_yaml(): assert nwbfile.subject.subject_id == "1" assert "ElectricalSeriesAP" in nwbfile.acquisition - with NWBHDF5IO(path=OUTPUT_PATH / "example_converter_spec_2.nwb", mode="r") as io: + nwbfile_path_2 = OUTPUT_PATH / "example_converter_spec_2.nwb" + assert nwbfile_path_2.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_2}'!" + with NWBHDF5IO(path=nwbfile_path_2, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" @@ -68,7 +73,9 @@ def test_run_conversion_from_yaml(): assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") assert nwbfile.subject.subject_id == "002" - with NWBHDF5IO(path=OUTPUT_PATH / "example_converter_spec_3.nwb", mode="r") as io: + nwbfile_path_3 = OUTPUT_PATH / "example_converter_spec_3.nwb" + assert nwbfile_path_3.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path_3}'!" + with NWBHDF5IO(path=nwbfile_path_3, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Auto-generated by neuroconv" assert nwbfile.lab == "My Lab" @@ -93,7 +100,9 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): overwrite=True, ) - with NWBHDF5IO(path=self.test_folder / "sub-Mouse_1_ses-20201009T211909.nwb", mode="r") as io: + nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201009T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" @@ -101,14 +110,31 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" assert "ElectricalSeriesAP" in nwbfile.acquisition - with NWBHDF5IO(path=self.test_folder / "example_defined_name.nwb", mode="r") as io: + + nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Mouse 1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_defined_name.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") assert nwbfile.subject.subject_id == "MyMouse002" - with NWBHDF5IO(path=self.test_folder / "sub-Subject_Name_ses-20201011T211909.nwb", mode="r") as io: + + nwbfile_path = self.test_folder / "sub-Subject-Name_ses-20201011T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Auto-generated by neuroconv" assert nwbfile.lab == "My Lab" @@ -136,7 +162,7 @@ def test_run_conversion_from_yaml_no_nwbfile_name_or_other_metadata_assertion(se overwrite=True, ) - def test_run_conversion_from_yaml(self): + def test_run_conversion_from_yaml_on_behavior(self): path_to_test_yml_files = Path(__file__).parent / "conversion_specifications" yaml_file_path = path_to_test_yml_files / "GIN_conversion_specification_videos.yml" run_conversion_from_yaml( diff --git a/tests/test_on_data/test_yaml_conversion_specification_cli.py b/tests/test_on_data/test_yaml_conversion_specification_cli.py index b65fb4c7c..bdadc9072 100644 --- a/tests/test_on_data/test_yaml_conversion_specification_cli.py +++ b/tests/test_on_data/test_yaml_conversion_specification_cli.py @@ -24,7 +24,9 @@ def test_run_conversion_from_yaml_cli(self): ) ) - with NWBHDF5IO(path=self.test_folder / "example_converter_spec_1.nwb", mode="r") as io: + nwbfile_path = self.test_folder / "example_converter_spec_1.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" @@ -32,14 +34,20 @@ def test_run_conversion_from_yaml_cli(self): assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "1" assert "ElectricalSeriesAP" in nwbfile.acquisition - with NWBHDF5IO(path=self.test_folder / "example_converter_spec_2.nwb", mode="r") as io: + + nwbfile_path = self.test_folder / "example_converter_spec_2.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") assert nwbfile.subject.subject_id == "002" - with NWBHDF5IO(path=self.test_folder / "example_converter_spec_3.nwb", mode="r") as io: + + nwbfile_path = self.test_folder / "example_converter_spec_3.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Auto-generated by neuroconv" assert nwbfile.lab == "My Lab" @@ -60,7 +68,9 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): ) ) - with NWBHDF5IO(path=self.test_folder / "sub-Mouse_1_ses-20201009T211909.nwb", mode="r") as io: + nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201009T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" @@ -68,14 +78,31 @@ def test_run_conversion_from_yaml_default_nwbfile_name(self): assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-09T21:19:09+00:00") assert nwbfile.subject.subject_id == "Mouse 1" assert "ElectricalSeriesAP" in nwbfile.acquisition - with NWBHDF5IO(path=self.test_folder / "example_defined_name.nwb", mode="r") as io: + + nwbfile_path = self.test_folder / "sub-Mouse-1_ses-20201109T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: + nwbfile = io.read() + assert nwbfile.session_description == "Subject navigating a Y-shaped maze." + assert nwbfile.lab == "My Lab" + assert nwbfile.institution == "My Institution" + assert nwbfile.session_start_time == datetime.fromisoformat("2020-11-09T21:19:09+00:00") + assert nwbfile.subject.subject_id == "Mouse 1" + assert "ElectricalSeriesAP" in nwbfile.acquisition + + nwbfile_path = self.test_folder / "example_defined_name.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Subject navigating a Y-shaped maze." assert nwbfile.lab == "My Lab" assert nwbfile.institution == "My Institution" assert nwbfile.session_start_time == datetime.fromisoformat("2020-10-10T21:19:09+00:00") assert nwbfile.subject.subject_id == "MyMouse002" - with NWBHDF5IO(path=self.test_folder / "sub-Subject_Name_ses-20201011T211909.nwb", mode="r") as io: + + nwbfile_path = self.test_folder / "sub-Subject-Name_ses-20201011T211909.nwb" + assert nwbfile_path.exists(), f"`run_conversion_from_yaml` failed to create the file at '{nwbfile_path}'! " + with NWBHDF5IO(path=nwbfile_path, mode="r") as io: nwbfile = io.read() assert nwbfile.session_description == "Auto-generated by neuroconv" assert nwbfile.lab == "My Lab"