From 58ccd217f734c87327cc092e5a0c8f28c592b6ff Mon Sep 17 00:00:00 2001 From: Georg Raiser Date: Wed, 11 Dec 2024 14:18:42 +0000 Subject: [PATCH] cleanups --- ...inwide_map_processed_only_local_testing.py | 1 - ...rt_brainwide_map_raw_only_local_testing.py | 33 +++++--------- src/ibl_to_nwb/_scripts/download_data.py | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 src/ibl_to_nwb/_scripts/download_data.py diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py index bddd4bd..836c8a8 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_processed_only_local_testing.py @@ -24,7 +24,6 @@ from ibl_to_nwb.testing._consistency_checks import check_written_nwbfile_for_consistency base_path = Path.home() / "ibl_scratch" # local directory -# session = "d32876dd-8303-4720-8e7e-20678dc2fd71" session = "caa5dddc-9290-4e27-9f5e-575ba3598614" # a BWM session with dual probe nwbfile_path = base_path / "nwbfiles" / session / f"{session}.nwb" diff --git a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py index 3c162c8..37b4751 100644 --- a/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py +++ b/src/ibl_to_nwb/_scripts/convert_brainwide_map_raw_only_local_testing.py @@ -4,29 +4,18 @@ from ibl_to_nwb.converters import BrainwideMapConverter, IblSpikeGlxConverter from ibl_to_nwb.datainterfaces import RawVideoInterface -# eid = "d32876dd-8303-4720-8e7e-20678dc2fd71" -eid = "caa5dddc-9290-4e27-9f5e-575ba3598614" # a BWM session with dual probe +# select eid +# -> run download_data_local first with this eid to set up the local folder structure and one cache +eid = "caa5dddc-9290-4e27-9f5e-575ba3598614" -# %% -# one_cache_folder = '/home/georg/ibl_scratch/ibl_conversion/caa5dddc-9290-4e27-9f5e-575ba3598614/cache' -# data_folder = Path( -# "/media/georg/openlab/Downloads/ONE/openalyx.internationalbrainlab.org/steinmetzlab/Subjects/NR_0031/2023-07-14/001" -# ) -# spikeglx_source_folder_path = data_folder / "raw_ephys_data" - -# Specify the revision of the pose estimation data -# Setting to 'None' will use whatever the latest released revision is -# revision = None - -# base_path = Path("E:/IBL") -base_path = Path.home() / "ibl_scratch" # local directory +# folders +base_path = Path.home() / "ibl_scratch" base_path.mkdir(exist_ok=True) nwbfiles_folder_path = base_path / "nwbfiles" nwbfiles_folder_path.mkdir(exist_ok=True) # Initialize IBL (ONE) client to download processed data for this session -# one_cache_folder_path = base_path / "cache" -one_cache_folder_path = "/home/georg/ibl_scratch/ibl_conversion/caa5dddc-9290-4e27-9f5e-575ba3598614/cache" +one_cache_folder_path = base_path / 'ibl_conversion' / eid / 'cache' one = ONE( base_url="https://openalyx.internationalbrainlab.org", password="international", @@ -37,14 +26,12 @@ data_interfaces = [] # %% ephys -# session_folder = one.eid2path(eid) -# spikeglx_source_folder_path = session_folder / 'raw_ephys_data' - +session_folder = one.eid2path(eid) +spikeglx_source_folder_path = session_folder / 'raw_ephys_data' # Specify the path to the SpikeGLX files on the server but use ONE API for timestamps -# spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=one, eid=eid) -# data_interfaces.append(spikeglx_subconverter) - +spikeglx_subconverter = IblSpikeGlxConverter(folder_path=spikeglx_source_folder_path, one=one, eid=eid) +data_interfaces.append(spikeglx_subconverter) # %% video # Raw video takes some special handling diff --git a/src/ibl_to_nwb/_scripts/download_data.py b/src/ibl_to_nwb/_scripts/download_data.py new file mode 100644 index 0000000..b0e07ca --- /dev/null +++ b/src/ibl_to_nwb/_scripts/download_data.py @@ -0,0 +1,44 @@ +# %% +from pathlib import Path +from one.api import ONE + +# %% +eid = "caa5dddc-9290-4e27-9f5e-575ba3598614" # a BWM eid with dual probe + +base_path = Path.home() / "ibl_scratch" # local directory + +# Download behavior and spike sorted data for this eid +session_path = base_path / "ibl_conversion" / eid +cache_folder = base_path / "ibl_conversion" / eid / "cache" +session_one = ONE( + base_url="https://openalyx.internationalbrainlab.org", + password="international", + silent=False, + cache_dir=cache_folder, +) + +# %% latest revision +revisions = session_one.list_revisions(eid) +revision = revisions[-1] + +# %% list all datasets +datasets = session_one.list_datasets(eid) + +# %% list all collections +collections = session_one.list_collections(eid) + +# %% +for dataset in datasets: + session_one.load_dataset(eid, dataset, download_only=True) + +# %% downloads all raw ephys data! +collections = session_one.list_collections(eid, collection="raw_ephys_data/*") +for collection in collections: + datasets = session_one.list_datasets(eid, collection=collection) + for dataset in datasets: + session_one.load_dataset(eid, dataset, download_only=True) + +# %% just the video data +datasets = session_one.list_datasets(eid, collection="raw_video_data") +for dataset in datasets: + session_one.load_dataset(eid, dataset, download_only=True)