From 3b798bcdd9e3274a6e3915702d43a4bff40e7510 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 8 Jul 2024 08:54:15 -0600 Subject: [PATCH 01/31] Add option to supress tqdm progress bar in `VideoContext` (#937) --- CHANGELOG.md | 1 + .../datainterfaces/behavior/video/video_utils.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fcfff01e..25a4f1f60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Improvements * Make annotations from the raw format available on `IntanRecordingInterface`. [PR #934](https://github.com/catalystneuro/neuroconv/pull/943) +* Add an option to suppress display the progress bar (tqdm) in `VideoContext` [PR #937](https://github.com/catalystneuro/neuroconv/pull/937) ## v0.4.11 (June 14, 2024) diff --git a/src/neuroconv/datainterfaces/behavior/video/video_utils.py b/src/neuroconv/datainterfaces/behavior/video/video_utils.py index df70ee77b..5000c468b 100644 --- a/src/neuroconv/datainterfaces/behavior/video/video_utils.py +++ b/src/neuroconv/datainterfaces/behavior/video/video_utils.py @@ -9,7 +9,9 @@ from ....utils import FilePathType -def get_video_timestamps(file_path: FilePathType, max_frames: Optional[int] = None) -> list: +def get_video_timestamps( + file_path: FilePathType, max_frames: Optional[int] = None, display_progress: bool = True +) -> list: """Extract the timestamps of the video located in file_path Parameters @@ -26,7 +28,7 @@ def get_video_timestamps(file_path: FilePathType, max_frames: Optional[int] = No """ with VideoCaptureContext(str(file_path)) as video_context: - timestamps = video_context.get_video_timestamps(max_frames=max_frames) + timestamps = video_context.get_video_timestamps(max_frames=max_frames, display_progress=display_progress) return timestamps @@ -43,14 +45,20 @@ def __init__(self, file_path: FilePathType): self._frame_count = None self._video_open_msg = "The video file is not open!" - def get_video_timestamps(self, max_frames=None): + def get_video_timestamps(self, max_frames: Optional[int] = None, display_progress: bool = True): """Return numpy array of the timestamps(s) for a video file.""" cv2 = get_package(package_name="cv2", installation_instructions="pip install opencv-python-headless") timestamps = [] total_frames = self.get_video_frame_count() frames_to_extract = min(total_frames, max_frames) if max_frames else total_frames - for _ in tqdm(range(frames_to_extract), desc="retrieving timestamps"): + + iterator = ( + tqdm(range(frames_to_extract), desc="retrieving timestamps") + if display_progress + else range(frames_to_extract) + ) + for _ in iterator: success, _ = self.vc.read() if not success: break From 54f12f79c123bd91b4b88f1b66d87b412cb158bf Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 8 Jul 2024 09:27:45 -0600 Subject: [PATCH 02/31] Remove wrong assumptions about electrode metadata in Intan (#933) Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- CHANGELOG.md | 1 + .../ecephys/intan/intandatainterface.py | 71 +------------------ 2 files changed, 2 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a4f1f60..fe0f90ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Fixed the conversion option schema of a `SpikeGLXConverter` when used inside another `NWBConverter`. [PR #922](https://github.com/catalystneuro/neuroconv/pull/922) * Fixed a case of the `NeuroScopeSortingExtractor` when the optional `xml_file_path` is not specified. [PR #926](https://github.com/catalystneuro/neuroconv/pull/926) * Fixed `Can't specify experiment type when converting .abf to .nwb with Neuroconv`. [PR #609](https://github.com/catalystneuro/neuroconv/pull/609) +* Remove assumption that the ports of the Intan acquisition system correspond to electrode groupings in `IntanRecordingInterface` [PR #933](https://github.com/catalystneuro/neuroconv/pull/933) ### Improvements * Make annotations from the raw format available on `IntanRecordingInterface`. [PR #934](https://github.com/catalystneuro/neuroconv/pull/943) diff --git a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py index fae795cc4..2952a3dee 100644 --- a/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py +++ b/src/neuroconv/datainterfaces/ecephys/intan/intandatainterface.py @@ -9,32 +9,6 @@ from ....utils import FilePathType, get_schema_from_hdmf_class -def extract_electrode_metadata(recording_extractor) -> dict: - - neo_version = get_package_version(name="neo") - - # The native native_channel_name in Intan have the following form: A-000, A-001, A-002, B-000, B-001, B-002, etc. - if neo_version > Version("0.13.0"): # TODO: Remove after the release of neo 0.14.0 - native_channel_names = recording_extractor.get_channel_ids() - else: - # Previous to version 0.13.1 the native_channel_name was stored as channel_name - native_channel_names = recording_extractor.get_property("channel_name") - - group_names = [channel.split("-")[0] for channel in native_channel_names] - unique_group_names = set(group_names) - group_electrode_numbers = [int(channel.split("-")[1]) for channel in native_channel_names] - custom_names = list() - - electrodes_metadata = dict( - group_names=group_names, - unique_group_names=unique_group_names, - group_electrode_numbers=group_electrode_numbers, - custom_names=custom_names, - ) - - return electrodes_metadata - - class IntanRecordingInterface(BaseRecordingExtractorInterface): """ Primary data interface class for converting Intan data using the @@ -85,7 +59,7 @@ def __init__( ) self.stream_id = stream_id else: - self.stream_id = "0" + self.stream_id = "0" # These are the amplifier channels or to the stream_name 'RHD2000 amplifier channel' init_kwargs = dict( file_path=file_path, @@ -108,22 +82,6 @@ def __init__( init_kwargs["ignore_integrity_checks"] = ignore_integrity_checks super().__init__(**init_kwargs) - electrodes_metadata = extract_electrode_metadata(recording_extractor=self.recording_extractor) - - group_names = electrodes_metadata["group_names"] - group_electrode_numbers = electrodes_metadata["group_electrode_numbers"] - unique_group_names = electrodes_metadata["unique_group_names"] - custom_names = electrodes_metadata["custom_names"] - - channel_ids = self.recording_extractor.get_channel_ids() - self.recording_extractor.set_property(key="group_name", ids=channel_ids, values=group_names) - if len(unique_group_names) > 1: - self.recording_extractor.set_property( - key="group_electrode_number", ids=channel_ids, values=group_electrode_numbers - ) - - if any(custom_names): - self.recording_extractor.set_property(key="custom_channel_name", ids=channel_ids, values=custom_names) def get_metadata_schema(self) -> dict: metadata_schema = super().get_metadata_schema() @@ -145,36 +103,9 @@ def get_metadata(self) -> dict: device_list = [device] ecephys_metadata.update(Device=device_list) - # Add electrode group - unique_group_name = set(self.recording_extractor.get_property("group_name")) - electrode_group_list = [ - dict( - name=group_name, - description=f"Group {group_name} electrodes.", - device="Intan", - location="", - ) - for group_name in unique_group_name - ] - ecephys_metadata.update(ElectrodeGroup=electrode_group_list) - # Add electrodes and electrode groups ecephys_metadata.update( - Electrodes=[ - dict(name="group_name", description="The name of the ElectrodeGroup this electrode is a part of.") - ], ElectricalSeriesRaw=dict(name="ElectricalSeriesRaw", description="Raw acquisition traces."), ) - # Add group electrode number if available - recording_extractor_properties = self.recording_extractor.get_property_keys() - if "group_electrode_number" in recording_extractor_properties: - ecephys_metadata["Electrodes"].append( - dict(name="group_electrode_number", description="0-indexed channel within a group.") - ) - if "custom_channel_name" in recording_extractor_properties: - ecephys_metadata["Electrodes"].append( - dict(name="custom_channel_name", description="Custom channel name assigned in Intan.") - ) - return metadata From dd8ef4fbd031f4abec852fdd19e6c6abb625e8c5 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:03:31 -0400 Subject: [PATCH 03/31] [Cloud Deployment IIa] Rclone docker image extension for config file (#902) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: CodyCBakerPhD --- ...upload_docker_image_rclone_with_config.yml | 36 ++++++++ ...sting.yml => neuroconv_docker_testing.yml} | 2 +- .github/workflows/rclone_docker_testing.yml | 39 ++++++++ CHANGELOG.md | 3 + dockerfiles/rclone_with_config | 5 ++ docs/developer_guide/docker_images.rst | 6 +- docs/user_guide/docker_demo.rst | 33 ++++++- tests/docker_rclone_with_config_cli.py | 89 +++++++++++++++++++ ...ocker_yaml_conversion_specification_cli.py | 1 - 9 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build_and_upload_docker_image_rclone_with_config.yml rename .github/workflows/{docker_testing.yml => neuroconv_docker_testing.yml} (99%) create mode 100644 .github/workflows/rclone_docker_testing.yml create mode 100644 dockerfiles/rclone_with_config create mode 100644 tests/docker_rclone_with_config_cli.py diff --git a/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml b/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml new file mode 100644 index 000000000..7ff197bdc --- /dev/null +++ b/.github/workflows/build_and_upload_docker_image_rclone_with_config.yml @@ -0,0 +1,36 @@ +name: Build and Upload Docker Image of Rclone With Config to GHCR + +on: + schedule: + - cron: "0 16 * * 1" # Weekly at noon EST on Monday + workflow_dispatch: + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + release-image: + name: Build and Upload Docker Image of Rclone With Config to GHCR + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ secrets.DOCKER_UPLOADER_USERNAME }} + password: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + push: true # Push is a shorthand for --output=type=registry + tags: ghcr.io/catalystneuro/rclone_with_config:latest + context: . + file: dockerfiles/rclone_with_config + provenance: false diff --git a/.github/workflows/docker_testing.yml b/.github/workflows/neuroconv_docker_testing.yml similarity index 99% rename from .github/workflows/docker_testing.yml rename to .github/workflows/neuroconv_docker_testing.yml index 6916e0e4e..282da7937 100644 --- a/.github/workflows/docker_testing.yml +++ b/.github/workflows/neuroconv_docker_testing.yml @@ -1,4 +1,4 @@ -name: Docker CLI tests +name: NeuroConv Docker CLI tests on: schedule: - cron: "0 16 * * *" # Daily at noon EST diff --git a/.github/workflows/rclone_docker_testing.yml b/.github/workflows/rclone_docker_testing.yml new file mode 100644 index 000000000..2e8ea9e17 --- /dev/null +++ b/.github/workflows/rclone_docker_testing.yml @@ -0,0 +1,39 @@ +name: Rclone Docker Tests +on: + schedule: + - cron: "0 16 * * *" # Daily at noon EST + workflow_dispatch: + +jobs: + run: + name: ${{ matrix.os }} Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - run: git fetch --prune --unshallow --tags + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Global Setup + run: python -m pip install -U pip # Official recommended way + + - name: Install pytest and neuroconv minimal + run: | + pip install pytest + pip install . + + - name: Pull docker image + run: docker pull ghcr.io/catalystneuro/rclone_with_config:latest + - name: Run docker tests + run: pytest tests/docker_rclone_with_config_cli.py -vv -rsx + env: + RCLONE_DRIVE_ACCESS_TOKEN: ${{ secrets.RCLONE_DRIVE_ACCESS_TOKEN }} + RCLONE_DRIVE_REFRESH_TOKEN: ${{ secrets.RCLONE_DRIVE_REFRESH_TOKEN }} + RCLONE_EXPIRY_TOKEN: ${{ secrets.RCLONE_EXPIRY_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index fe0f90ed6..3e988ac17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Upcoming +### Features +* Added docker image and tests for an automated Rclone configuration (with file stream passed via an environment variable). [PR #902](https://github.com/catalystneuro/neuroconv/pull/902) + ### Bug fixes * Fixed the conversion option schema of a `SpikeGLXConverter` when used inside another `NWBConverter`. [PR #922](https://github.com/catalystneuro/neuroconv/pull/922) * Fixed a case of the `NeuroScopeSortingExtractor` when the optional `xml_file_path` is not specified. [PR #926](https://github.com/catalystneuro/neuroconv/pull/926) diff --git a/dockerfiles/rclone_with_config b/dockerfiles/rclone_with_config new file mode 100644 index 000000000..985ce8019 --- /dev/null +++ b/dockerfiles/rclone_with_config @@ -0,0 +1,5 @@ +FROM rclone/rclone:latest +LABEL org.opencontainers.image.source=https://github.com/catalystneuro/neuroconv +LABEL org.opencontainers.image.description="A simple extension of the basic Rclone docker image to automatically create a local .conf file from contents passed via an environment variable." +CMD printf "$RCLONE_CONFIG" > ./rclone.conf && eval "$RCLONE_COMMAND" +ENTRYPOINT [""] diff --git a/docs/developer_guide/docker_images.rst b/docs/developer_guide/docker_images.rst index d78b3cfd0..0310734a3 100644 --- a/docs/developer_guide/docker_images.rst +++ b/docs/developer_guide/docker_images.rst @@ -54,6 +54,8 @@ After building the image itself, we can publish the container with... Though it may appear confusing, the use of the ``IMAGE_NAME`` in these steps determines only the _name_ of the package as available from the 'packages' screen of the host repository; the ``LABEL`` itself ensured the upload and linkage to the NeuroConv GHCR. +All our docker images can be built in GitHub Actions (for Ubuntu) and pushed automatically to the GHCR by manually triggering their respective workflow. Keep in mind that most of them are on semi-regular CRON schedules, though. + Run Docker container on local YAML conversion specification file @@ -73,12 +75,14 @@ and can then run the entrypoint (equivalent to the usual command line usage) on +.. _developer_docker_details: + Run Docker container on YAML conversion specification environment variable -------------------------------------------------------------------------- An alternative approach that simplifies usage on systems such as AWS Batch is to specify the YAML contents as an environment variable. The YAML file is constructed in the first step of the container launch. -The only potential downside with this usage is the maximum size of an environment variable (~13,000 characters). Typical YAML specification files should not come remotely close to this limit. +The only potential downside with this usage is the maximum size of an environment variable (~13,000 characters). Typical YAML specification files should not come remotely close to this limit. This is contrasted to the limits on the CMD line of any docker container, which is either 8192 characters for Windows or either 64 or 128 KiB depending on UNIX build. Otherwise, in any cloud deployment, the YAML file transfer will have to be managed separately, likely as a part of the data transfer or an entirely separate step. diff --git a/docs/user_guide/docker_demo.rst b/docs/user_guide/docker_demo.rst index 92a1d5b2f..e089b5748 100644 --- a/docs/user_guide/docker_demo.rst +++ b/docs/user_guide/docker_demo.rst @@ -1,5 +1,5 @@ -Docker Demo ------------ +NeuroConv Docker Demo +--------------------- The following is an explicit demonstration of how to use the Docker-based NeuroConv YAML specification via the command line. @@ -116,3 +116,32 @@ Voilà! If everything occurred successfully, you should see... Metadata is valid! conversion_options is valid! NWB file saved at /demo_neuroconv_docker/demo_output/phy_from_docker_yaml.nwb! + + + + +RClone With Config Docker Demo +------------------------------ + +NeuroConv also supports a convenient Docker image for running data transfers via `Rclone `_. + +To use this image, you must first configure the remote locally by calling: + +.. code:: + + rclone config + +And following all interactive instructions (defaults are usually sufficient). + +The Docker image requires two environment variables to be set (see :ref:`developer_docker_details` for more details in a related process). + +- ``RCLONE_CONFIG``: The full file content of the rclone.conf file on your system. You can find this by calling ``rclone config file``. On UNIX, for example, you can set this variable using ``RCLONE_CONFIG=$(