diff --git a/.github/workflows/test_containers_docker.yml b/.github/workflows/test_containers_docker.yml index f623212137..7a3d894195 100644 --- a/.github/workflows/test_containers_docker.yml +++ b/.github/workflows/test_containers_docker.yml @@ -5,8 +5,6 @@ on: workflow_dispatch jobs: test-images: - env: - SPIKEINTERFACE_DEV_PATH: ${{ github.workspace }} name: Test on (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: @@ -30,10 +28,6 @@ jobs: pip install pytest pip install -e .[full] pip install docker - - name: Test that containers install the local CI version of spikeinterface - run: | - echo $SPIKEINTERFACE_DEV_PATH - python -c "import os; assert os.getenv('SPIKEINTERFACE_DEV_PATH') is not None" - name: Run test docker containers run: | pytest -vv --capture=tee-sys -rA src/spikeinterface/sorters/external/tests/test_docker_containers.py diff --git a/.github/workflows/test_containers_singularity.yml b/.github/workflows/test_containers_singularity.yml index be97cd8642..26526c3211 100644 --- a/.github/workflows/test_containers_singularity.yml +++ b/.github/workflows/test_containers_singularity.yml @@ -5,8 +5,6 @@ on: workflow_dispatch jobs: test-images: - env: - SPIKEINTERFACE_DEV_PATH: ${{ github.workspace }} name: Test on (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: @@ -34,11 +32,6 @@ jobs: pip install pytest pip install -e .[full] pip install spython - - name: Test that containers install the local CI version of spikeinterface - run: | - echo $SPIKEINTERFACE_DEV_PATH - python -c "import os; assert os.getenv('SPIKEINTERFACE_DEV_PATH') is not None" - ls -l - name: Run test singularity containers run: | pytest -vv --capture=tee-sys -rA src/spikeinterface/sorters/external/tests/test_singularity_containers.py diff --git a/.github/workflows/test_containers_singularity_gpu.yml b/.github/workflows/test_containers_singularity_gpu.yml index d075f5a6ef..e74fbeb4a5 100644 --- a/.github/workflows/test_containers_singularity_gpu.yml +++ b/.github/workflows/test_containers_singularity_gpu.yml @@ -46,6 +46,5 @@ jobs: - name: Run test singularity containers with GPU env: REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - SPIKEINTERFACE_DEV_PATH: ${{ github.workspace }} run: | pytest -vv --capture=tee-sys -rA src/spikeinterface/sorters/external/tests/test_singularity_containers_gpu.py diff --git a/src/spikeinterface/__init__.py b/src/spikeinterface/__init__.py index 99325d7d65..2e17a71000 100644 --- a/src/spikeinterface/__init__.py +++ b/src/spikeinterface/__init__.py @@ -25,3 +25,9 @@ import spikeinterface.full as si """ + +# This flag must be set to False for release +# This avoids using versioning that contains ".dev0" (and this is a better choice) +# This is mainly useful when using run_sorter in a container and spikeinterface install +DEV_MODE = True +# DEV_MODE = False diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index ccf589b6e7..2585a2bd73 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -443,3 +443,13 @@ def convert_bytes_to_str(byte_value: int) -> str: byte_value /= 1024 i += 1 return f"{byte_value:.2f} {suffixes[i]}" + + +def is_editable_mode() -> bool: + """ + Check if spikeinterface is installed in editable mode + pip install -e . + """ + import spikeinterface + + return (Path(spikeinterface.__file__).parents[2] / "README.md").exists() diff --git a/src/spikeinterface/sorters/__init__.py b/src/spikeinterface/sorters/__init__.py index bb04af2443..b219a091fe 100644 --- a/src/spikeinterface/sorters/__init__.py +++ b/src/spikeinterface/sorters/__init__.py @@ -1,4 +1,5 @@ from .basesorter import BaseSorter from .sorterlist import * -from .runsorter import * +from .container_tools import ContainerClient, install_package_in_container +from .runsorter import run_sorter, run_sorter_local, run_sorter_container, read_sorter_folder from .launcher import run_sorter_jobs, run_sorter_by_property diff --git a/src/spikeinterface/sorters/container_tools.py b/src/spikeinterface/sorters/container_tools.py new file mode 100644 index 0000000000..cae79ee596 --- /dev/null +++ b/src/spikeinterface/sorters/container_tools.py @@ -0,0 +1,275 @@ +from pathlib import Path +import platform +import os +import random +import string + +# TODO move this inside functions +try: + HAS_DOCKER = True + import docker +except ModuleNotFoundError: + HAS_DOCKER = False + + +from spikeinterface.core.core_tools import recursive_path_modifier + + +def find_recording_folders(d): + """Finds all recording folders 'paths' in a dict""" + folders_to_mount = [] + + def append_parent_folder(p): + p = Path(p) + folders_to_mount.append(p.resolve().absolute().parent) + return p + + _ = recursive_path_modifier(d, append_parent_folder, target="path", copy=True) + + try: # this will fail if on different drives (Windows) + base_folders_to_mount = [Path(os.path.commonpath(folders_to_mount))] + except ValueError: + base_folders_to_mount = folders_to_mount + + # let's not mount root if dries are /home/..., /mnt1/... + if len(base_folders_to_mount) == 1: + if len(str(base_folders_to_mount[0])) == 1: + base_folders_to_mount = folders_to_mount + + return base_folders_to_mount + + +def path_to_unix(path): + """Convert a Windows path to unix format""" + path = Path(path) + if platform.system() == "Windows": + path = Path(str(path)[str(path).find(":") + 1 :]) + return path.as_posix() + + +def windows_extractor_dict_to_unix(d): + d = recursive_path_modifier(d, path_to_unix, target="path", copy=True) + return d + + +class ContainerClient: + """ + Small abstraction class to run commands in: + * docker with "docker" python package + * singularity with "spython" python package + """ + + def __init__(self, mode, container_image, volumes, py_user_base, extra_kwargs): + """ + Parameters + ---------- + mode: "docker" | "singularity" + The container mode + container_image: str + container image name and tag + volumes: dict + dict of volumes to bind + py_user_base: str + Python user base folder to set as PYTHONUSERBASE env var in Singularity mode + Prevents from overwriting user's packages when running pip install + extra_kwargs: dict + Extra kwargs to start container + """ + assert mode in ("docker", "singularity") + self.mode = mode + self.py_user_base = py_user_base + container_requires_gpu = extra_kwargs.get("container_requires_gpu", None) + + if mode == "docker": + if not HAS_DOCKER: + raise ModuleNotFoundError("No module named 'docker'") + client = docker.from_env() + if container_requires_gpu is not None: + extra_kwargs.pop("container_requires_gpu") + extra_kwargs["device_requests"] = [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] + + if self._get_docker_image(container_image) is None: + print(f"Docker: pulling image {container_image}") + client.images.pull(container_image) + + self.docker_container = client.containers.create(container_image, tty=True, volumes=volumes, **extra_kwargs) + + elif mode == "singularity": + assert self.py_user_base, "py_user_base folder must be set in singularity mode" + from spython.main import Client + + # load local image file if it exists, otherwise search dockerhub + sif_file = Client._get_filename(container_image) + singularity_image = None + if Path(container_image).exists(): + singularity_image = container_image + elif Path(sif_file).exists(): + singularity_image = sif_file + else: + if HAS_DOCKER: + docker_image = self._get_docker_image(container_image) + if docker_image and len(docker_image.tags) > 0: + tag = docker_image.tags[0] + print(f"Building singularity image from local docker image: {tag}") + singularity_image = Client.build(f"docker-daemon://{tag}", sif_file, sudo=False) + if not singularity_image: + print(f"Singularity: pulling image {container_image}") + singularity_image = Client.pull(f"docker://{container_image}") + + if not Path(singularity_image).exists(): + raise FileNotFoundError(f"Unable to locate container image {container_image}") + + # bin options + singularity_bind = ",".join([f'{volume_src}:{volume["bind"]}' for volume_src, volume in volumes.items()]) + options = ["--bind", singularity_bind] + + # gpu options + if container_requires_gpu: + # only nvidia at the moment + options += ["--nv"] + + self.client_instance = Client.instance(singularity_image, start=False, options=options) + + @staticmethod + def _get_docker_image(container_image): + docker_client = docker.from_env(timeout=300) + try: + docker_image = docker_client.images.get(container_image) + except docker.errors.ImageNotFound: + docker_image = None + return docker_image + + def start(self): + if self.mode == "docker": + self.docker_container.start() + elif self.mode == "singularity": + self.client_instance.start() + + def stop(self): + if self.mode == "docker": + self.docker_container.stop() + self.docker_container.remove(force=True) + elif self.mode == "singularity": + self.client_instance.stop() + + def run_command(self, command): + if self.mode == "docker": + res = self.docker_container.exec_run(command) + return res.output.decode(encoding="utf-8", errors="ignore") + elif self.mode == "singularity": + from spython.main import Client + + options = ["--cleanenv", "--env", f"PYTHONUSERBASE={self.py_user_base}"] + res = Client.execute(self.client_instance, command, options=options) + if isinstance(res, dict): + res = res["message"] + return res + + +def install_package_in_container( + container_client, + package_name, + installation_mode="pypi", + extra=None, + version=None, + tag=None, + github_url=None, + container_folder_source=None, + verbose=False, +): + """ + Install a package in a container with different modes: + + * pypi: pip install package_name + * github: pip install {github_url}/archive/{tag/version}.tar.gz#egg=package_name + * folder: pip install folder + + Parameters + ---------- + container_client: ContainerClient + The container client + package_name: str + The package name + installation_mode: str + The installation mode + extra: str + Extra pip install arguments, e.g. [full] + version: str + The package version to install + tag: str + The github tag to install + github_url: str + The github url to install (needed for github mode) + container_folder_source: str + The container folder source (needed for folder mode) + verbose: bool + If True, print output of pip install command + + Returns + ------- + res_output: str + The output of the pip install command + """ + assert installation_mode in ("pypi", "github", "folder") + + if "[" in package_name: + raise ValueError("Extra pip install should not be in package_name but like this extra='[full]'") + + if extra is not None: + assert extra[0] == "[" and extra[-1] == "]", "extra should be like this: '[full]'" + + if verbose: + print(f"Installing {package_name} with {installation_mode} in container") + + if installation_mode == "pypi": + cmd = f"pip install --user --upgrade --no-input --no-build-isolation {package_name}" + + if extra is not None: + cmd += f"{extra}" + + if version is not None: + cmd += f"=={version}" + res_output = container_client.run_command(cmd) + + elif installation_mode == "github": + if version is None and tag is None: + tag_or_version = "main" + elif tag is not None: + tag_or_version = tag + elif version is not None: + tag_or_version = version + + if github_url is None: + github_url = "https://github.com/SpikeInterface/spikeinterface" + + cmd = f"pip install --user --upgrade --no-input {github_url}/archive/{tag_or_version}.tar.gz#egg={package_name}" + if extra is not None: + cmd += f"{extra}" + res_output = container_client.run_command(cmd) + + elif installation_mode == "folder": + assert tag is None + + if container_client.mode == "singularity": + folder_copy = f"{container_client.py_user_base}/sources/" + else: + folder_copy = "/sources/" + + # create a folder for source copy + rand_str = "".join(random.choices(string.ascii_uppercase + string.digits, k=8)) + folder_copy += rand_str + cmd = f"mkdir -p {folder_copy}" + res_output = container_client.run_command(cmd) + + cmd = f"cp -r {container_folder_source} {folder_copy}/{package_name}" + res_output = container_client.run_command(cmd) + + cmd = f"pip install --user --no-input {folder_copy}/{package_name}" + if extra is not None: + cmd += f"{extra}" + res_output = container_client.run_command(cmd) + + else: + raise ValueError(f"install_package_incontainer, wrong installation_mode={installation_mode}") + + return res_output diff --git a/src/spikeinterface/sorters/external/tests/test_docker_containers.py b/src/spikeinterface/sorters/external/tests/test_docker_containers.py index 3add7f7b7b..4af25bc9f0 100644 --- a/src/spikeinterface/sorters/external/tests/test_docker_containers.py +++ b/src/spikeinterface/sorters/external/tests/test_docker_containers.py @@ -4,6 +4,7 @@ import pytest from pathlib import Path +from spikeinterface.core.core_tools import is_editable_mode import spikeinterface.extractors as se import spikeinterface.sorters as ss @@ -18,8 +19,7 @@ def check_gh_settings(): if ON_GITHUB: - si_dev_path = os.getenv("SPIKEINTERFACE_DEV_PATH") - assert si_dev_path is not None, "Tests on GITHUB CI must run with the SPIKEINTERFACE_DEV_PATH" + assert is_editable_mode(), "Tests on GITHUB CI must run with SpikeInterface in editable mode" def generate_run_kwargs(): @@ -29,6 +29,7 @@ def generate_run_kwargs(): test_recording.set_channel_offsets(0) run_kwargs = dict(recording=test_recording, verbose=True) run_kwargs["docker_image"] = True + run_kwargs["installation_mode"] = "dev" return run_kwargs diff --git a/src/spikeinterface/sorters/external/tests/test_singularity_containers.py b/src/spikeinterface/sorters/external/tests/test_singularity_containers.py index ca45bf6a9a..6c4d096104 100644 --- a/src/spikeinterface/sorters/external/tests/test_singularity_containers.py +++ b/src/spikeinterface/sorters/external/tests/test_singularity_containers.py @@ -3,6 +3,7 @@ import pytest from pathlib import Path +from spikeinterface.core.core_tools import is_editable_mode import spikeinterface.extractors as se import spikeinterface.sorters as ss @@ -24,8 +25,7 @@ def clean_singularity_cache(): def check_gh_settings(): if ON_GITHUB: - si_dev_path = os.getenv("SPIKEINTERFACE_DEV_PATH") - assert si_dev_path is not None, "Tests on GITHUB CI must run with the SPIKEINTERFACE_DEV_PATH" + assert is_editable_mode(), "Tests on GITHUB CI must run with SpikeInterface in editable mode" def generate_run_kwargs(): @@ -35,6 +35,7 @@ def generate_run_kwargs(): test_recording.set_channel_offsets(0) run_kwargs = dict(recording=test_recording, verbose=True) run_kwargs["singularity_image"] = True + run_kwargs["installation_mode"] = "dev" return run_kwargs diff --git a/src/spikeinterface/sorters/external/tests/test_singularity_containers_gpu.py b/src/spikeinterface/sorters/external/tests/test_singularity_containers_gpu.py index c412d5f56e..16159a03da 100644 --- a/src/spikeinterface/sorters/external/tests/test_singularity_containers_gpu.py +++ b/src/spikeinterface/sorters/external/tests/test_singularity_containers_gpu.py @@ -3,6 +3,7 @@ import pytest +from spikeinterface.core.core_tools import is_editable_mode import spikeinterface.extractors as se import spikeinterface.sorters as ss @@ -18,8 +19,7 @@ def clean_singularity_cache(): def check_gh_settings(): if ON_GITHUB: - si_dev_path = os.getenv("SPIKEINTERFACE_DEV_PATH") - assert si_dev_path is not None, "Tests on GITHUB CI must run with the SPIKEINTERFACE_DEV_PATH" + assert is_editable_mode(), "Tests on GITHUB CI must run with SpikeInterface in editable mode" def generate_run_kwargs(): @@ -29,6 +29,7 @@ def generate_run_kwargs(): test_recording.set_channel_offsets(0) run_kwargs = dict(recording=test_recording, verbose=True) run_kwargs["singularity_image"] = True + run_kwargs["installation_mode"] = "dev" return run_kwargs diff --git a/src/spikeinterface/sorters/runsorter.py b/src/spikeinterface/sorters/runsorter.py index bdc1393c66..488eee03ba 100644 --- a/src/spikeinterface/sorters/runsorter.py +++ b/src/spikeinterface/sorters/runsorter.py @@ -7,18 +7,25 @@ from warnings import warn from typing import Optional, Union -from ..core import BaseRecording, NumpySorting +from spikeinterface import DEV_MODE +import spikeinterface + + from .. import __version__ as si_version -from spikeinterface.core.npzsortingextractor import NpzSortingExtractor -from spikeinterface.core.core_tools import check_json, recursive_path_modifier + + +from ..core import BaseRecording, NumpySorting, load_extractor +from ..core.core_tools import check_json, is_editable_mode from .sorterlist import sorter_dict from .utils import SpikeSortingError, has_nvidia +from .container_tools import ( + find_recording_folders, + path_to_unix, + windows_extractor_dict_to_unix, + ContainerClient, + install_package_in_container, +) -try: - HAS_DOCKER = True - import docker -except ModuleNotFoundError: - HAS_DOCKER = False REGISTRY = "spikeinterface" @@ -73,17 +80,34 @@ If True, pull the default docker container for the sorter and run the sorter in that container using singularity. Use a str to specify a non-default container. If that container is not local it will be pulled from Docker Hub. If False, the sorter is run locally - delete_container_files: bool, default: True - If True, the container temporary files are deleted after the sorting is done with_output: bool, default: True If True, the output Sorting is returned as a Sorting + delete_container_files: bool, default: True + If True, the container temporary files are deleted after the sorting is done + extra_requirements: list, default: None + List of extra requirements to install in the container + installation_mode: "auto" | "pypi" | "github" | "folder" | "dev" | "no-install", default: "auto" + How spikeinterface is installed in the container: + * "auto": if host installation is a pip release then use "github" with tag + if host installation is DEV_MODE=True then use "dev" + * "pypi": use pypi with pip install spikeinterface + * "github": use github with `pip install git+https` + * "folder": mount a folder in container and install from this one. + So the version in the container is a different spikeinterface version from host, useful for + cross checks + * "dev": same as "folder", but the folder is the spikeinterface.__file__ to ensure same version as host + * "no-install": do not install spikeinterface in the container because it is already installed + spikeinterface_version: str, default: None + The spikeinterface version to install in the container. If None, the current version is used + spikeinterface_folder_source: Path or None, default: None + In case of installation_mode="folder", the spikeinterface folder source to use to install in the container **sorter_params: keyword args Spike sorter specific arguments (they can be retrieved with `get_default_sorter_params(sorter_name_or_class)`) Returns ------- - sortingextractor: SortingExtractor - The spike sorted data + BaseSorting | None + The spike sorted data (it `with_output` is True) or None (if `with_output` is False) """ @@ -162,6 +186,30 @@ def run_sorter_local( with_output=True, **sorter_params, ): + """ + Runs a sorter locally. + + Parameters + ---------- + sorter_name: str + The sorter name + recording: RecordingExtractor + The recording extractor to be spike sorted + output_folder: str or Path + Path to output folder. If None, a folder is created in the current directory + remove_existing_folder: bool, default: True + If True and output_folder exists yet then delete + delete_output_folder: bool, default: False + If True, output folder is deleted + verbose: bool, default: False + If True, output is verbose + raise_error: bool, default: True + If True, an error is raised if spike sorting fails. + If False, the process continues and the error is logged in the log file + with_output: bool, default: True + If True, the output Sorting is returned as a Sorting + **sorter_params: keyword args + """ if isinstance(recording, list): raise Exception("If you want to run several sorters/recordings use run_sorter_jobs(...)") @@ -192,155 +240,6 @@ def run_sorter_local( return sorting -def find_recording_folders(d): - folders_to_mount = [] - - def append_parent_folder(p): - p = Path(p) - folders_to_mount.append(p.resolve().absolute().parent) - return p - - _ = recursive_path_modifier(d, append_parent_folder, target="path", copy=True) - - try: # this will fail if on different drives (Windows) - base_folders_to_mount = [Path(os.path.commonpath(folders_to_mount))] - except ValueError: - base_folders_to_mount = folders_to_mount - - # let's not mount root if dries are /home/..., /mnt1/... - if len(base_folders_to_mount) == 1: - if len(str(base_folders_to_mount[0])) == 1: - base_folders_to_mount = folders_to_mount - - return base_folders_to_mount - - -def path_to_unix(path): - path = Path(path) - if platform.system() == "Windows": - path = Path(str(path)[str(path).find(":") + 1 :]) - return path.as_posix() - - -def windows_extractor_dict_to_unix(d): - d = recursive_path_modifier(d, path_to_unix, target="path", copy=True) - return d - - -class ContainerClient: - """ - Small abstraction class to run commands in: - * docker with "docker" python package - * singularity with "spython" python package - """ - - def __init__(self, mode, container_image, volumes, py_user_base, extra_kwargs): - """ - Parameters - ---------- - mode: "docker" | "singularity" - The container mode - container_image: str - container image name and tag - volumes: dict - dict of volumes to bind - py_user_base: str - Python user base folder to set as PYTHONUSERBASE env var in Singularity mode - Prevents from overwriting user's packages when running pip install - extra_kwargs: dict - Extra kwargs to start container - """ - assert mode in ("docker", "singularity") - self.mode = mode - self.py_user_base = py_user_base - container_requires_gpu = extra_kwargs.get("container_requires_gpu", None) - - if mode == "docker": - if not HAS_DOCKER: - raise ModuleNotFoundError("No module named 'docker'") - client = docker.from_env() - if container_requires_gpu is not None: - extra_kwargs.pop("container_requires_gpu") - extra_kwargs["device_requests"] = [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] - - if self._get_docker_image(container_image) is None: - print(f"Docker: pulling image {container_image}") - client.images.pull(container_image) - - self.docker_container = client.containers.create(container_image, tty=True, volumes=volumes, **extra_kwargs) - - elif mode == "singularity": - assert self.py_user_base, "py_user_base folder must be set in singularity mode" - from spython.main import Client - - # load local image file if it exists, otherwise search dockerhub - sif_file = Client._get_filename(container_image) - singularity_image = None - if Path(container_image).exists(): - singularity_image = container_image - elif Path(sif_file).exists(): - singularity_image = sif_file - else: - if HAS_DOCKER: - docker_image = self._get_docker_image(container_image) - if docker_image and len(docker_image.tags) > 0: - tag = docker_image.tags[0] - print(f"Building singularity image from local docker image: {tag}") - singularity_image = Client.build(f"docker-daemon://{tag}", sif_file, sudo=False) - if not singularity_image: - print(f"Singularity: pulling image {container_image}") - singularity_image = Client.pull(f"docker://{container_image}") - - if not Path(singularity_image).exists(): - raise FileNotFoundError(f"Unable to locate container image {container_image}") - - # bin options - singularity_bind = ",".join([f'{volume_src}:{volume["bind"]}' for volume_src, volume in volumes.items()]) - options = ["--bind", singularity_bind] - - # gpu options - if container_requires_gpu: - # only nvidia at the moment - options += ["--nv"] - - self.client_instance = Client.instance(singularity_image, start=False, options=options) - - @staticmethod - def _get_docker_image(container_image): - docker_client = docker.from_env(timeout=300) - try: - docker_image = docker_client.images.get(container_image) - except docker.errors.ImageNotFound: - docker_image = None - return docker_image - - def start(self): - if self.mode == "docker": - self.docker_container.start() - elif self.mode == "singularity": - self.client_instance.start() - - def stop(self): - if self.mode == "docker": - self.docker_container.stop() - self.docker_container.remove(force=True) - elif self.mode == "singularity": - self.client_instance.stop() - - def run_command(self, command): - if self.mode == "docker": - res = self.docker_container.exec_run(command) - return res.output.decode(encoding="utf-8", errors="ignore") - elif self.mode == "singularity": - from spython.main import Client - - options = ["--cleanenv", "--env", f"PYTHONUSERBASE={self.py_user_base}"] - res = Client.execute(self.client_instance, command, options=options) - if isinstance(res, dict): - res = res["message"] - return res - - def run_sorter_container( sorter_name: str, recording: BaseRecording, @@ -354,6 +253,9 @@ def run_sorter_container( with_output: bool = True, delete_container_files: bool = True, extra_requirements=None, + installation_mode="auto", + spikeinterface_version=None, + spikeinterface_folder_source=None, **sorter_params, ): """ @@ -384,10 +286,28 @@ def run_sorter_container( If True, the container temporary files are deleted after the sorting is done extra_requirements: list, default: None List of extra requirements to install in the container + installation_mode: "auto" | "pypi" | "github" | "folder" | "dev" | "no-install", default: "auto" + How spikeinterface is installed in the container: + * "auto": if host installation is a pip release then use "github" with tag + if host installation is DEV_MODE=True then use "dev" + * "pypi": use pypi with pip install spikeinterface + * "github": use github with `pip install git+https` + * "folder": mount a folder in container and install from this one. + So the version in the container is a different spikeinterface version from host, useful for + cross checks + * "dev": same as "folder", but the folder is the spikeinterface.__file__ to ensure same version as host + * "no-install": do not install spikeinterface in the container because it is already installed + spikeinterface_version: str, default: None + The spikeinterface version to install in the container. If None, the current version is used + spikeinterface_folder_source: Path or None, default: None + In case of installation_mode="folder", the spikeinterface folder source to use to install in the container **sorter_params: keyword args for the sorter """ + assert installation_mode in ("auto", "pypi", "github", "folder", "dev", "no-install") + spikeinterface_version = spikeinterface_version or si_version + if extra_requirements is None: extra_requirements = [] @@ -422,20 +342,20 @@ def run_sorter_container( elif recording.check_serializability("pickle"): (parent_folder / "in_container_recording.pickle").write_bytes(pickle.dumps(rec_dict)) else: - raise RuntimeError("To use run_sorter with container the recording must be serializable") + raise RuntimeError("To use run_sorter with a container the recording must be serializable") # need to share specific parameters (parent_folder / "in_container_params.json").write_text( json.dumps(check_json(sorter_params), indent=4), encoding="utf8" ) - npz_sorting_path = output_folder / "in_container_sorting" + in_container_sorting_folder = output_folder / "in_container_sorting" # if in Windows, skip C: parent_folder_unix = path_to_unix(parent_folder) output_folder_unix = path_to_unix(output_folder) recording_input_folders_unix = [path_to_unix(rf) for rf in recording_input_folders] - npz_sorting_path_unix = path_to_unix(npz_sorting_path) + in_container_sorting_folder_unix = path_to_unix(in_container_sorting_folder) # the py script py_script = f""" @@ -465,7 +385,7 @@ def run_sorter_container( remove_existing_folder={remove_existing_folder}, delete_output_folder=False, verbose={verbose}, raise_error={raise_error}, with_output=True, **sorter_params ) - sorting.save_to_folder(folder='{npz_sorting_path_unix}') + sorting.save(folder='{in_container_sorting_folder_unix}') """ (parent_folder / "in_container_sorter_script.py").write_text(py_script, encoding="utf8") @@ -476,15 +396,32 @@ def run_sorter_container( volumes[str(recording_folder)] = {"bind": str(recording_folder_unix), "mode": "ro"} volumes[str(parent_folder)] = {"bind": str(parent_folder_unix), "mode": "rw"} - si_dev_path = os.getenv("SPIKEINTERFACE_DEV_PATH", None) + host_folder_source = None + if installation_mode == "auto": + if DEV_MODE: + if is_editable_mode(): + installation_mode = "dev" + else: + installation_mode = "github" + else: + installation_mode = "github" + if verbose: + print(f"installation_mode='auto' switching to installation_mode: '{installation_mode}'") + + if installation_mode == "folder": + assert ( + spikeinterface_folder_source is not None + ), "for installation_mode='folder', spikeinterface_folder_source must be provided" + host_folder_source = Path(spikeinterface_folder_source) - install_si_from_source = False - if "dev" in si_version and si_dev_path is not None: - install_si_from_source = True - # Making sure to get rid of last / or \ - si_dev_path = str(Path(si_dev_path).absolute().resolve()) - si_dev_path_unix = path_to_unix(si_dev_path) - volumes[si_dev_path] = {"bind": si_dev_path_unix, "mode": "ro"} + if installation_mode == "dev": + host_folder_source = Path(spikeinterface.__file__).parents[2] + + if host_folder_source is not None: + host_folder_source = host_folder_source.resolve() + # this bind is read only and will be copy later + container_folder_source_ro = "/spikeinterface" + volumes[str(host_folder_source)] = {"bind": container_folder_source_ro, "mode": "ro"} extra_kwargs = {} @@ -513,68 +450,92 @@ def run_sorter_container( py_user_base_folder = parent_folder / "in_container_python_base" py_user_base_folder.mkdir(parents=True, exist_ok=True) py_user_base_unix = path_to_unix(py_user_base_folder) - si_source_folder = f"{py_user_base_unix}/sources" - else: - si_source_folder = "/sources" + container_client = ContainerClient(mode, container_image, volumes, py_user_base_unix, extra_kwargs) if verbose: print("Starting container") container_client.start() - if verbose and install_si_from_source: - print("******") - print("Container started with the following paths") - print(si_dev_path_unix, si_source_folder) - - # check if container contains spikeinterface already - cmd_1 = ["python", "-c", "import spikeinterface; print(spikeinterface.__version__)"] - cmd_2 = ["python", "-c", "from spikeinterface.sorters import run_sorter_local"] - res_output = "" - for cmd in [cmd_1, cmd_2]: - res_output += str(container_client.run_command(cmd)) - need_si_install = "ModuleNotFoundError" in res_output + if installation_mode == "no-install": + need_si_install = False + else: + cmd_1 = ["python", "-c", "import spikeinterface; print(spikeinterface.__version__)"] + cmd_2 = ["python", "-c", "from spikeinterface.sorters import run_sorter_local"] + res_output = "" + for cmd in [cmd_1, cmd_2]: + res_output += str(container_client.run_command(cmd)) + need_si_install = "ModuleNotFoundError" in res_output if need_si_install: - if "dev" in si_version: - if verbose: - print(f"Installing spikeinterface from sources in {container_image}") - - # TODO later check output - if install_si_from_source: - si_source = "local machine" - # install in local copy of host SI folder in sources/spikeinterface to avoid permission errors - cmd = f"mkdir {si_source_folder}" - res_output = container_client.run_command(cmd) - cmd = f"cp -r {si_dev_path_unix} {si_source_folder}" - res_output = container_client.run_command(cmd) - cmd = f"pip install --user {si_source_folder}/spikeinterface[full]" + # update pip in container + cmd = f"pip install --user --upgrade pip" + res_output = container_client.run_command(cmd) + + if installation_mode == "pypi": + install_package_in_container( + container_client, + "spikeinterface", + installation_mode="pypi", + extra="[full]", + version=spikeinterface_version, + verbose=verbose, + ) + + elif installation_mode == "github": + if DEV_MODE: + install_package_in_container( + container_client, + "spikeinterface", + installation_mode="github", + github_url="https://github.com/SpikeInterface/spikeinterface", + extra="[full]", + tag="main", + verbose=verbose, + ) else: - si_source = "remote repository" - cmd = "pip install --user --upgrade --no-input git+https://github.com/SpikeInterface/spikeinterface.git#egg=spikeinterface[full]" - if verbose: - print(f"Installing dev spikeinterface from {si_source}") - res_output = container_client.run_command(cmd) - cmd = "pip install --user --upgrade --no-input https://github.com/NeuralEnsemble/python-neo/archive/master.zip" - res_output = container_client.run_command(cmd) - else: - if verbose: - print(f"Installing spikeinterface=={si_version} in {container_image}") - cmd = f"pip install --user --upgrade --no-input spikeinterface[full]=={si_version}" - res_output = container_client.run_command(cmd) - else: - # TODO version checking - if verbose: - print(f"spikeinterface is already installed in {container_image}") + install_package_in_container( + container_client, + "spikeinterface", + installation_mode="github", + github_url="https://github.com/SpikeInterface/spikeinterface", + extra="[full]", + version=spikeinterface_version, + verbose=verbose, + ) + elif host_folder_source is not None: + # this is "dev" + "folder" + install_package_in_container( + container_client, + "spikeinterface", + installation_mode="folder", + extra="[full]", + container_folder_source=container_folder_source_ro, + verbose=verbose, + ) + + if installation_mode == "dev": + # also install neo from github + # cmd = "pip install --user --upgrade --no-input https://github.com/NeuralEnsemble/python-neo/archive/master.zip" + # res_output = container_client.run_command(cmd) + install_package_in_container( + container_client, + "neo", + installation_mode="github", + github_url="https://github.com/NeuralEnsemble/python-neo", + tag="master", + ) if hasattr(recording, "extra_requirements"): extra_requirements.extend(recording.extra_requirements) # install additional required dependencies if extra_requirements: - if verbose: - print(f"Installing extra requirements: {extra_requirements}") - cmd = f"pip install --user --upgrade --no-input {' '.join(extra_requirements)}" + # if verbose: + # print(f"Installing extra requirements: {extra_requirements}") + # cmd = f"pip install --user --upgrade --no-input {' '.join(extra_requirements)}" res_output = container_client.run_command(cmd) + for package_name in extra_requirements: + install_package_in_container(container_client, package_name, installation_mode="pypi", verbose=verbose) # run sorter on folder if verbose: @@ -634,14 +595,8 @@ def run_sorter_container( try: sorting = SorterClass.get_result_from_folder(output_folder) except Exception as e: - if verbose: - print( - "Failed to get result with sorter specific extractor.\n" - f"Error Message: {e}\n" - "Getting result from in-container saved NpzSortingExtractor" - ) try: - sorting = NpzSortingExtractor.load_from_folder(npz_sorting_path) + sorting = load_extractor(in_container_sorting_folder) except FileNotFoundError: SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\n{run_sorter_output}") @@ -652,14 +607,6 @@ def run_sorter_container( return sorting -_common_run_doc = ( - """ - Runs {} sorter - """ - + _common_param_doc -) - - def read_sorter_folder(output_folder, register_recording=True, sorting_info=True, raise_error=True): """ Load a sorting object from a spike sorting output folder. diff --git a/src/spikeinterface/sorters/tests/test_container_tools.py b/src/spikeinterface/sorters/tests/test_container_tools.py new file mode 100644 index 0000000000..2247385839 --- /dev/null +++ b/src/spikeinterface/sorters/tests/test_container_tools.py @@ -0,0 +1,131 @@ +import pytest +from pathlib import Path +from copy import deepcopy +import shutil +import os + +import spikeinterface as si +from spikeinterface.extractors import toy_example +from spikeinterface.sorters.container_tools import find_recording_folders, ContainerClient, install_package_in_container + +ON_GITHUB = bool(os.getenv("GITHUB_ACTIONS")) + +if hasattr(pytest, "global_test_folder"): + cache_folder = pytest.global_test_folder / "sorters" +else: + cache_folder = Path("cache_folder") / "sorters" + + +def setup_module(): + test_dirs = [cache_folder / "mono", cache_folder / "multi"] + for test_dir in test_dirs: + if test_dir.exists(): + shutil.rmtree(test_dir) + rec1, _ = toy_example(num_segments=1) + rec1 = rec1.save(folder=cache_folder / "mono") + + rec2, _ = toy_example(num_segments=3) + rec2 = rec2.save(folder=cache_folder / "multi") + + +def test_find_recording_folders(): + rec1 = si.load_extractor(cache_folder / "mono") + rec2 = si.load_extractor(cache_folder / "multi" / "binary.json", base_folder=cache_folder / "multi") + + d1 = rec1.to_dict() + d2 = rec2.to_dict() + d3 = deepcopy(d2) + d3["kwargs"]["file_paths"][0] = "/mnt1/my-path/my-folder1" + d3["kwargs"]["file_paths"][1] = "/mnt2/my-path/my-folder2" + d3["kwargs"]["file_paths"][2] = "/mnt3/my-path/my-folder3" + + # print(d1) + # print(d2) + # print(d3) + + f1 = find_recording_folders(d1) + f2 = find_recording_folders(d2) + f3 = find_recording_folders(d3) + + # print(f1) + # print(f2) + # print(f3) + + assert len(f1) == 1 + assert str(f1[0]) == str(cache_folder.absolute()) + + assert len(f2) == 1 + assert str(f2[0]) == str((cache_folder / "multi").absolute()) + + # in this case the paths are in 3 separate drives + assert len(f3) == 3 + + +@pytest.mark.skipif(ON_GITHUB, reason="Docker tests don't run on github: test locally") +def test_ContainerClient(): + mode = "docker" + container_image = "spikeinterface/tridesclous-base" + volumes = {} + py_user_base = None + extra_kwargs = {} + container_client = ContainerClient(mode, container_image, volumes, py_user_base, extra_kwargs) + container_client.start() + txt = container_client.run_command("pip list") + print(txt) + txt = container_client.run_command("pip install --user --upgrade pip") + print(txt) + txt = container_client.run_command("pip list") + print(txt) + container_client.stop() + + +@pytest.mark.skipif(ON_GITHUB, reason="Docker tests don't run on github: test locally") +def test_install_package_in_container(): + host_spikeinterface_source = Path(__file__).parents[4].as_posix() + + mode = "docker" + container_image = "spikeinterface/tridesclous-base" + volumes = {host_spikeinterface_source: {"bind": "/spikeinterface_container_sources", "mode": "ro"}} + py_user_base = None + extra_kwargs = {} + + container_client = ContainerClient(mode, container_image, volumes, py_user_base, extra_kwargs) + container_client.start() + + txt = container_client.run_command("pip install --user --upgrade pip") + + # # pypi installation + txt = install_package_in_container(container_client, "neo", installation_mode="pypi", version="0.11.0") + # print(txt) + txt = container_client.run_command("pip list") + # print(txt) + + # # github installation + txt = install_package_in_container( + container_client, "spikeinterface", extra="[full]", installation_mode="github", version="0.99.0" + ) + # print(txt) + txt = container_client.run_command("pip list") + # print(txt) + + # folder installation + txt = install_package_in_container( + container_client, + "spikeinterface", + extra="[full]", + installation_mode="folder", + container_folder_source="/spikeinterface_container_sources", + ) + # print(txt) + + txt = container_client.run_command("pip list") + # print(txt) + + container_client.stop() + + +if __name__ == "__main__": + # setup_module() + # test_find_recording_folders() + # test_ContainerClient() + test_install_package_in_container() diff --git a/src/spikeinterface/sorters/tests/test_find_folders.py b/src/spikeinterface/sorters/tests/test_find_folders.py deleted file mode 100644 index bcfe04f135..0000000000 --- a/src/spikeinterface/sorters/tests/test_find_folders.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytest -from pathlib import Path -from copy import deepcopy -import shutil - -import spikeinterface as si -from spikeinterface.extractors import toy_example -from spikeinterface.sorters.runsorter import find_recording_folders - - -if hasattr(pytest, "global_test_folder"): - cache_folder = pytest.global_test_folder / "sorters" -else: - cache_folder = Path("cache_folder") / "sorters" - - -def setup_module(): - test_dirs = [cache_folder / "mono", cache_folder / "multi"] - for test_dir in test_dirs: - if test_dir.exists(): - shutil.rmtree(test_dir) - rec1, _ = toy_example(num_segments=1) - rec1 = rec1.save(folder=cache_folder / "mono") - - rec2, _ = toy_example(num_segments=3) - rec2 = rec2.save(folder=cache_folder / "multi") - - -def test_find_recording_folders(): - rec1 = si.load_extractor(cache_folder / "mono") - rec2 = si.load_extractor(cache_folder / "multi" / "binary.json", base_folder=cache_folder / "multi") - - d1 = rec1.to_dict() - d2 = rec2.to_dict() - d3 = deepcopy(d2) - d3["kwargs"]["file_paths"][0] = "/mnt1/my-path/my-folder1" - d3["kwargs"]["file_paths"][1] = "/mnt2/my-path/my-folder2" - d3["kwargs"]["file_paths"][2] = "/mnt3/my-path/my-folder3" - - # print(d1) - # print(d2) - # print(d3) - - f1 = find_recording_folders(d1) - f2 = find_recording_folders(d2) - f3 = find_recording_folders(d3) - - # print(f1) - # print(f2) - # print(f3) - - assert len(f1) == 1 - assert str(f1[0]) == str(cache_folder.absolute()) - - assert len(f2) == 1 - assert str(f2[0]) == str((cache_folder / "multi").absolute()) - - # in this case the paths are in 3 separate drives - assert len(f3) == 3 - - -if __name__ == "__main__": - setup_module() - test_find_recording_folders() diff --git a/src/spikeinterface/sorters/tests/test_runsorter.py b/src/spikeinterface/sorters/tests/test_runsorter.py index 8cd812b57b..df7389e844 100644 --- a/src/spikeinterface/sorters/tests/test_runsorter.py +++ b/src/spikeinterface/sorters/tests/test_runsorter.py @@ -1,8 +1,10 @@ import os import pytest from pathlib import Path +import shutil -from spikeinterface import download_dataset +import spikeinterface as si +from spikeinterface import download_dataset, generate_ground_truth_recording, load_extractor from spikeinterface.extractors import read_mearec from spikeinterface.sorters import run_sorter @@ -14,10 +16,20 @@ else: cache_folder = Path("cache_folder") / "sorters" +rec_folder = cache_folder / "recording" + + +def setup_module(): + if rec_folder.exists(): + shutil.rmtree(rec_folder) + recording, sorting_gt = generate_ground_truth_recording(num_channels=8, durations=[10.0], seed=2205) + recording = recording.save(folder=rec_folder) + def test_run_sorter_local(): - local_path = download_dataset(remote_path="mearec/mearec_test_10s.h5") - recording, sorting_true = read_mearec(local_path) + # local_path = download_dataset(remote_path="mearec/mearec_test_10s.h5") + # recording, sorting_true = read_mearec(local_path) + recording = load_extractor(rec_folder) sorter_params = {"detect_threshold": 4.9} @@ -37,61 +49,79 @@ def test_run_sorter_local(): @pytest.mark.skipif(ON_GITHUB, reason="Docker tests don't run on github: test locally") def test_run_sorter_docker(): - mearec_filename = download_dataset(remote_path="mearec/mearec_test_10s.h5", unlock=True) - output_folder = cache_folder / "sorting_tdc_docker" + # mearec_filename = download_dataset(remote_path="mearec/mearec_test_10s.h5", unlock=True) + # recording, sorting_true = read_mearec(mearec_filename) - recording, sorting_true = read_mearec(mearec_filename) + recording = load_extractor(rec_folder) sorter_params = {"detect_threshold": 4.9} - docker_image = "spikeinterface/tridesclous-base:1.6.4-1" + docker_image = "spikeinterface/tridesclous-base" - sorting = run_sorter( - "tridesclous", - recording, - output_folder=output_folder, - remove_existing_folder=True, - delete_output_folder=False, - verbose=True, - raise_error=True, - docker_image=docker_image, - with_output=False, - **sorter_params, - ) - assert sorting is None - # TODO: Add another run with `with_output=True` and check sorting result + for installation_mode in ("dev", "pypi", "github"): + print(f"\nTest with installation_mode {installation_mode}") + output_folder = cache_folder / f"sorting_tdc_docker_{installation_mode}" + + sorting = run_sorter( + "tridesclous", + recording, + output_folder=output_folder, + remove_existing_folder=True, + delete_output_folder=False, + verbose=True, + raise_error=True, + docker_image=docker_image, + with_output=True, + installation_mode=installation_mode, + spikeinterface_version="0.99.1", + **sorter_params, + ) + print(sorting) + + shutil.rmtree(output_folder) @pytest.mark.skipif(ON_GITHUB, reason="Singularity tests don't run on github: test it locally") def test_run_sorter_singularity(): - mearec_filename = download_dataset(remote_path="mearec/mearec_test_10s.h5", unlock=True) - output_folder = cache_folder / "sorting_tdc_singularity" + # mearec_filename = download_dataset(remote_path="mearec/mearec_test_10s.h5", unlock=True) + # recording, sorting_true = read_mearec(mearec_filename) - recording, sorting_true = read_mearec(mearec_filename) + # use an output folder outside of the package. otherwise dev mode will not work + singularity_cache_folder = Path(si.__file__).parents[3] / "sandbox" + singularity_cache_folder.mkdir(exist_ok=True) - sorter_params = {"detect_threshold": 4.9} + recording = load_extractor(rec_folder) - singularity_image = "spikeinterface/tridesclous-base:1.6.4-1" + sorter_params = {"detect_threshold": 4.9} - sorting = run_sorter( - "tridesclous", - recording, - output_folder=output_folder, - remove_existing_folder=True, - delete_output_folder=False, - verbose=True, - raise_error=True, - singularity_image=singularity_image, - **sorter_params, - ) - print(sorting) + sorter_params = {"detect_threshold": 4.9} - # basic check to confirm sorting was successful - assert "Tridesclous" in sorting.to_dict()["class"] - assert len(sorting.get_unit_ids()) > 0 + singularity_image = "spikeinterface/tridesclous-base" + + for installation_mode in ("dev", "pypi", "github"): + print(f"\nTest with installation_mode {installation_mode}") + output_folder = singularity_cache_folder / f"sorting_tdc_singularity_{installation_mode}" + sorting = run_sorter( + "tridesclous", + recording, + output_folder=output_folder, + remove_existing_folder=True, + delete_output_folder=False, + verbose=True, + raise_error=True, + singularity_image=singularity_image, + delete_container_files=True, + installation_mode=installation_mode, + spikeinterface_version="0.99.1", + **sorter_params, + ) + print(sorting) + + shutil.rmtree(output_folder) if __name__ == "__main__": + setup_module() # test_run_sorter_local() # test_run_sorter_docker() test_run_sorter_singularity()