From c3113dafc14c135e677be715ccc0b98086b03426 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 12 Apr 2023 13:21:29 +0200 Subject: [PATCH] add support for running tools with singularity --- planemo/galaxy/config.py | 74 +++++++++++++++++++++++--------------- planemo/options.py | 78 +++++++++++++++++++++++++++++----------- 2 files changed, 103 insertions(+), 49 deletions(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 25e6a0835..989e77532 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -27,6 +27,7 @@ ) from galaxy.tool_util.deps import docker_util +from galaxy.tool_util.deps import singularity_util from galaxy.tool_util.deps.container_volumes import DockerVolume from galaxy.util.commands import argv_to_str from galaxy.util.yaml_util import ordered_dump @@ -117,11 +118,6 @@ "planemo_dest": { "runner": "planemo_runner", "require_container": False, - "docker_enabled": False, - "docker_sudo": False, - "docker_sudo_cmd": docker_util.DEFAULT_SUDO_COMMAND, - "docker_cmd": docker_util.DEFAULT_DOCKER_COMMAND, - "docker_volumes": "$defaults", }, "upload_dest": {"runner": "planemo_runner", "docker_enabled": False}, }, @@ -325,11 +321,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds): # Duplicate block in docker variant above. if kwds.get("mulled_containers", False): - if not kwds.get("docker", False): - if ctx.get_option_source("docker") != OptionSource.cli: + if not (kwds.get("docker", False) or kwds.get("singularity", False)): + if ( + ctx.get_option_source("docker") != OptionSource.cli + and ctx.get_option_source("singularity") != OptionSource.cli + ): kwds["docker"] = True else: - raise Exception("Specified no docker and mulled containers together.") + raise Exception("Specified --no-docker/--no-singularity and mulled containers together.") conda_default_options = ("conda_auto_init", "conda_auto_install") use_conda_options = ("dependency_resolution", "conda_use_local", "conda_prefix", "conda_exec") if not any(kwds.get(_) for _ in use_conda_options) and all( @@ -709,7 +708,7 @@ class GalaxyConfig(GalaxyInterface, metaclass=abc.ABCMeta): This assumes more than an API connection is available - Planemo needs to be able to start and stop the Galaxy instance, recover logs, etc... There are currently two - implementations - a locally executed Galaxy and one running inside a Docker containe + implementations - a locally executed Galaxy and one running inside a Docker container """ @abc.abstractproperty @@ -1322,26 +1321,43 @@ def _handle_job_config_file( "job_conf.yml", ) planemo_dest = JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] - planemo_dest["docker_enabled"] = kwds.get("docker", False) - planemo_dest["docker_sudo"] = kwds.get("docker_sudo", False) - planemo_dest["docker_sudo_cmd"] = kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND) - planemo_dest["docker_cmd"] = kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND) - - docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) - if docker_host: - planemo_dest["docker_host"] = docker_host - - volumes = list(kwds.get("docker_extra_volume") or []) - if test_data_dir: - volumes.append(f"{test_data_dir}:ro") - - docker_volumes_str = "$defaults" - if volumes: - # exclude tool directories, these are mounted :ro by $defaults - all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} - extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs) - docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}" - planemo_dest["docker_volumes"] = docker_volumes_str + + for container_type in ["docker", "singularity"]: + if not kwds.get(container_type, False): + continue + planemo_dest[f"{container_type}_enabled"] = kwds.get(container_type, False) + planemo_dest[f"{container_type}_sudo"] = kwds.get(f"{container_type}_sudo", False) + planemo_dest[f"{container_type}_sudo_cmd"] = kwds.get( + f"{container_type}_sudo_cmd", + docker_util.DEFAULT_SUDO_COMMAND + if container_type == "docker" + else singularity_util.DEFAULT_SUDO_COMMAND, + ) + planemo_dest[f"{container_type}_cmd"] = kwds.get( + f"{container_type}_cmd", + docker_util.DEFAULT_DOCKER_COMMAND + if container_type == "docker" + else singularity_util.DEFAULT_SINGULARITY_COMMAND, + ) + if container_type == "docker": + docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) + if docker_host: + planemo_dest["docker_host"] = docker_host + + volumes = list(kwds.get(f"{container_type}_extra_volume") or []) + if test_data_dir: + volumes.append(f"{test_data_dir}:ro") + volumes_str = "$defaults" + if volumes: + # exclude tool directories, these are mounted :ro by $defaults + all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} + extra_volumes_str = ",".join( + str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs + ) + volumes_str = f"{volumes_str},{extra_volumes_str}" + planemo_dest[f"{container_type}_volumes"] = volumes_str + break + JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] = planemo_dest with open(job_config_file, "w") as job_config_fh: ordered_dump(JOB_CONFIG_LOCAL, job_config_fh) diff --git a/planemo/options.py b/planemo/options.py index 143417adc..8c5f06f68 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -5,7 +5,10 @@ import os import click -from galaxy.tool_util.deps import docker_util +from galaxy.tool_util.deps import ( + docker_util, + singularity_util, +) from galaxy.tool_util.verify.interactor import DEFAULT_TOOL_TEST_WAIT from .config import planemo_option @@ -418,7 +421,7 @@ def mulled_containers_option(): "--mulled_containers", "--biocontainers", is_flag=True, - help="Test tools against mulled containers (forces --docker). Disables conda resolution unless any conda option has been set explicitly.", + help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.", ) @@ -458,13 +461,23 @@ def docker_extra_volume_option(): readable=True, resolve_path=True, ) - return planemo_option( - "--docker_extra_volume", - type=arg_type, - default=None, - use_global_config=True, - multiple=True, - help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."), + return _compose( + planemo_option( + "--docker_extra_volume", + type=arg_type, + default=None, + use_global_config=True, + multiple=True, + help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."), + ), + planemo_option( + "--singularity_extra_volume", + type=arg_type, + default=None, + use_global_config=True, + multiple=True, + help=("Extra path to mount if `--biocontainers` and `--singularity`."), + ), ) @@ -914,27 +927,52 @@ def no_cleanup_option(): def docker_enable_option(): - return planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")) + return _compose( + planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")), + planemo_option( + "--singularity/--no_singularity", default=False, help=("Run Galaxy tools in Singularity if enabled.") + ), + ) def docker_cmd_option(): - return planemo_option( - "--docker_cmd", - default=docker_util.DEFAULT_DOCKER_COMMAND, - help="Command used to launch docker (defaults to docker).", + return _compose( + planemo_option( + "--docker_cmd", + default=docker_util.DEFAULT_DOCKER_COMMAND, + help=f"Command used to launch docker (defaults to {docker_util.DEFAULT_DOCKER_COMMAND}).", + ), + planemo_option( + "--singularity_cmd", + default=singularity_util.DEFAULT_SINGULARITY_COMMAND, + help=f"Command used to launch singularity (defaults to {singularity_util.DEFAULT_SINGULARITY_COMMAND}).", + ), ) def docker_sudo_option(): - return planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker.") + return _compose( + planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker."), + planemo_option( + "--singularity_sudo/--no_singularity_sudo", is_flag=True, help="Flag to use sudo when running singularity." + ), + ) def docker_sudo_cmd_option(): - return planemo_option( - "--docker_sudo_cmd", - help="sudo command to use when --docker_sudo is enabled " + "(defaults to sudo).", - default=docker_util.DEFAULT_SUDO_COMMAND, - use_global_config=True, + return _compose( + planemo_option( + "--docker_sudo_cmd", + help=f"sudo command to use when --docker_sudo is enabled (defaults to {docker_util.DEFAULT_SUDO_COMMAND}).", + default=docker_util.DEFAULT_SUDO_COMMAND, + use_global_config=True, + ), + planemo_option( + "--singularity_sudo_cmd", + help=f"sudo command to use when --singularity_sudo is enabled (defaults to {singularity_util.DEFAULT_SUDO_COMMAND}).", + default=docker_util.DEFAULT_SUDO_COMMAND, + use_global_config=True, + ), )