Skip to content

Commit

Permalink
add support for running tools with singularity
Browse files Browse the repository at this point in the history
  • Loading branch information
@bernt-matthias
bernt-matthias committed Apr 12, 2023
1 parent e8d0fd2 commit c3113da
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 49 deletions.
74 changes: 45 additions & 29 deletions planemo/galaxy/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
)

from galaxy.tool_util.deps import docker_util
from galaxy.tool_util.deps import singularity_util
from galaxy.tool_util.deps.container_volumes import DockerVolume
from galaxy.util.commands import argv_to_str
from galaxy.util.yaml_util import ordered_dump
Expand Down Expand Up @@ -117,11 +118,6 @@
"planemo_dest": {
"runner": "planemo_runner",
"require_container": False,
"docker_enabled": False,
"docker_sudo": False,
"docker_sudo_cmd": docker_util.DEFAULT_SUDO_COMMAND,
"docker_cmd": docker_util.DEFAULT_DOCKER_COMMAND,
"docker_volumes": "$defaults",
},
"upload_dest": {"runner": "planemo_runner", "docker_enabled": False},
},
Expand Down Expand Up @@ -325,11 +321,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):

# Duplicate block in docker variant above.
if kwds.get("mulled_containers", False):
if not kwds.get("docker", False):
if ctx.get_option_source("docker") != OptionSource.cli:
if not (kwds.get("docker", False) or kwds.get("singularity", False)):
if (
ctx.get_option_source("docker") != OptionSource.cli
and ctx.get_option_source("singularity") != OptionSource.cli
):
kwds["docker"] = True
else:
raise Exception("Specified no docker and mulled containers together.")
raise Exception("Specified --no-docker/--no-singularity and mulled containers together.")
conda_default_options = ("conda_auto_init", "conda_auto_install")
use_conda_options = ("dependency_resolution", "conda_use_local", "conda_prefix", "conda_exec")
if not any(kwds.get(_) for _ in use_conda_options) and all(
Expand Down Expand Up @@ -709,7 +708,7 @@ class GalaxyConfig(GalaxyInterface, metaclass=abc.ABCMeta):
This assumes more than an API connection is available - Planemo needs to be able to
start and stop the Galaxy instance, recover logs, etc... There are currently two
implementations - a locally executed Galaxy and one running inside a Docker containe
implementations - a locally executed Galaxy and one running inside a Docker container
"""

@abc.abstractproperty
Expand Down Expand Up @@ -1322,26 +1321,43 @@ def _handle_job_config_file(
"job_conf.yml",
)
planemo_dest = JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"]
planemo_dest["docker_enabled"] = kwds.get("docker", False)
planemo_dest["docker_sudo"] = kwds.get("docker_sudo", False)
planemo_dest["docker_sudo_cmd"] = kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)
planemo_dest["docker_cmd"] = kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)

docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST)
if docker_host:
planemo_dest["docker_host"] = docker_host

volumes = list(kwds.get("docker_extra_volume") or [])
if test_data_dir:
volumes.append(f"{test_data_dir}:ro")

docker_volumes_str = "$defaults"
if volumes:
# exclude tool directories, these are mounted :ro by $defaults
all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths}
extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs)
docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}"
planemo_dest["docker_volumes"] = docker_volumes_str

for container_type in ["docker", "singularity"]:
if not kwds.get(container_type, False):
continue
planemo_dest[f"{container_type}_enabled"] = kwds.get(container_type, False)
planemo_dest[f"{container_type}_sudo"] = kwds.get(f"{container_type}_sudo", False)
planemo_dest[f"{container_type}_sudo_cmd"] = kwds.get(
f"{container_type}_sudo_cmd",
docker_util.DEFAULT_SUDO_COMMAND
if container_type == "docker"
else singularity_util.DEFAULT_SUDO_COMMAND,
)
planemo_dest[f"{container_type}_cmd"] = kwds.get(
f"{container_type}_cmd",
docker_util.DEFAULT_DOCKER_COMMAND
if container_type == "docker"
else singularity_util.DEFAULT_SINGULARITY_COMMAND,
)
if container_type == "docker":
docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST)
if docker_host:
planemo_dest["docker_host"] = docker_host

volumes = list(kwds.get(f"{container_type}_extra_volume") or [])
if test_data_dir:
volumes.append(f"{test_data_dir}:ro")
volumes_str = "$defaults"
if volumes:
# exclude tool directories, these are mounted :ro by $defaults
all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths}
extra_volumes_str = ",".join(
str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs
)
volumes_str = f"{volumes_str},{extra_volumes_str}"
planemo_dest[f"{container_type}_volumes"] = volumes_str
break

JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] = planemo_dest
with open(job_config_file, "w") as job_config_fh:
ordered_dump(JOB_CONFIG_LOCAL, job_config_fh)
Expand Down
78 changes: 58 additions & 20 deletions planemo/options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@
import os

import click
from galaxy.tool_util.deps import docker_util
from galaxy.tool_util.deps import (
docker_util,
singularity_util,
)
from galaxy.tool_util.verify.interactor import DEFAULT_TOOL_TEST_WAIT

from .config import planemo_option
Expand Down Expand Up @@ -418,7 +421,7 @@ def mulled_containers_option():
"--mulled_containers",
"--biocontainers",
is_flag=True,
help="Test tools against mulled containers (forces --docker). Disables conda resolution unless any conda option has been set explicitly.",
help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.",
)


Expand Down Expand Up @@ -458,13 +461,23 @@ def docker_extra_volume_option():
readable=True,
resolve_path=True,
)
return planemo_option(
"--docker_extra_volume",
type=arg_type,
default=None,
use_global_config=True,
multiple=True,
help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."),
return _compose(
planemo_option(
"--docker_extra_volume",
type=arg_type,
default=None,
use_global_config=True,
multiple=True,
help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."),
),
planemo_option(
"--singularity_extra_volume",
type=arg_type,
default=None,
use_global_config=True,
multiple=True,
help=("Extra path to mount if `--biocontainers` and `--singularity`."),
),
)


Expand Down Expand Up @@ -914,27 +927,52 @@ def no_cleanup_option():


def docker_enable_option():
return planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled."))
return _compose(
planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")),
planemo_option(
"--singularity/--no_singularity", default=False, help=("Run Galaxy tools in Singularity if enabled.")
),
)


def docker_cmd_option():
return planemo_option(
"--docker_cmd",
default=docker_util.DEFAULT_DOCKER_COMMAND,
help="Command used to launch docker (defaults to docker).",
return _compose(
planemo_option(
"--docker_cmd",
default=docker_util.DEFAULT_DOCKER_COMMAND,
help=f"Command used to launch docker (defaults to {docker_util.DEFAULT_DOCKER_COMMAND}).",
),
planemo_option(
"--singularity_cmd",
default=singularity_util.DEFAULT_SINGULARITY_COMMAND,
help=f"Command used to launch singularity (defaults to {singularity_util.DEFAULT_SINGULARITY_COMMAND}).",
),
)


def docker_sudo_option():
return planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker.")
return _compose(
planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker."),
planemo_option(
"--singularity_sudo/--no_singularity_sudo", is_flag=True, help="Flag to use sudo when running singularity."
),
)


def docker_sudo_cmd_option():
return planemo_option(
"--docker_sudo_cmd",
help="sudo command to use when --docker_sudo is enabled " + "(defaults to sudo).",
default=docker_util.DEFAULT_SUDO_COMMAND,
use_global_config=True,
return _compose(
planemo_option(
"--docker_sudo_cmd",
help=f"sudo command to use when --docker_sudo is enabled (defaults to {docker_util.DEFAULT_SUDO_COMMAND}).",
default=docker_util.DEFAULT_SUDO_COMMAND,
use_global_config=True,
),
planemo_option(
"--singularity_sudo_cmd",
help=f"sudo command to use when --singularity_sudo is enabled (defaults to {singularity_util.DEFAULT_SUDO_COMMAND}).",
default=docker_util.DEFAULT_SUDO_COMMAND,
use_global_config=True,
),
)


Expand Down

0 comments on commit c3113da

Please sign in to comment.