From 8f945a31f3add351684dbfb07e85ec9eec3683fd Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 11 Apr 2023 21:00:01 +0200 Subject: [PATCH 1/4] use yml job config --- planemo/galaxy/config.py | 82 ++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 46 deletions(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index adeb99670..be39570ef 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -29,6 +29,7 @@ from galaxy.tool_util.deps import docker_util from galaxy.tool_util.deps.container_volumes import DockerVolume from galaxy.util.commands import argv_to_str +from galaxy.util.yaml_util import ordered_dump from packaging.version import parse as parse_version from planemo import git @@ -121,32 +122,28 @@ """ -JOB_CONFIG_LOCAL = """ - - - - - - - - ${require_container} - ${docker_enable} - ${docker_sudo} - ${docker_sudo_cmd} - ${docker_cmd} - ${docker_volumes} - - ${docker_host_param} - - - false - - - - - - -""" +JOB_CONFIG_LOCAL: Dict[str, Any] = { + "runners": {"planemo_runner": {"load": "galaxy.jobs.runners.local:LocalJobRunner", "workers": 4}}, + "execution": { + "default": "planemo_dest", + "environments": { + "planemo_dest": { + "runner": "planemo_runner", + "require_container": False, + "docker_enabled": False, + "docker_sudo": False, + "docker_sudo_cmd": docker_util.DEFAULT_SUDO_COMMAND, + "docker_cmd": docker_util.DEFAULT_DOCKER_COMMAND, + "docker_volumes": "$defaults", + }, + "upload_dest": {"runner": "planemo_runner", "docker_enabled": False}, + }, + }, + "tools": [ + {"id": "upload1", "environment": "upload_dest"}, + ], +} + REFGENIE_CONFIG_TEMPLATE = """ config_version: %s @@ -1371,16 +1368,19 @@ def _handle_job_config_file( ): job_config_file = kwds.get("job_config_file", None) if not job_config_file: - template_str = JOB_CONFIG_LOCAL job_config_file = os.path.join( config_directory, - "job_conf.xml", + "job_conf.yml", ) - docker_enable = str(kwds.get("docker", False)) + planemo_dest = JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] + planemo_dest["docker_enabled"] = kwds.get("docker", False) + planemo_dest["docker_sudo"] = kwds.get("docker_sudo", False) + planemo_dest["docker_sudo_cmd"] = kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND) + planemo_dest["docker_cmd"] = kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND) + docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) - docker_host_param = "" if docker_host: - docker_host_param = f"""{docker_host}""" + planemo_dest["docker_host"] = docker_host volumes = list(kwds.get("docker_extra_volume") or []) if test_data_dir: @@ -1392,21 +1392,11 @@ def _handle_job_config_file( all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs) docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}" - - conf_contents = Template(template_str).safe_substitute( - { - "server_name": server_name, - "docker_enable": docker_enable, - "require_container": "false", - "docker_sudo": str(kwds.get("docker_sudo", False)), - "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)), - "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)), - "docker_host_param": docker_host_param, - "docker_volumes": docker_volumes_str, - "docker_run_extra_arguments": kwds.get("docker_run_extra_arguments", ""), - } - ) - write_file(job_config_file, conf_contents) + planemo_dest["docker_volumes"] = docker_volumes_str + planemo_dest["docker_run_extra_arguments"] = kwds.get("docker_run_extra_arguments", "") + JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] = planemo_dest + with open(job_config_file, "w") as job_config_fh: + ordered_dump(JOB_CONFIG_LOCAL, job_config_fh) kwds["job_config_file"] = job_config_file From 987ff880252f27a943de9e9098b5d037f4f332cc Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 12 Apr 2023 13:21:29 +0200 Subject: [PATCH 2/4] add support for running tools with singularity --- planemo/galaxy/config.py | 75 ++++++++++++++++++++++---------------- planemo/options.py | 78 +++++++++++++++++++++++++++++----------- 2 files changed, 103 insertions(+), 50 deletions(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index be39570ef..0a7008dd4 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -27,6 +27,7 @@ ) from galaxy.tool_util.deps import docker_util +from galaxy.tool_util.deps import singularity_util from galaxy.tool_util.deps.container_volumes import DockerVolume from galaxy.util.commands import argv_to_str from galaxy.util.yaml_util import ordered_dump @@ -130,11 +131,6 @@ "planemo_dest": { "runner": "planemo_runner", "require_container": False, - "docker_enabled": False, - "docker_sudo": False, - "docker_sudo_cmd": docker_util.DEFAULT_SUDO_COMMAND, - "docker_cmd": docker_util.DEFAULT_DOCKER_COMMAND, - "docker_volumes": "$defaults", }, "upload_dest": {"runner": "planemo_runner", "docker_enabled": False}, }, @@ -338,11 +334,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds): # Duplicate block in docker variant above. if kwds.get("mulled_containers", False): - if not kwds.get("docker", False): - if ctx.get_option_source("docker") != OptionSource.cli: + if not (kwds.get("docker", False) or kwds.get("singularity", False)): + if ( + ctx.get_option_source("docker") != OptionSource.cli + and ctx.get_option_source("singularity") != OptionSource.cli + ): kwds["docker"] = True else: - raise Exception("Specified no docker and mulled containers together.") + raise Exception("Specified --no-docker/--no-singularity and mulled containers together.") conda_default_options = ("conda_auto_init", "conda_auto_install") use_conda_options = ("dependency_resolution", "conda_use_local", "conda_prefix", "conda_exec") if not any(kwds.get(_) for _ in use_conda_options) and all( @@ -733,7 +732,7 @@ class GalaxyConfig(GalaxyInterface, metaclass=abc.ABCMeta): This assumes more than an API connection is available - Planemo needs to be able to start and stop the Galaxy instance, recover logs, etc... There are currently two - implementations - a locally executed Galaxy and one running inside a Docker containe + implementations - a locally executed Galaxy and one running inside a Docker container """ @abc.abstractproperty @@ -1373,27 +1372,43 @@ def _handle_job_config_file( "job_conf.yml", ) planemo_dest = JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] - planemo_dest["docker_enabled"] = kwds.get("docker", False) - planemo_dest["docker_sudo"] = kwds.get("docker_sudo", False) - planemo_dest["docker_sudo_cmd"] = kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND) - planemo_dest["docker_cmd"] = kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND) - - docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) - if docker_host: - planemo_dest["docker_host"] = docker_host - - volumes = list(kwds.get("docker_extra_volume") or []) - if test_data_dir: - volumes.append(f"{test_data_dir}:ro") - - docker_volumes_str = "$defaults" - if volumes: - # exclude tool directories, these are mounted :ro by $defaults - all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} - extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs) - docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}" - planemo_dest["docker_volumes"] = docker_volumes_str - planemo_dest["docker_run_extra_arguments"] = kwds.get("docker_run_extra_arguments", "") + + for container_type in ["docker", "singularity"]: + if not kwds.get(container_type, False): + continue + planemo_dest[f"{container_type}_enabled"] = kwds.get(container_type, False) + planemo_dest[f"{container_type}_sudo"] = kwds.get(f"{container_type}_sudo", False) + planemo_dest[f"{container_type}_sudo_cmd"] = kwds.get( + f"{container_type}_sudo_cmd", + docker_util.DEFAULT_SUDO_COMMAND + if container_type == "docker" + else singularity_util.DEFAULT_SUDO_COMMAND, + ) + planemo_dest[f"{container_type}_cmd"] = kwds.get( + f"{container_type}_cmd", + docker_util.DEFAULT_DOCKER_COMMAND + if container_type == "docker" + else singularity_util.DEFAULT_SINGULARITY_COMMAND, + ) + if container_type == "docker": + docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) + if docker_host: + planemo_dest["docker_host"] = docker_host + + volumes = list(kwds.get(f"{container_type}_extra_volume") or []) + if test_data_dir: + volumes.append(f"{test_data_dir}:ro") + volumes_str = "$defaults" + if volumes: + # exclude tool directories, these are mounted :ro by $defaults + all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} + extra_volumes_str = ",".join( + str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs + ) + volumes_str = f"{volumes_str},{extra_volumes_str}" + planemo_dest[f"{container_type}_volumes"] = volumes_str + break + JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] = planemo_dest with open(job_config_file, "w") as job_config_fh: ordered_dump(JOB_CONFIG_LOCAL, job_config_fh) diff --git a/planemo/options.py b/planemo/options.py index 867d637c2..1e22b5e1b 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -4,7 +4,10 @@ import os import click -from galaxy.tool_util.deps import docker_util +from galaxy.tool_util.deps import ( + docker_util, + singularity_util, +) from galaxy.tool_util.verify.interactor import DEFAULT_TOOL_TEST_WAIT from .config import planemo_option @@ -431,7 +434,7 @@ def mulled_containers_option(): "--mulled_containers", "--biocontainers", is_flag=True, - help="Test tools against mulled containers (forces --docker). Disables conda resolution unless any conda option has been set explicitly.", + help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.", ) @@ -471,13 +474,23 @@ def docker_extra_volume_option(): readable=True, resolve_path=True, ) - return planemo_option( - "--docker_extra_volume", - type=arg_type, - default=None, - use_global_config=True, - multiple=True, - help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."), + return _compose( + planemo_option( + "--docker_extra_volume", + type=arg_type, + default=None, + use_global_config=True, + multiple=True, + help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."), + ), + planemo_option( + "--singularity_extra_volume", + type=arg_type, + default=None, + use_global_config=True, + multiple=True, + help=("Extra path to mount if `--biocontainers` and `--singularity`."), + ), ) @@ -945,27 +958,52 @@ def no_cleanup_option(): def docker_enable_option(): - return planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")) + return _compose( + planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")), + planemo_option( + "--singularity/--no_singularity", default=False, help=("Run Galaxy tools in Singularity if enabled.") + ), + ) def docker_cmd_option(): - return planemo_option( - "--docker_cmd", - default=docker_util.DEFAULT_DOCKER_COMMAND, - help="Command used to launch docker (defaults to docker).", + return _compose( + planemo_option( + "--docker_cmd", + default=docker_util.DEFAULT_DOCKER_COMMAND, + help=f"Command used to launch docker (defaults to {docker_util.DEFAULT_DOCKER_COMMAND}).", + ), + planemo_option( + "--singularity_cmd", + default=singularity_util.DEFAULT_SINGULARITY_COMMAND, + help=f"Command used to launch singularity (defaults to {singularity_util.DEFAULT_SINGULARITY_COMMAND}).", + ), ) def docker_sudo_option(): - return planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker.") + return _compose( + planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker."), + planemo_option( + "--singularity_sudo/--no_singularity_sudo", is_flag=True, help="Flag to use sudo when running singularity." + ), + ) def docker_sudo_cmd_option(): - return planemo_option( - "--docker_sudo_cmd", - help="sudo command to use when --docker_sudo is enabled " + "(defaults to sudo).", - default=docker_util.DEFAULT_SUDO_COMMAND, - use_global_config=True, + return _compose( + planemo_option( + "--docker_sudo_cmd", + help=f"sudo command to use when --docker_sudo is enabled (defaults to {docker_util.DEFAULT_SUDO_COMMAND}).", + default=docker_util.DEFAULT_SUDO_COMMAND, + use_global_config=True, + ), + planemo_option( + "--singularity_sudo_cmd", + help=f"sudo command to use when --singularity_sudo is enabled (defaults to {singularity_util.DEFAULT_SUDO_COMMAND}).", + default=docker_util.DEFAULT_SUDO_COMMAND, + use_global_config=True, + ), ) From ec615c3126d0971d39ca6c37dc6c5666f699a7a4 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 22 Aug 2023 16:56:45 +0200 Subject: [PATCH 3/4] add option to specify container resolver config file --- planemo/galaxy/config.py | 2 ++ planemo/options.py | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 0a7008dd4..6edc6de5b 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -1433,6 +1433,8 @@ def _handle_container_resolution(ctx, kwds, galaxy_properties): involucro_context = build_involucro_context(ctx, **kwds) galaxy_properties["involucro_auto_init"] = "False" # Use planemo's galaxy_properties["involucro_path"] = involucro_context.involucro_bin + if kwds.get("container_resolvers_config_file"): + galaxy_properties["container_resolvers_config_file"] = kwds.get("container_resolvers_config_file") def _handle_job_metrics(config_directory, kwds): diff --git a/planemo/options.py b/planemo/options.py index 1e22b5e1b..3c3225ba4 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -429,12 +429,20 @@ def tool_data_path_option(): def mulled_containers_option(): - return planemo_option( - "mulled_containers", - "--mulled_containers", - "--biocontainers", - is_flag=True, - help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.", + return _compose( + planemo_option( + "mulled_containers", + "--mulled_containers", + "--biocontainers", + is_flag=True, + help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.", + ), + planemo_option( + "--container_resolvers_config_file", + type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True), + help="Container resolver config file", + default=None, + ), ) From ba53197531a64718f69a47bc489a93ea1cf73d9e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 18 Dec 2023 18:09:36 +0100 Subject: [PATCH 4/4] fix import style --- planemo/galaxy/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 6edc6de5b..5d6a06204 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -26,8 +26,10 @@ TYPE_CHECKING, ) -from galaxy.tool_util.deps import docker_util -from galaxy.tool_util.deps import singularity_util +from galaxy.tool_util.deps import ( + docker_util, + singularity_util, +) from galaxy.tool_util.deps.container_volumes import DockerVolume from galaxy.util.commands import argv_to_str from galaxy.util.yaml_util import ordered_dump