diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index adeb99670..5d6a06204 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -26,9 +26,13 @@ TYPE_CHECKING, ) -from galaxy.tool_util.deps import docker_util +from galaxy.tool_util.deps import ( + docker_util, + singularity_util, +) from galaxy.tool_util.deps.container_volumes import DockerVolume from galaxy.util.commands import argv_to_str +from galaxy.util.yaml_util import ordered_dump from packaging.version import parse as parse_version from planemo import git @@ -121,32 +125,23 @@ """ -JOB_CONFIG_LOCAL = """ - - - - - - - - ${require_container} - ${docker_enable} - ${docker_sudo} - ${docker_sudo_cmd} - ${docker_cmd} - ${docker_volumes} - - ${docker_host_param} - - - false - - - - - - -""" +JOB_CONFIG_LOCAL: Dict[str, Any] = { + "runners": {"planemo_runner": {"load": "galaxy.jobs.runners.local:LocalJobRunner", "workers": 4}}, + "execution": { + "default": "planemo_dest", + "environments": { + "planemo_dest": { + "runner": "planemo_runner", + "require_container": False, + }, + "upload_dest": {"runner": "planemo_runner", "docker_enabled": False}, + }, + }, + "tools": [ + {"id": "upload1", "environment": "upload_dest"}, + ], +} + REFGENIE_CONFIG_TEMPLATE = """ config_version: %s @@ -341,11 +336,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds): # Duplicate block in docker variant above. if kwds.get("mulled_containers", False): - if not kwds.get("docker", False): - if ctx.get_option_source("docker") != OptionSource.cli: + if not (kwds.get("docker", False) or kwds.get("singularity", False)): + if ( + ctx.get_option_source("docker") != OptionSource.cli + and ctx.get_option_source("singularity") != OptionSource.cli + ): kwds["docker"] = True else: - raise Exception("Specified no docker and mulled containers together.") + raise Exception("Specified --no-docker/--no-singularity and mulled containers together.") conda_default_options = ("conda_auto_init", "conda_auto_install") use_conda_options = ("dependency_resolution", "conda_use_local", "conda_prefix", "conda_exec") if not any(kwds.get(_) for _ in use_conda_options) and all( @@ -736,7 +734,7 @@ class GalaxyConfig(GalaxyInterface, metaclass=abc.ABCMeta): This assumes more than an API connection is available - Planemo needs to be able to start and stop the Galaxy instance, recover logs, etc... There are currently two - implementations - a locally executed Galaxy and one running inside a Docker containe + implementations - a locally executed Galaxy and one running inside a Docker container """ @abc.abstractproperty @@ -1371,42 +1369,51 @@ def _handle_job_config_file( ): job_config_file = kwds.get("job_config_file", None) if not job_config_file: - template_str = JOB_CONFIG_LOCAL job_config_file = os.path.join( config_directory, - "job_conf.xml", + "job_conf.yml", ) - docker_enable = str(kwds.get("docker", False)) - docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) - docker_host_param = "" - if docker_host: - docker_host_param = f"""{docker_host}""" - - volumes = list(kwds.get("docker_extra_volume") or []) - if test_data_dir: - volumes.append(f"{test_data_dir}:ro") - - docker_volumes_str = "$defaults" - if volumes: - # exclude tool directories, these are mounted :ro by $defaults - all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} - extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs) - docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}" - - conf_contents = Template(template_str).safe_substitute( - { - "server_name": server_name, - "docker_enable": docker_enable, - "require_container": "false", - "docker_sudo": str(kwds.get("docker_sudo", False)), - "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)), - "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)), - "docker_host_param": docker_host_param, - "docker_volumes": docker_volumes_str, - "docker_run_extra_arguments": kwds.get("docker_run_extra_arguments", ""), - } - ) - write_file(job_config_file, conf_contents) + planemo_dest = JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] + + for container_type in ["docker", "singularity"]: + if not kwds.get(container_type, False): + continue + planemo_dest[f"{container_type}_enabled"] = kwds.get(container_type, False) + planemo_dest[f"{container_type}_sudo"] = kwds.get(f"{container_type}_sudo", False) + planemo_dest[f"{container_type}_sudo_cmd"] = kwds.get( + f"{container_type}_sudo_cmd", + docker_util.DEFAULT_SUDO_COMMAND + if container_type == "docker" + else singularity_util.DEFAULT_SUDO_COMMAND, + ) + planemo_dest[f"{container_type}_cmd"] = kwds.get( + f"{container_type}_cmd", + docker_util.DEFAULT_DOCKER_COMMAND + if container_type == "docker" + else singularity_util.DEFAULT_SINGULARITY_COMMAND, + ) + if container_type == "docker": + docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST) + if docker_host: + planemo_dest["docker_host"] = docker_host + + volumes = list(kwds.get(f"{container_type}_extra_volume") or []) + if test_data_dir: + volumes.append(f"{test_data_dir}:ro") + volumes_str = "$defaults" + if volumes: + # exclude tool directories, these are mounted :ro by $defaults + all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths} + extra_volumes_str = ",".join( + str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs + ) + volumes_str = f"{volumes_str},{extra_volumes_str}" + planemo_dest[f"{container_type}_volumes"] = volumes_str + break + + JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] = planemo_dest + with open(job_config_file, "w") as job_config_fh: + ordered_dump(JOB_CONFIG_LOCAL, job_config_fh) kwds["job_config_file"] = job_config_file @@ -1428,6 +1435,8 @@ def _handle_container_resolution(ctx, kwds, galaxy_properties): involucro_context = build_involucro_context(ctx, **kwds) galaxy_properties["involucro_auto_init"] = "False" # Use planemo's galaxy_properties["involucro_path"] = involucro_context.involucro_bin + if kwds.get("container_resolvers_config_file"): + galaxy_properties["container_resolvers_config_file"] = kwds.get("container_resolvers_config_file") def _handle_job_metrics(config_directory, kwds): diff --git a/planemo/options.py b/planemo/options.py index 867d637c2..3c3225ba4 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -4,7 +4,10 @@ import os import click -from galaxy.tool_util.deps import docker_util +from galaxy.tool_util.deps import ( + docker_util, + singularity_util, +) from galaxy.tool_util.verify.interactor import DEFAULT_TOOL_TEST_WAIT from .config import planemo_option @@ -426,12 +429,20 @@ def tool_data_path_option(): def mulled_containers_option(): - return planemo_option( - "mulled_containers", - "--mulled_containers", - "--biocontainers", - is_flag=True, - help="Test tools against mulled containers (forces --docker). Disables conda resolution unless any conda option has been set explicitly.", + return _compose( + planemo_option( + "mulled_containers", + "--mulled_containers", + "--biocontainers", + is_flag=True, + help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.", + ), + planemo_option( + "--container_resolvers_config_file", + type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True), + help="Container resolver config file", + default=None, + ), ) @@ -471,13 +482,23 @@ def docker_extra_volume_option(): readable=True, resolve_path=True, ) - return planemo_option( - "--docker_extra_volume", - type=arg_type, - default=None, - use_global_config=True, - multiple=True, - help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."), + return _compose( + planemo_option( + "--docker_extra_volume", + type=arg_type, + default=None, + use_global_config=True, + multiple=True, + help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."), + ), + planemo_option( + "--singularity_extra_volume", + type=arg_type, + default=None, + use_global_config=True, + multiple=True, + help=("Extra path to mount if `--biocontainers` and `--singularity`."), + ), ) @@ -945,27 +966,52 @@ def no_cleanup_option(): def docker_enable_option(): - return planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")) + return _compose( + planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")), + planemo_option( + "--singularity/--no_singularity", default=False, help=("Run Galaxy tools in Singularity if enabled.") + ), + ) def docker_cmd_option(): - return planemo_option( - "--docker_cmd", - default=docker_util.DEFAULT_DOCKER_COMMAND, - help="Command used to launch docker (defaults to docker).", + return _compose( + planemo_option( + "--docker_cmd", + default=docker_util.DEFAULT_DOCKER_COMMAND, + help=f"Command used to launch docker (defaults to {docker_util.DEFAULT_DOCKER_COMMAND}).", + ), + planemo_option( + "--singularity_cmd", + default=singularity_util.DEFAULT_SINGULARITY_COMMAND, + help=f"Command used to launch singularity (defaults to {singularity_util.DEFAULT_SINGULARITY_COMMAND}).", + ), ) def docker_sudo_option(): - return planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker.") + return _compose( + planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker."), + planemo_option( + "--singularity_sudo/--no_singularity_sudo", is_flag=True, help="Flag to use sudo when running singularity." + ), + ) def docker_sudo_cmd_option(): - return planemo_option( - "--docker_sudo_cmd", - help="sudo command to use when --docker_sudo is enabled " + "(defaults to sudo).", - default=docker_util.DEFAULT_SUDO_COMMAND, - use_global_config=True, + return _compose( + planemo_option( + "--docker_sudo_cmd", + help=f"sudo command to use when --docker_sudo is enabled (defaults to {docker_util.DEFAULT_SUDO_COMMAND}).", + default=docker_util.DEFAULT_SUDO_COMMAND, + use_global_config=True, + ), + planemo_option( + "--singularity_sudo_cmd", + help=f"sudo command to use when --singularity_sudo is enabled (defaults to {singularity_util.DEFAULT_SUDO_COMMAND}).", + default=docker_util.DEFAULT_SUDO_COMMAND, + use_global_config=True, + ), )