From b818540ef170f9c9ea0bcafa65f2db0b652ba166 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 2 Sep 2020 10:58:00 +0200 Subject: [PATCH 01/23] prettify profile_list cmd --- planemo/commands/cmd_profile_list.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/planemo/commands/cmd_profile_list.py b/planemo/commands/cmd_profile_list.py index 815ab87ea..925e0e080 100644 --- a/planemo/commands/cmd_profile_list.py +++ b/planemo/commands/cmd_profile_list.py @@ -5,11 +5,15 @@ from planemo.cli import command_function from planemo.galaxy import profiles +from planemo.io import info @click.command('profile_list') @command_function def cli(ctx, **kwds): """List configured profile names.""" + info("Looking for profiles...") profile_names = profiles.list_profiles(ctx, **kwds) - print(profile_names) + for profile in profile_names: + print(profile) + info("{} configured profiles are available.".format(len(profile_names))) \ No newline at end of file From 7d6a879eb23086e4eedec4eacdee6bb44192a034 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 2 Sep 2020 13:09:40 +0200 Subject: [PATCH 02/23] allow creation of external_galaxy profiles --- planemo/commands/cmd_profile_create.py | 3 +++ planemo/galaxy/api.py | 13 +++++++++++++ planemo/galaxy/profiles.py | 24 +++++++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/planemo/commands/cmd_profile_create.py b/planemo/commands/cmd_profile_create.py index 74f614268..6f6660d38 100644 --- a/planemo/commands/cmd_profile_create.py +++ b/planemo/commands/cmd_profile_create.py @@ -13,6 +13,9 @@ @options.profile_database_options() @options.serve_engine_option() @options.docker_config_options() +@options.galaxy_url_option() +@options.galaxy_user_key_option() +@options.galaxy_admin_key_option() @command_function def cli(ctx, profile_name, **kwds): """Create a profile.""" diff --git a/planemo/galaxy/api.py b/planemo/galaxy/api.py index df8154978..19850bd2e 100644 --- a/planemo/galaxy/api.py +++ b/planemo/galaxy/api.py @@ -23,6 +23,19 @@ def gi(port=None, url=None, key=None): ) +def test_credentials_valid(port=None, url=None, key=None, is_admin=False): + """Test if provided API credentials are valid""" + test_gi = gi(port, url, key) + try: + test_gi.datasets.get_datasets(limit=0) + if is_admin: + test_gi.groups.get_groups() + except Exception: + return False + else: + return True + + def user_api_key(admin_gi): """Use an admin authenticated account to generate a user API key.""" ensure_module() diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py index 95a20dee4..cccd9fac4 100644 --- a/planemo/galaxy/profiles.py +++ b/planemo/galaxy/profiles.py @@ -13,6 +13,7 @@ OptionSource, ) from planemo.database import create_database_source +from planemo.galaxy.api import test_credentials_valid from .config import DATABASE_LOCATION_TEMPLATE PROFILE_OPTIONS_JSON_NAME = "planemo_profile_options.json" @@ -56,7 +57,14 @@ def create_profile(ctx, profile_name, **kwds): raise Exception(message) os.makedirs(profile_directory) - create_for_engine = _create_profile_docker if engine_type == "docker_galaxy" else _create_profile_local + + if engine_type == "docker_galaxy": + create_for_engine = _create_profile_docker + elif engine_type == "external_galaxy" or kwds.get("galaxy_url"): + create_for_engine = _create_profile_external + else: + create_for_engine = _create_profile_local + stored_profile_options = create_for_engine(ctx, profile_directory, profile_name, kwds) profile_options_path = _stored_profile_options_path(profile_directory) @@ -100,6 +108,20 @@ def _create_profile_local(ctx, profile_directory, profile_name, kwds): } +def _create_profile_external(ctx, profile_directory, profile_name, kwds): + url = kwds.get("galaxy_url") + api_key = kwds.get("galaxy_admin_key") or kwds.get("galaxy_user_key") + if test_credentials_valid(url=url, key=api_key, is_admin=kwds.get("galaxy_admin_key")): + return { + "galaxy_url": url, + "galaxy_user_key": kwds.get("galaxy_user_key"), + "galaxy_admin_key": kwds.get("galaxy_admin_key"), + "engine": "external_galaxy", + } + else: + raise ConnectionError('The credentials provided for an external Galaxy instance are not valid.') + + def ensure_profile(ctx, profile_name, **kwds): """Ensure a Galaxy profile exists and return profile defaults.""" if not profile_exists(ctx, profile_name, **kwds): From 5929da8245887879f0260334c142cb262faecbcc Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Thu, 3 Sep 2020 16:45:59 +0200 Subject: [PATCH 03/23] allow execution of workflows with only an ID --- planemo/commands/cmd_run.py | 19 ++++++++++++------- planemo/commands/cmd_test.py | 4 ---- planemo/engine/interface.py | 3 +-- planemo/galaxy/activity.py | 12 ++++++++++-- planemo/galaxy/config.py | 2 +- planemo/galaxy/workflows.py | 9 ++++++--- planemo/options.py | 17 +++++++++++++++++ planemo/runnable.py | 13 ++++++++++++- 8 files changed, 59 insertions(+), 20 deletions(-) diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index 23b2733fb..66cf3f9bb 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -2,6 +2,7 @@ from __future__ import print_function import json +import os import click from galaxy.util import unicodify @@ -10,11 +11,12 @@ from planemo.cli import command_function from planemo.engine import engine_context from planemo.io import warn +from planemo.runnable import for_path, for_id from planemo.tools import uri_to_path @click.command('run') -@options.required_tool_arg(allow_uris=True) +@options.required_runnable_arg() @options.required_job_arg() @options.galaxy_run_options() @options.galaxy_config_options() @@ -24,14 +26,19 @@ @options.run_output_json_option() @options.engine_options() @command_function -def cli(ctx, uri, job_path, **kwds): +def cli(ctx, runnable_identifier, job_path, **kwds): """Planemo command for running tools and jobs. \b % planemo run cat1-tool.cwl cat-job.json """ - path = uri_to_path(ctx, uri) - # TODO: convert UI to runnable and do a better test of cwl. + path = uri_to_path(ctx, runnable_identifier) + if os.path.exists(path): + runnable = for_path(path) + else: # assume galaxy workflow id + runnable = for_id(runnable_identifier) + + # TODO: do a better test of cwl. is_cwl = path.endswith(".cwl") kwds["cwl"] = is_cwl if kwds.get("engine", None) is None: @@ -41,14 +48,12 @@ def cli(ctx, uri, job_path, **kwds): kwds["engine"] = "external_galaxy" else: kwds["engine"] = "galaxy" - with engine_context(ctx, **kwds) as engine: - run_result = engine.run(path, job_path) + run_result = engine.run(runnable, job_path) if not run_result.was_successful: warn("Run failed [%s]" % unicodify(run_result)) ctx.exit(1) - outputs_dict = run_result.outputs_dict print(outputs_dict) output_json = kwds.get("output_json", None) diff --git a/planemo/commands/cmd_test.py b/planemo/commands/cmd_test.py index b37195ed8..c4054551a 100644 --- a/planemo/commands/cmd_test.py +++ b/planemo/commands/cmd_test.py @@ -32,10 +32,6 @@ "instances to limit generated traffic.", default="0", ) -@click.option( - "--history_name", - help="Name for history (if a history is generated as part of testing.)" -) @options.galaxy_target_options() @options.galaxy_config_options() @options.test_options() diff --git a/planemo/engine/interface.py b/planemo/engine/interface.py index 0f4178dd7..cf5ee9edb 100644 --- a/planemo/engine/interface.py +++ b/planemo/engine/interface.py @@ -53,9 +53,8 @@ def can_run(self, runnable): def cleanup(self): """Default no-op cleanup method.""" - def run(self, path, job_path): + def run(self, runnable, job_path): """Run a job using a compatible artifact (workflow or tool).""" - runnable = for_path(path) self._check_can_run(runnable) run_response = self._run(runnable, job_path) return run_response diff --git a/planemo/galaxy/activity.py b/planemo/galaxy/activity.py index f789f8b53..a05c97a04 100644 --- a/planemo/galaxy/activity.py +++ b/planemo/galaxy/activity.py @@ -118,7 +118,6 @@ def _execute(ctx, config, runnable, job_path, **kwds): except Exception: ctx.vlog("Problem with staging in data for Galaxy activities...") raise - if runnable.type in [RunnableType.galaxy_tool, RunnableType.cwl_tool]: response_class = GalaxyToolRunResponse tool_id = _verified_tool_id(runnable, user_gi) @@ -154,7 +153,11 @@ def _execute(ctx, config, runnable, job_path, **kwds): summarize_history(ctx, user_gi, history_id) elif runnable.type in [RunnableType.galaxy_workflow, RunnableType.cwl_workflow]: response_class = GalaxyWorkflowRunResponse - workflow_id = config.workflow_id(runnable.path) + if runnable.workflow_id: + runnable.workflow_dict = get_dict_from_workflow(user_gi, runnable.workflow_id) + workflow_id = runnable.workflow_id + else: + workflow_id = config.workflow_id(runnable.path) ctx.vlog("Found Galaxy workflow ID [%s] for path [%s]" % (workflow_id, runnable.path)) # TODO: Use the following when BioBlend 0.14 is released # invocation = user_gi.worklfows.invoke_workflow( @@ -344,6 +347,7 @@ def get_dataset(dataset_details, filename=None): return {"path": destination, "basename": basename} ctx.vlog("collecting outputs to directory %s" % output_directory) + for runnable_output in get_outputs(self._runnable): output_id = runnable_output.get_id() if not output_id: @@ -590,6 +594,10 @@ def _history_id(gi, **kwds): return history_id +def get_dict_from_workflow(gi, workflow_id): + return gi.workflows.export_workflow_dict(workflow_id) + + def _wait_for_invocation(ctx, gi, history_id, workflow_id, invocation_id, polling_backoff=0): def state_func(): diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index bc052597d..8aefedbe0 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -757,7 +757,7 @@ def ready(): def install_workflows(self): for runnable in self.runnables: - if runnable.type.name in ["galaxy_workflow", "cwl_workflow"]: + if runnable.type.name in ["galaxy_workflow", "cwl_workflow"] and not runnable.workflow_id: self._install_workflow(runnable) def _install_workflow(self, runnable): diff --git a/planemo/galaxy/workflows.py b/planemo/galaxy/workflows.py index 8fcc22a53..76226c58c 100644 --- a/planemo/galaxy/workflows.py +++ b/planemo/galaxy/workflows.py @@ -19,7 +19,6 @@ def load_shed_repos(runnable): if runnable.type.name != "galaxy_workflow": return [] - path = runnable.path if path.endswith(".ga"): generate_tool_list_from_ga_workflow_files.generate_tool_list_from_workflow([path], "Tools from workflows", "tools.yml") @@ -112,9 +111,13 @@ def register_tool_ids(tool_ids, workflow): WorkflowOutput = namedtuple("WorkflowOutput", ["order_index", "output_name", "label"]) -def describe_outputs(path): +def describe_outputs(runnable): """Return a list of :class:`WorkflowOutput` objects for target workflow.""" - workflow = _raw_dict(path) + if runnable.workflow_dict: + workflow = runnable.workflow_dict + else: + workflow = _raw_dict(runnable.path) + outputs = [] for (order_index, step) in workflow["steps"].items(): step_outputs = step.get("workflow_outputs", []) diff --git a/planemo/options.py b/planemo/options.py index 09aa1ee0b..fbeeec558 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -460,6 +460,14 @@ def galaxy_user_key_option(): ) +def history_name(): + return planemo_option( + "--history_name", + type=str, + help="Name to give a Galaxy history, if one is created.", + ) + + def no_cache_galaxy_option(): return planemo_option( "--no_cache_galaxy", @@ -709,6 +717,14 @@ def required_job_arg(): return click.argument("job_path", metavar="JOB_PATH", type=arg_type) +def required_runnable_arg(): + return click.argument( + 'runnable_identifier', + metavar="RUNNABLE_PATH_OR_ID", + type=str, + ) + + def _optional_tools_default(ctx, param, value): if param.name in ["paths", "uris"] and len(value) == 0: return [os.path.abspath(os.getcwd())] @@ -1416,6 +1432,7 @@ def engine_options(): galaxy_url_option(), galaxy_admin_key_option(), galaxy_user_key_option(), + history_name() ) diff --git a/planemo/runnable.py b/planemo/runnable.py index 53e6367fb..c3fd44140 100644 --- a/planemo/runnable.py +++ b/planemo/runnable.py @@ -70,6 +70,10 @@ def is_galaxy_artifact(runnable_type): class Runnable(_Runnable): """Abstraction describing tools and workflows.""" + def __init__(self, path, type): + self.workflow_id = None + self.workflow_dict = None + @property def test_data_search_path(self): """During testing, path to search for test data files.""" @@ -171,6 +175,13 @@ def for_paths(paths, temp_path=None): return [for_path(path, temp_path=temp_path) for path in paths] +def for_id(runnable_id): + """Produce a class:`Runnable` for supplied Galaxy workflow ID.""" + runnable = Runnable(None, RunnableType.galaxy_workflow) + runnable.workflow_id = runnable_id + return runnable + + def cases(runnable): """Build a `list` of :class:`TestCase` objects for specified runnable.""" cases = [] @@ -435,7 +446,7 @@ def get_outputs(runnable): outputs = [ToolOutput(o) for o in output_datasets.values()] return outputs elif runnable.type == RunnableType.galaxy_workflow: - workflow_outputs = describe_outputs(runnable.path) + workflow_outputs = describe_outputs(runnable) return [GalaxyWorkflowOutput(o) for o in workflow_outputs] elif runnable.type == RunnableType.cwl_workflow: workflow = workflow_proxy(runnable.path, strict_cwl_validation=False) From f2a768d7292f8c3dec2bb22756fe8e6446092032 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 4 Sep 2020 17:08:30 +0200 Subject: [PATCH 04/23] amend workflow utils test --- tests/test_galaxy_workflow_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_galaxy_workflow_utils.py b/tests/test_galaxy_workflow_utils.py index bb3d1f8e0..763c557ac 100644 --- a/tests/test_galaxy_workflow_utils.py +++ b/tests/test_galaxy_workflow_utils.py @@ -2,12 +2,14 @@ import os from planemo.galaxy.workflows import describe_outputs +from planemo.runnable import for_path from .test_utils import TEST_DATA_DIR def test_describe_outputs(): wf_path = os.path.join(TEST_DATA_DIR, "wf1.gxwf.yml") - outputs = describe_outputs(wf_path) + runnable = for_path(wf_path) + outputs = describe_outputs(runnable) assert len(outputs) == 1 output = outputs[0] assert output.order_index == 1 From cd0217265f4b2e7c81f5b6e997b1d9d17cc10d14 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 7 Sep 2020 10:17:36 +0200 Subject: [PATCH 05/23] linting --- planemo/commands/cmd_profile_list.py | 2 +- planemo/commands/cmd_run.py | 2 +- planemo/engine/interface.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/planemo/commands/cmd_profile_list.py b/planemo/commands/cmd_profile_list.py index 925e0e080..4a1f54d0c 100644 --- a/planemo/commands/cmd_profile_list.py +++ b/planemo/commands/cmd_profile_list.py @@ -16,4 +16,4 @@ def cli(ctx, **kwds): profile_names = profiles.list_profiles(ctx, **kwds) for profile in profile_names: print(profile) - info("{} configured profiles are available.".format(len(profile_names))) \ No newline at end of file + info("{} configured profiles are available.".format(len(profile_names))) diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index 66cf3f9bb..96582e0e0 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -11,7 +11,7 @@ from planemo.cli import command_function from planemo.engine import engine_context from planemo.io import warn -from planemo.runnable import for_path, for_id +from planemo.runnable import for_id, for_path from planemo.tools import uri_to_path diff --git a/planemo/engine/interface.py b/planemo/engine/interface.py index cf5ee9edb..e678e9b95 100644 --- a/planemo/engine/interface.py +++ b/planemo/engine/interface.py @@ -12,7 +12,6 @@ from planemo.io import error from planemo.runnable import ( cases, - for_path, RunnableType, ) from planemo.test.results import StructuredData From ea9d7fc261b1bf6e9c5631390ea015089dcd59dc Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Sat, 12 Sep 2020 12:54:55 +0200 Subject: [PATCH 06/23] start to add alias functionality --- planemo/galaxy/profiles.py | 34 ++++++++++++++++++++++++++++------ planemo/github_util.py | 1 + planemo/options.py | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py index cccd9fac4..e916a50ee 100644 --- a/planemo/galaxy/profiles.py +++ b/planemo/galaxy/profiles.py @@ -130,16 +130,37 @@ def ensure_profile(ctx, profile_name, **kwds): return _profile_options(ctx, profile_name, **kwds) +def create_alias(ctx, alias, obj, profile_name, **kwds): + if not profile_exists(ctx, profile_name, **kwds): + raise Exception("That profile does not exist. Create it with `planemo profile_create`") + profile_directory = _profile_directory(ctx, profile_name) + # profile_options = _read_profile_options(profile_directory) + profile_options_path = _stored_profile_options_path(profile_directory) + with open(profile_options_path) as f: + profile_options = json.load(f) + + if profile_options.get('aliases'): + profile_options['aliases'][alias] = obj + else: # no aliases yet defined + profile_options['aliases'] = {alias: obj} + + with open(profile_options_path, 'w') as f: + json.dump(profile_options, f) + + # aliases = profile_options['aliases'] + return 0 + + def _profile_options(ctx, profile_name, **kwds): profile_directory = _profile_directory(ctx, profile_name) profile_options = _read_profile_options(profile_directory) - specified_engine_type = kwds.get("engine", "galaxy") - profile_engine_type = profile_options["engine"] - if specified_engine_type != profile_engine_type: - if ctx.get_option_source("engine") == OptionSource.cli: - raise Exception("Configured profile engine type [%s] does not match specified engine type [%s].") + # specified_engine_type = kwds.get("engine", "galaxy") + # profile_engine_type = profile_options["engine"] + # if specified_engine_type != profile_engine_type: + # if ctx.get_option_source("engine") == OptionSource.cli: + # raise Exception("Configured profile engine type [%s] does not match specified engine type [%s].") - if profile_engine_type == "docker_galaxy": + if profile_options["engine"] == "docker_galaxy": engine_options = dict( export_directory=os.path.join(profile_directory, "export") ) @@ -158,6 +179,7 @@ def _profile_options(ctx, profile_name, **kwds): ) profile_options.update(engine_options) profile_options["galaxy_brand"] = profile_name + print(profile_options) return profile_options diff --git a/planemo/github_util.py b/planemo/github_util.py index a69a84d64..556e40691 100644 --- a/planemo/github_util.py +++ b/planemo/github_util.py @@ -64,6 +64,7 @@ def pull_request(ctx, path, message=None, **kwds): cmd = [hub_path, "pull-request"] if message is not None: cmd.extend(["-m", message]) + print(cmd) communicate(cmd, env=hub_env) diff --git a/planemo/options.py b/planemo/options.py index fbeeec558..8c1e1bb5b 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -1172,7 +1172,7 @@ def daemon_option(): def profile_option(): return planemo_option( "--profile", - type=str, + type=click.STRING, default=None, help=("Name of profile (created with the profile_create command) to use " "with this command.") From 73f330c40aef67b66a7776c8bbd6a0bd683aa228 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 15 Sep 2020 12:20:47 +0200 Subject: [PATCH 07/23] complete basic alias functionality --- planemo/commands/cmd_create_alias.py | 35 +++++++++++++++++++++ planemo/commands/cmd_delete_alias.py | 33 +++++++++++++++++++ planemo/commands/cmd_list_alias.py | 39 +++++++++++++++++++++++ planemo/commands/cmd_run.py | 5 ++- planemo/galaxy/profiles.py | 47 ++++++++++++++++++++++------ planemo/options.py | 13 +++++++- 6 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 planemo/commands/cmd_create_alias.py create mode 100644 planemo/commands/cmd_delete_alias.py create mode 100644 planemo/commands/cmd_list_alias.py diff --git a/planemo/commands/cmd_create_alias.py b/planemo/commands/cmd_create_alias.py new file mode 100644 index 000000000..138dfbf85 --- /dev/null +++ b/planemo/commands/cmd_create_alias.py @@ -0,0 +1,35 @@ +"""Module describing the planemo ``create_alias`` command.""" +import click + +from planemo.galaxy import profiles +from planemo import options +from planemo.cli import command_function +from planemo.config import planemo_option + +try: + import namesgenerator +except ImportError: + namesgenerator = None + + +@click.command('create_alias') +@click.argument( + "obj", + metavar="OBJ", + type=click.STRING, +) +@options.alias_option() +@options.profile_option(required=True) +@command_function +def cli(ctx, alias, obj, profile, **kwds): + """ + Add an alias for a path or a workflow or dataset ID. Aliases are associated with a particular planemo profile. + """ + if not alias: + if not namesgenerator: + raise ImportError('Random generation of aliases requires installation of the namesgenerator package. Either install this, or specify the alias name with --alias.') + alias = namesgenerator.get_random_name() + + exit_code = profiles.create_alias(ctx, alias, obj, profile) + ctx.exit(exit_code) + return \ No newline at end of file diff --git a/planemo/commands/cmd_delete_alias.py b/planemo/commands/cmd_delete_alias.py new file mode 100644 index 000000000..303d29daf --- /dev/null +++ b/planemo/commands/cmd_delete_alias.py @@ -0,0 +1,33 @@ +"""Module describing the planemo ``delete_alias`` command.""" +import click +import json + +from planemo.galaxy import profiles +from planemo import options +from planemo.cli import command_function +from planemo.config import planemo_option +from planemo.io import info, error + +try: + from tabulate import tabulate +except ImportError: + tabulate = None + + +@click.command('delete_alias') +@options.alias_option(required=True) +@options.profile_option(required=True) +@command_function +def cli(ctx, alias, profile, **kwds): + """ + List aliases for a path or a workflow or dataset ID. Aliases are associated with a particular planemo profile. + """ + info("Looking for profiles...") + exit_code = profiles.delete_alias(ctx, alias, profile) + if exit_code == 0: + info('Alias {} was successfully deleted from profile {}'.format(alias, profile)) + else: + error('Alias {} does not exist, so was not deleted from profile {}'.format(alias, profile)) + + ctx.exit(exit_code) + return \ No newline at end of file diff --git a/planemo/commands/cmd_list_alias.py b/planemo/commands/cmd_list_alias.py new file mode 100644 index 000000000..037946da6 --- /dev/null +++ b/planemo/commands/cmd_list_alias.py @@ -0,0 +1,39 @@ +"""Module describing the planemo ``list_alias`` command.""" +import click +import json + +from planemo.galaxy import profiles +from planemo import options +from planemo.cli import command_function +from planemo.config import planemo_option +from planemo.io import info + +try: + from tabulate import tabulate +except ImportError: + tabulate = None + +# try: +# import namesgenerator +# except ImportError: +# namesgenerator = None + + +@click.command('list_alias') +@options.profile_option(required=True) +@command_function +def cli(ctx, profile, **kwds): + """ + List aliases for a path or a workflow or dataset ID. Aliases are associated with a particular planemo profile. + """ + info("Looking for profiles...") + aliases = profiles.list_alias(ctx, profile) + if tabulate: + print(tabulate({"Alias": aliases.keys(), "Object": aliases.values()}, headers="keys")) + else: + print(json.dumps(aliases, indent=4, sort_keys=True)) + + info("{} aliases were found for profile {}.".format(len(aliases), profile)) + + ctx.exit(0) + return \ No newline at end of file diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index 96582e0e0..c136df1b1 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -13,7 +13,7 @@ from planemo.io import warn from planemo.runnable import for_id, for_path from planemo.tools import uri_to_path - +from planemo.galaxy.profiles import translate_alias @click.command('run') @options.required_runnable_arg() @@ -32,6 +32,7 @@ def cli(ctx, runnable_identifier, job_path, **kwds): \b % planemo run cat1-tool.cwl cat-job.json """ + runnable_identifier = translate_alias(ctx, runnable_identifier, kwds.get('profile')) path = uri_to_path(ctx, runnable_identifier) if os.path.exists(path): runnable = for_path(path) @@ -50,12 +51,10 @@ def cli(ctx, runnable_identifier, job_path, **kwds): kwds["engine"] = "galaxy" with engine_context(ctx, **kwds) as engine: run_result = engine.run(runnable, job_path) - if not run_result.was_successful: warn("Run failed [%s]" % unicodify(run_result)) ctx.exit(1) outputs_dict = run_result.outputs_dict - print(outputs_dict) output_json = kwds.get("output_json", None) if output_json: with open(output_json, "w") as f: diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py index e916a50ee..b962a43f1 100644 --- a/planemo/galaxy/profiles.py +++ b/planemo/galaxy/profiles.py @@ -131,13 +131,7 @@ def ensure_profile(ctx, profile_name, **kwds): def create_alias(ctx, alias, obj, profile_name, **kwds): - if not profile_exists(ctx, profile_name, **kwds): - raise Exception("That profile does not exist. Create it with `planemo profile_create`") - profile_directory = _profile_directory(ctx, profile_name) - # profile_options = _read_profile_options(profile_directory) - profile_options_path = _stored_profile_options_path(profile_directory) - with open(profile_options_path) as f: - profile_options = json.load(f) + profile_options, profile_options_path = _load_profile_to_json(ctx, profile_name) if profile_options.get('aliases'): profile_options['aliases'][alias] = obj @@ -147,10 +141,45 @@ def create_alias(ctx, alias, obj, profile_name, **kwds): with open(profile_options_path, 'w') as f: json.dump(profile_options, f) - # aliases = profile_options['aliases'] return 0 +def list_alias(ctx, profile_name, **kwds): + profile_options, _ = _load_profile_to_json(ctx, profile_name) + return profile_options.get('aliases', {}) + + +def delete_alias(ctx, alias, profile_name, **kwds): + profile_options, profile_options_path = _load_profile_to_json(ctx, profile_name) + if alias not in profile_options.get('aliases', {}): + return 1 + else: + del profile_options['aliases'][alias] + + with open(profile_options_path, 'w') as f: + json.dump(profile_options, f) + + return 0 + + +def translate_alias(ctx, alias, profile_name): + if not profile_name: + return alias + # print(_load_profile_to_json(ctx, profile_name)) + aliases = _load_profile_to_json(ctx, profile_name)[0].get('aliases', {}) + return aliases.get(alias, alias) + + +def _load_profile_to_json(ctx, profile_name): + if not profile_exists(ctx, profile_name): + raise Exception("That profile does not exist. Create it with `planemo profile_create`") + profile_directory = _profile_directory(ctx, profile_name) + profile_options_path = _stored_profile_options_path(profile_directory) + with open(profile_options_path) as f: + profile_options = json.load(f) + return profile_options, profile_options_path + + def _profile_options(ctx, profile_name, **kwds): profile_directory = _profile_directory(ctx, profile_name) profile_options = _read_profile_options(profile_directory) @@ -179,7 +208,7 @@ def _profile_options(ctx, profile_name, **kwds): ) profile_options.update(engine_options) profile_options["galaxy_brand"] = profile_name - print(profile_options) + # print(profile_options) return profile_options diff --git a/planemo/options.py b/planemo/options.py index 8c1e1bb5b..ab8341e40 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -1169,16 +1169,27 @@ def daemon_option(): ) -def profile_option(): +def profile_option(required=False): return planemo_option( "--profile", type=click.STRING, + required=required, default=None, help=("Name of profile (created with the profile_create command) to use " "with this command.") ) +def alias_option(required=False): + return planemo_option( + "--alias", + type=click.STRING, + required=required, + default=None, + help=("Name of an alias.") + ) + + def galaxy_serve_options(): return _compose( galaxy_run_options(), From abf8f08da0a5f35a557cbec401bdc568d4cac078 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 15 Sep 2020 15:41:28 +0200 Subject: [PATCH 08/23] ensure history_name cannot be None) --- planemo/galaxy/activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planemo/galaxy/activity.py b/planemo/galaxy/activity.py index a05c97a04..77dc66051 100644 --- a/planemo/galaxy/activity.py +++ b/planemo/galaxy/activity.py @@ -589,7 +589,7 @@ def _tool_id(tool_path): def _history_id(gi, **kwds): history_id = kwds.get("history_id", None) if history_id is None: - history_name = kwds.get("history_name", DEFAULT_HISTORY_NAME) + history_name = kwds.get("history_name", DEFAULT_HISTORY_NAME) or DEFAULT_HISTORY_NAME history_id = gi.histories.create_history(history_name)["id"] return history_id From 1ffc575705ec1c68f60e0b3537939b60fd2e03ba Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 16 Sep 2020 10:01:03 +0200 Subject: [PATCH 09/23] [skip ci] From 033410472d9a801a80f09eb72cd1548c9d11fc80 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 5 Oct 2020 13:53:21 +0200 Subject: [PATCH 10/23] improve test_credentials_valid function --- planemo/galaxy/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/planemo/galaxy/api.py b/planemo/galaxy/api.py index 19850bd2e..d311a1173 100644 --- a/planemo/galaxy/api.py +++ b/planemo/galaxy/api.py @@ -27,13 +27,13 @@ def test_credentials_valid(port=None, url=None, key=None, is_admin=False): """Test if provided API credentials are valid""" test_gi = gi(port, url, key) try: - test_gi.datasets.get_datasets(limit=0) + current_user = test_gi.users.get_current_user() if is_admin: - test_gi.groups.get_groups() + return current_user['is_admin'] + else: + return True except Exception: return False - else: - return True def user_api_key(admin_gi): From f421e87668789e0b35d6eca02845e7ec9b288bbd Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Mon, 12 Oct 2020 17:16:08 +0200 Subject: [PATCH 11/23] add list_invocations command --- planemo/commands/cmd_list_alias.py | 5 --- planemo/commands/cmd_list_invocations.py | 55 ++++++++++++++++++++++++ planemo/galaxy/api.py | 6 +++ 3 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 planemo/commands/cmd_list_invocations.py diff --git a/planemo/commands/cmd_list_alias.py b/planemo/commands/cmd_list_alias.py index 037946da6..10b2490bb 100644 --- a/planemo/commands/cmd_list_alias.py +++ b/planemo/commands/cmd_list_alias.py @@ -13,11 +13,6 @@ except ImportError: tabulate = None -# try: -# import namesgenerator -# except ImportError: -# namesgenerator = None - @click.command('list_alias') @options.profile_option(required=True) diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py new file mode 100644 index 000000000..8092e76a4 --- /dev/null +++ b/planemo/commands/cmd_list_invocations.py @@ -0,0 +1,55 @@ +"""Module describing the planemo ``create_alias`` command.""" +import click + +from planemo.galaxy import profiles +from planemo import options +from planemo.cli import command_function +from planemo.config import planemo_option +from planemo.galaxy.api import get_invocations +from planemo.io import error, info + +try: + from tabulate import tabulate +except ImportError: + tabulate = None + + +@click.command('list_invocations') +@click.argument( + "workflow_id", + type=click.STRING, +) +@options.profile_option(required=True) +@command_function +def cli(ctx, workflow_id, **kwds): + """ + Get a list of invocations for a particular workflow ID or alias. + """ + info("Looking for invocations for workflow {}...".format(workflow_id)) + workflow_id = profiles.translate_alias(ctx, workflow_id, kwds.get('profile')) + profile = profiles.ensure_profile(ctx, kwds.get('profile')) + + invocations = get_invocations(url=profile['galaxy_url'], key=profile['galaxy_admin_key'] or profile['galaxy_user_key'], workflow_id=workflow_id) + + if tabulate: + state_colors = { + 'ok': '\033[92m', # green + 'running': '\033[93m', # yellow + 'error': '\033[91m', # red + 'paused': '\033[96m', # cyan + 'deleted': '\033[95m', # magenta + 'deleted_new': '\033[95m', # magenta + 'new': '\033[96m', # cyan + 'queued': '\033[93m', # yellow + } + print(tabulate({ + "Invocation ID": invocations.keys(), + "Jobs status": [', '.join(['{}{} jobs {}\033[0m'.format(state_colors[k], v, k) for k, v in inv_states.items()]) for inv_states in invocations.values()], + "Invocation report URL": ['{}/api/invocations/{}/report.pdf'.format(profile['galaxy_url'].strip('/'), inv_id) for inv_id in invocations] + }, headers="keys")) + else: + error("The tabulate package is not installed, invocations could not be listed correctly.") + print(invocations) + info("{} invocations found.".format(len(invocations))) + + return \ No newline at end of file diff --git a/planemo/galaxy/api.py b/planemo/galaxy/api.py index d311a1173..ea1e807c7 100644 --- a/planemo/galaxy/api.py +++ b/planemo/galaxy/api.py @@ -111,6 +111,12 @@ def summarize_history(ctx, gi, history_id): print("|") +def get_invocations(url, key, workflow_id): + inv_gi = gi(None, url, key) + invocations = inv_gi.workflows.get_invocations(workflow_id) + return {invocation['id']: inv_gi.invocations.get_invocation_summary(invocation['id'])['states'] for invocation in invocations} + + def _format_for_summary(blob, empty_message, prefix="| "): contents = "\n".join(["%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r")]) return contents or "%s*%s*" % (prefix, empty_message) From d66fbe9d92925d46a5f5c9e60dd4cb1b52795831 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 14 Oct 2020 17:42:34 +0200 Subject: [PATCH 12/23] update workflow_edit command to work with external galaxies --- planemo/commands/cmd_list_invocations.py | 2 +- planemo/commands/cmd_workflow_edit.py | 17 +++++++++++++---- planemo/galaxy/workflows.py | 7 +------ planemo/options.py | 11 ++++------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py index 8092e76a4..7941ae0c2 100644 --- a/planemo/commands/cmd_list_invocations.py +++ b/planemo/commands/cmd_list_invocations.py @@ -40,7 +40,7 @@ def cli(ctx, workflow_id, **kwds): 'deleted': '\033[95m', # magenta 'deleted_new': '\033[95m', # magenta 'new': '\033[96m', # cyan - 'queued': '\033[93m', # yellow + 'queued': '\033[93m', # yellow } print(tabulate({ "Invocation ID": invocations.keys(), diff --git a/planemo/commands/cmd_workflow_edit.py b/planemo/commands/cmd_workflow_edit.py index ceceb194d..bac6daf61 100644 --- a/planemo/commands/cmd_workflow_edit.py +++ b/planemo/commands/cmd_workflow_edit.py @@ -1,5 +1,6 @@ """Module describing the planemo ``workflow_edit`` command.""" import click +import os from planemo import options from planemo.cli import command_function @@ -7,8 +8,10 @@ engine_context, is_galaxy_engine, ) +from planemo.galaxy.profiles import translate_alias from planemo.galaxy.serve import sleep_for_serve from planemo.runnable import ( + for_id, for_path, ) @@ -17,17 +20,23 @@ @options.required_workflow_arg() @options.galaxy_serve_options() @command_function -def cli(ctx, workflow_path, output=None, force=False, **kwds): +def cli(ctx, workflow_identifier, output=None, force=False, **kwds): """Open a synchronized Galaxy workflow editor. """ assert is_galaxy_engine(**kwds) + workflow_identifier = translate_alias(ctx, workflow_identifier, kwds.get('profile')) + if os.path.exists(workflow_identifier): + runnable = for_path(workflow_identifier) + else: # assume galaxy workflow id + runnable = for_id(workflow_identifier) + kwds["workflows_from_path"] = True - runnable = for_path(workflow_path) with engine_context(ctx, **kwds) as galaxy_engine: with galaxy_engine.ensure_runnables_served([runnable]) as config: - workflow_id = config.workflow_id(workflow_path) + workflow_id = runnable.workflow_id or config.workflow_id(runnable.path) url = "%s/workflow/editor?id=%s" % (config.galaxy_url, workflow_id) click.launch(url) - sleep_for_serve() + if kwds["engine"] != "external_galaxy": + sleep_for_serve() diff --git a/planemo/galaxy/workflows.py b/planemo/galaxy/workflows.py index 76226c58c..8cde4d648 100644 --- a/planemo/galaxy/workflows.py +++ b/planemo/galaxy/workflows.py @@ -65,13 +65,8 @@ def import_workflow(path, admin_gi, user_gi, from_path=False): workflow = _raw_dict(path, importer) return user_gi.workflows.import_workflow_dict(workflow) else: - # TODO: Update bioblend to allow from_path. path = os.path.abspath(path) - payload = dict( - from_path=path - ) - workflows_url = user_gi.url + '/workflows' - workflow = user_gi.workflows._post(payload, url=workflows_url) + workflow = user_gi.workflows.import_workflow_from_local_path(path) return workflow diff --git a/planemo/options.py b/planemo/options.py index ab8341e40..18889c6b8 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -690,14 +690,11 @@ def single_user_mode_option(): def required_workflow_arg(): - arg_type = click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=False, + return click.argument( + 'workflow_identifier', + metavar="WORKFLOW_PATH_OR_ID", + type=str, ) - return click.argument("workflow_path", metavar="WORKFLOW_PATH", type=arg_type) def split_job_and_test(): From 1f9b3bb6b7979dcd21f4990b23924ff5dc2327a9 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 20 Oct 2020 09:57:22 +0200 Subject: [PATCH 13/23] lint --- planemo/commands/cmd_create_alias.py | 10 +++++----- planemo/commands/cmd_delete_alias.py | 12 +++++------- planemo/commands/cmd_list_alias.py | 12 ++++++------ planemo/commands/cmd_list_invocations.py | 13 +++++++------ planemo/commands/cmd_workflow_edit.py | 3 ++- planemo/galaxy/profiles.py | 7 ++----- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/planemo/commands/cmd_create_alias.py b/planemo/commands/cmd_create_alias.py index 138dfbf85..fb762a9ef 100644 --- a/planemo/commands/cmd_create_alias.py +++ b/planemo/commands/cmd_create_alias.py @@ -1,10 +1,9 @@ """Module describing the planemo ``create_alias`` command.""" import click -from planemo.galaxy import profiles from planemo import options from planemo.cli import command_function -from planemo.config import planemo_option +from planemo.galaxy import profiles try: import namesgenerator @@ -27,9 +26,10 @@ def cli(ctx, alias, obj, profile, **kwds): """ if not alias: if not namesgenerator: - raise ImportError('Random generation of aliases requires installation of the namesgenerator package. Either install this, or specify the alias name with --alias.') + raise ImportError(("Random generation of aliases requires installation of the namesgenerator package." + "Either install this, or specify the alias name with --alias.")) alias = namesgenerator.get_random_name() - + exit_code = profiles.create_alias(ctx, alias, obj, profile) ctx.exit(exit_code) - return \ No newline at end of file + return diff --git a/planemo/commands/cmd_delete_alias.py b/planemo/commands/cmd_delete_alias.py index 303d29daf..d2a71d503 100644 --- a/planemo/commands/cmd_delete_alias.py +++ b/planemo/commands/cmd_delete_alias.py @@ -1,12 +1,10 @@ """Module describing the planemo ``delete_alias`` command.""" import click -import json -from planemo.galaxy import profiles from planemo import options from planemo.cli import command_function -from planemo.config import planemo_option -from planemo.io import info, error +from planemo.galaxy import profiles +from planemo.io import error, info try: from tabulate import tabulate @@ -22,12 +20,12 @@ def cli(ctx, alias, profile, **kwds): """ List aliases for a path or a workflow or dataset ID. Aliases are associated with a particular planemo profile. """ - info("Looking for profiles...") + info("Looking for profiles...") exit_code = profiles.delete_alias(ctx, alias, profile) if exit_code == 0: info('Alias {} was successfully deleted from profile {}'.format(alias, profile)) else: error('Alias {} does not exist, so was not deleted from profile {}'.format(alias, profile)) - + ctx.exit(exit_code) - return \ No newline at end of file + return diff --git a/planemo/commands/cmd_list_alias.py b/planemo/commands/cmd_list_alias.py index 10b2490bb..1f08d3827 100644 --- a/planemo/commands/cmd_list_alias.py +++ b/planemo/commands/cmd_list_alias.py @@ -1,11 +1,11 @@ """Module describing the planemo ``list_alias`` command.""" -import click import json -from planemo.galaxy import profiles +import click + from planemo import options from planemo.cli import command_function -from planemo.config import planemo_option +from planemo.galaxy import profiles from planemo.io import info try: @@ -21,14 +21,14 @@ def cli(ctx, profile, **kwds): """ List aliases for a path or a workflow or dataset ID. Aliases are associated with a particular planemo profile. """ - info("Looking for profiles...") + info("Looking for profiles...") aliases = profiles.list_alias(ctx, profile) if tabulate: print(tabulate({"Alias": aliases.keys(), "Object": aliases.values()}, headers="keys")) else: print(json.dumps(aliases, indent=4, sort_keys=True)) - + info("{} aliases were found for profile {}.".format(len(aliases), profile)) ctx.exit(0) - return \ No newline at end of file + return diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py index 7941ae0c2..28fe82bf6 100644 --- a/planemo/commands/cmd_list_invocations.py +++ b/planemo/commands/cmd_list_invocations.py @@ -1,10 +1,9 @@ """Module describing the planemo ``create_alias`` command.""" import click -from planemo.galaxy import profiles from planemo import options from planemo.cli import command_function -from planemo.config import planemo_option +from planemo.galaxy import profiles from planemo.galaxy.api import get_invocations from planemo.io import error, info @@ -30,7 +29,7 @@ def cli(ctx, workflow_id, **kwds): profile = profiles.ensure_profile(ctx, kwds.get('profile')) invocations = get_invocations(url=profile['galaxy_url'], key=profile['galaxy_admin_key'] or profile['galaxy_user_key'], workflow_id=workflow_id) - + if tabulate: state_colors = { 'ok': '\033[92m', # green @@ -44,12 +43,14 @@ def cli(ctx, workflow_id, **kwds): } print(tabulate({ "Invocation ID": invocations.keys(), - "Jobs status": [', '.join(['{}{} jobs {}\033[0m'.format(state_colors[k], v, k) for k, v in inv_states.items()]) for inv_states in invocations.values()], - "Invocation report URL": ['{}/api/invocations/{}/report.pdf'.format(profile['galaxy_url'].strip('/'), inv_id) for inv_id in invocations] + "Jobs status": [', '.join(['{}{} jobs {}\033[0m'.format(state_colors[k], v, k) for k, v in inv_states.items()] + ) for inv_states in invocations.values()], + "Invocation report URL": ['{}/api/invocations/{}/report.pdf'.format(profile['galaxy_url'].strip('/' + ), inv_id) for inv_id in invocations] }, headers="keys")) else: error("The tabulate package is not installed, invocations could not be listed correctly.") print(invocations) info("{} invocations found.".format(len(invocations))) - return \ No newline at end of file + return diff --git a/planemo/commands/cmd_workflow_edit.py b/planemo/commands/cmd_workflow_edit.py index bac6daf61..12afca024 100644 --- a/planemo/commands/cmd_workflow_edit.py +++ b/planemo/commands/cmd_workflow_edit.py @@ -1,7 +1,8 @@ """Module describing the planemo ``workflow_edit`` command.""" -import click import os +import click + from planemo import options from planemo.cli import command_function from planemo.engine import ( diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py index b962a43f1..68f910149 100644 --- a/planemo/galaxy/profiles.py +++ b/planemo/galaxy/profiles.py @@ -9,9 +9,6 @@ from galaxy.util.commands import which -from planemo.config import ( - OptionSource, -) from planemo.database import create_database_source from planemo.galaxy.api import test_credentials_valid from .config import DATABASE_LOCATION_TEMPLATE @@ -132,12 +129,12 @@ def ensure_profile(ctx, profile_name, **kwds): def create_alias(ctx, alias, obj, profile_name, **kwds): profile_options, profile_options_path = _load_profile_to_json(ctx, profile_name) - + if profile_options.get('aliases'): profile_options['aliases'][alias] = obj else: # no aliases yet defined profile_options['aliases'] = {alias: obj} - + with open(profile_options_path, 'w') as f: json.dump(profile_options, f) From a7a88a61c2e41bf8628caaae8fe4c5beae6c1a19 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 20 Oct 2020 10:27:07 +0200 Subject: [PATCH 14/23] tidy up print statements --- planemo/galaxy/profiles.py | 7 ------- planemo/github_util.py | 1 - 2 files changed, 8 deletions(-) diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py index 68f910149..e64652414 100644 --- a/planemo/galaxy/profiles.py +++ b/planemo/galaxy/profiles.py @@ -162,7 +162,6 @@ def delete_alias(ctx, alias, profile_name, **kwds): def translate_alias(ctx, alias, profile_name): if not profile_name: return alias - # print(_load_profile_to_json(ctx, profile_name)) aliases = _load_profile_to_json(ctx, profile_name)[0].get('aliases', {}) return aliases.get(alias, alias) @@ -180,11 +179,6 @@ def _load_profile_to_json(ctx, profile_name): def _profile_options(ctx, profile_name, **kwds): profile_directory = _profile_directory(ctx, profile_name) profile_options = _read_profile_options(profile_directory) - # specified_engine_type = kwds.get("engine", "galaxy") - # profile_engine_type = profile_options["engine"] - # if specified_engine_type != profile_engine_type: - # if ctx.get_option_source("engine") == OptionSource.cli: - # raise Exception("Configured profile engine type [%s] does not match specified engine type [%s].") if profile_options["engine"] == "docker_galaxy": engine_options = dict( @@ -205,7 +199,6 @@ def _profile_options(ctx, profile_name, **kwds): ) profile_options.update(engine_options) profile_options["galaxy_brand"] = profile_name - # print(profile_options) return profile_options diff --git a/planemo/github_util.py b/planemo/github_util.py index 556e40691..a69a84d64 100644 --- a/planemo/github_util.py +++ b/planemo/github_util.py @@ -64,7 +64,6 @@ def pull_request(ctx, path, message=None, **kwds): cmd = [hub_path, "pull-request"] if message is not None: cmd.extend(["-m", message]) - print(cmd) communicate(cmd, env=hub_env) From eae8ca2ae77a674bda9204b936f171365be05ccf Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 20 Oct 2020 11:48:44 +0200 Subject: [PATCH 15/23] test fixes --- planemo/commands/cmd_delete_alias.py | 2 +- planemo/commands/cmd_list_alias.py | 2 +- planemo/commands/cmd_list_invocations.py | 2 +- planemo/commands/cmd_workflow_job_init.py | 6 +++--- planemo/commands/cmd_workflow_test_init.py | 14 +++++++------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/planemo/commands/cmd_delete_alias.py b/planemo/commands/cmd_delete_alias.py index d2a71d503..acb549c87 100644 --- a/planemo/commands/cmd_delete_alias.py +++ b/planemo/commands/cmd_delete_alias.py @@ -9,7 +9,7 @@ try: from tabulate import tabulate except ImportError: - tabulate = None + tabulate = None # type: ignore @click.command('delete_alias') diff --git a/planemo/commands/cmd_list_alias.py b/planemo/commands/cmd_list_alias.py index 1f08d3827..c37eeeec1 100644 --- a/planemo/commands/cmd_list_alias.py +++ b/planemo/commands/cmd_list_alias.py @@ -11,7 +11,7 @@ try: from tabulate import tabulate except ImportError: - tabulate = None + tabulate = None # type: ignore @click.command('list_alias') diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py index 28fe82bf6..e56556419 100644 --- a/planemo/commands/cmd_list_invocations.py +++ b/planemo/commands/cmd_list_invocations.py @@ -10,7 +10,7 @@ try: from tabulate import tabulate except ImportError: - tabulate = None + tabulate = None # type: ignore @click.command('list_invocations') diff --git a/planemo/commands/cmd_workflow_job_init.py b/planemo/commands/cmd_workflow_job_init.py index 407b485a3..9b5843d7e 100644 --- a/planemo/commands/cmd_workflow_job_init.py +++ b/planemo/commands/cmd_workflow_job_init.py @@ -13,7 +13,7 @@ @options.force_option() @options.workflow_output_artifact() @command_function -def cli(ctx, workflow_path, output=None, **kwds): +def cli(ctx, workflow_identifier, output=None, **kwds): """Initialize a Galaxy workflow job description for supplied workflow. Be sure to your lint your workflow with ``workflow_lint`` before calling this @@ -25,9 +25,9 @@ def cli(ctx, workflow_path, output=None, **kwds): as well so this command may be renamed to to job_init at something along those lines at some point. """ - job = job_template(workflow_path) + job = job_template(workflow_identifier) if output is None: - output = new_workflow_associated_path(workflow_path, suffix="job") + output = new_workflow_associated_path(workflow_identifier, suffix="job") if not can_write_to_path(output, **kwds): ctx.exit(1) with open(output, "w") as f_job: diff --git a/planemo/commands/cmd_workflow_test_init.py b/planemo/commands/cmd_workflow_test_init.py index 478fcf44c..20b0237fa 100644 --- a/planemo/commands/cmd_workflow_test_init.py +++ b/planemo/commands/cmd_workflow_test_init.py @@ -20,28 +20,28 @@ @options.workflow_output_artifact() @options.split_job_and_test() @command_function -def cli(ctx, workflow_path, output=None, split_test=False, **kwds): +def cli(ctx, workflow_identifier, output=None, split_test=False, **kwds): """Initialize a Galaxy workflow test description for supplied workflow. Be sure to your lint your workflow with ``workflow_lint`` before calling this to ensure inputs and outputs comply with best practices that make workflow testing easier. """ - path_basename = os.path.basename(workflow_path) - job = job_template(workflow_path) + path_basename = os.path.basename(workflow_identifier) + job = job_template(workflow_identifier) if output is None: - output = new_workflow_associated_path(workflow_path) - job_output = new_workflow_associated_path(workflow_path, suffix="job1") + output = new_workflow_associated_path(workflow_identifier) + job_output = new_workflow_associated_path(workflow_identifier, suffix="job1") if not can_write_to_path(output, **kwds): ctx.exit(1) test_description = [{ 'doc': 'Test outline for %s' % path_basename, 'job': job, - 'outputs': output_stubs_for_workflow(workflow_path), + 'outputs': output_stubs_for_workflow(workflow_identifier), }] if split_test: - job_output = new_workflow_associated_path(workflow_path, suffix="job1") + job_output = new_workflow_associated_path(workflow_identifier, suffix="job1") if not can_write_to_path(job_output, **kwds): ctx.exit(1) From c37a4d58944efefe3bafc0ab02c0ed5b2827cea3 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 20 Oct 2020 11:54:07 +0200 Subject: [PATCH 16/23] lint --- planemo/commands/cmd_list_invocations.py | 4 ++-- planemo/commands/cmd_run.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py index e56556419..fd34d7da1 100644 --- a/planemo/commands/cmd_list_invocations.py +++ b/planemo/commands/cmd_list_invocations.py @@ -45,8 +45,8 @@ def cli(ctx, workflow_id, **kwds): "Invocation ID": invocations.keys(), "Jobs status": [', '.join(['{}{} jobs {}\033[0m'.format(state_colors[k], v, k) for k, v in inv_states.items()] ) for inv_states in invocations.values()], - "Invocation report URL": ['{}/api/invocations/{}/report.pdf'.format(profile['galaxy_url'].strip('/' - ), inv_id) for inv_id in invocations] + "Invocation report URL": ['{}/api/invocations/{}/report.pdf'.format(profile['galaxy_url'].strip('/'), inv_id + ) for inv_id in invocations] }, headers="keys")) else: error("The tabulate package is not installed, invocations could not be listed correctly.") diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index c136df1b1..05c63e5c7 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -10,10 +10,11 @@ from planemo import options from planemo.cli import command_function from planemo.engine import engine_context +from planemo.galaxy.profiles import translate_alias from planemo.io import warn from planemo.runnable import for_id, for_path from planemo.tools import uri_to_path -from planemo.galaxy.profiles import translate_alias + @click.command('run') @options.required_runnable_arg() From bf05058dcded80ea9363bd62d73b75f4b6ea1da0 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 21 Oct 2020 14:20:11 +0200 Subject: [PATCH 17/23] add test --- planemo/commands/cmd_create_alias.py | 2 + planemo/commands/cmd_list_invocations.py | 2 + planemo/galaxy/profiles.py | 18 ++++--- tests/test_external_galaxy_commands.py | 66 ++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 tests/test_external_galaxy_commands.py diff --git a/planemo/commands/cmd_create_alias.py b/planemo/commands/cmd_create_alias.py index fb762a9ef..c255b57a4 100644 --- a/planemo/commands/cmd_create_alias.py +++ b/planemo/commands/cmd_create_alias.py @@ -4,6 +4,7 @@ from planemo import options from planemo.cli import command_function from planemo.galaxy import profiles +from planemo.io import info try: import namesgenerator @@ -31,5 +32,6 @@ def cli(ctx, alias, obj, profile, **kwds): alias = namesgenerator.get_random_name() exit_code = profiles.create_alias(ctx, alias, obj, profile) + info("Alias {} created.".format(alias)) ctx.exit(exit_code) return diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py index fd34d7da1..c8205bb88 100644 --- a/planemo/commands/cmd_list_invocations.py +++ b/planemo/commands/cmd_list_invocations.py @@ -5,6 +5,7 @@ from planemo.cli import command_function from planemo.galaxy import profiles from planemo.galaxy.api import get_invocations +from planemo.galaxy.profiles import translate_alias from planemo.io import error, info try: @@ -24,6 +25,7 @@ def cli(ctx, workflow_id, **kwds): """ Get a list of invocations for a particular workflow ID or alias. """ + workflow_id = translate_alias(ctx, workflow_id, kwds.get('profile')) info("Looking for invocations for workflow {}...".format(workflow_id)) workflow_id = profiles.translate_alias(ctx, workflow_id, kwds.get('profile')) profile = profiles.ensure_profile(ctx, kwds.get('profile')) diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py index e64652414..2aaaa52a3 100644 --- a/planemo/galaxy/profiles.py +++ b/planemo/galaxy/profiles.py @@ -32,14 +32,16 @@ def delete_profile(ctx, profile_name, **kwds): """Delete profile with the specified name.""" profile_directory = _profile_directory(ctx, profile_name) profile_options = _read_profile_options(profile_directory) - database_type = profile_options.get("database_type") - kwds["database_type"] = database_type - if database_type != "sqlite": - database_source = create_database_source(**kwds) - database_identifier = _profile_to_database_identifier(profile_name) - database_source.delete_database( - database_identifier, - ) + profile_options, profile_options_path = _load_profile_to_json(ctx, profile_name) + if profile_options["engine"] != 'external_galaxy': + database_type = profile_options.get("database_type") + kwds["database_type"] = database_type + if database_type != "sqlite": + database_source = create_database_source(**kwds) + database_identifier = _profile_to_database_identifier(profile_name) + database_source.delete_database( + database_identifier, + ) shutil.rmtree(profile_directory) diff --git a/tests/test_external_galaxy_commands.py b/tests/test_external_galaxy_commands.py new file mode 100644 index 000000000..9ee9c7b91 --- /dev/null +++ b/tests/test_external_galaxy_commands.py @@ -0,0 +1,66 @@ +"""Tests for planemo commands relating to external Galaxy instances +""" +import os + +from planemo import cli +from planemo.engine import engine_context +from planemo.runnable import for_path +from .test_utils import ( + CliTestCase, + PROJECT_TEMPLATES_DIR, + TEST_DATA_DIR, +) + + +class ExternalGalaxyCommandsTestCase(CliTestCase): + def test_plain_init(self): + ctx = cli.PlanemoCliContext() + ctx.planemo_directory = "/tmp/planemo-test-workspace" + cat_tool = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "cat.xml") + test_workflow_path = os.path.join(TEST_DATA_DIR, 'wf2.ga') + + with engine_context(ctx, extra_tools=(cat_tool,)) as galaxy_engine: + with galaxy_engine.ensure_runnables_served([for_path(test_workflow_path)]) as config: + wfid = config.workflow_id(test_workflow_path) + + # commands to test + profile_list_cmd = ["profile_list"] + profile_create_cmd = ["profile_create", "test_ext_profile", "--galaxy_url", config.galaxy_url, + "--galaxy_user_key", config.user_api_key] + alias_create_cmd = ["create_alias", wfid, "--alias", "test_wf_alias", "--profile", "test_ext_profile"] + alias_list_cmd = ["list_alias", "--profile", "test_ext_profile"] + alias_delete_cmd = ["delete_alias", "--alias", "test_wf_alias", "--profile", "test_ext_profile"] + profile_delete_cmd = ["profile_delete", "test_ext_profile"] + run_cmd = ["run", "test_wf_alias", os.path.join(TEST_DATA_DIR, "wf2-job.yml"), "--profile", "test_ext_profile"] + list_invocs_cmd = ["list_invocations", "test_wf_alias", "--profile", "test_ext_profile"] + + # test alias and profile creation + result = self._check_exit_code(profile_list_cmd) + assert '0 configured profiles are available.' in result.output + result = self._check_exit_code(profile_create_cmd) + assert 'Profile [test_ext_profile] created' in result.output + result = self._check_exit_code(profile_list_cmd) + assert 'test_ext_profile' in result.output + result = self._check_exit_code(alias_create_cmd) + assert 'Alias test_wf_alias created.' in result.output + result = self._check_exit_code(alias_list_cmd) + assert 'test_wf_alias' in result.output + assert wfid in result.output + assert '1 aliases were found for profile test_ext_profile.' in result.output + + # test WF execution (from wfid) using created profile and alias + result = self._check_exit_code(run_cmd) + assert 'Run failed' not in result.output + result = self._check_exit_code(list_invocs_cmd) + assert '1 invocations found.' in result.output + assert '1 jobs ok' in result.output + + # test alias and profile deletion + result = self._check_exit_code(alias_delete_cmd) + assert 'Alias test_wf_alias was successfully deleted from profile test_ext_profile' in result.output + result = self._check_exit_code(alias_list_cmd) + assert '0 aliases were found for profile test_ext_profile.' in result.output + result = self._check_exit_code(profile_delete_cmd) + assert 'Profile deleted.' in result.output + result = self._check_exit_code(profile_list_cmd) + assert '0 configured profiles are available.' in result.output From de6253e8bc88bad3c9a3872ee7d334044f926e4b Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 21 Oct 2020 14:59:01 +0200 Subject: [PATCH 18/23] small change to test --- tests/test_external_galaxy_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_external_galaxy_commands.py b/tests/test_external_galaxy_commands.py index 9ee9c7b91..2f8d5c00d 100644 --- a/tests/test_external_galaxy_commands.py +++ b/tests/test_external_galaxy_commands.py @@ -53,7 +53,7 @@ def test_plain_init(self): assert 'Run failed' not in result.output result = self._check_exit_code(list_invocs_cmd) assert '1 invocations found.' in result.output - assert '1 jobs ok' in result.output + assert '1 jobs ok' in result.output or "{'ok': 1}" in result.output # so it passes regardless if tabulate is installed or not # test alias and profile deletion result = self._check_exit_code(alias_delete_cmd) From 89541eb970637cfd1dab6b131c79edb30830d7c8 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 21 Oct 2020 16:14:25 +0200 Subject: [PATCH 19/23] small change to test 2 --- tests/test_external_galaxy_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_external_galaxy_commands.py b/tests/test_external_galaxy_commands.py index 2f8d5c00d..79679daa3 100644 --- a/tests/test_external_galaxy_commands.py +++ b/tests/test_external_galaxy_commands.py @@ -36,7 +36,7 @@ def test_plain_init(self): # test alias and profile creation result = self._check_exit_code(profile_list_cmd) - assert '0 configured profiles are available.' in result.output + assert 'test_ext_profile' not in result.output result = self._check_exit_code(profile_create_cmd) assert 'Profile [test_ext_profile] created' in result.output result = self._check_exit_code(profile_list_cmd) @@ -63,4 +63,4 @@ def test_plain_init(self): result = self._check_exit_code(profile_delete_cmd) assert 'Profile deleted.' in result.output result = self._check_exit_code(profile_list_cmd) - assert '0 configured profiles are available.' in result.output + assert 'test_ext_profile' not in result.output From 5fe51991712748a75daedf377a85dba4fa94def7 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 21 Oct 2020 18:03:14 +0200 Subject: [PATCH 20/23] improve list_invocations command --- planemo/commands/cmd_list_invocations.py | 17 ++++++++++------- planemo/galaxy/api.py | 5 ++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/planemo/commands/cmd_list_invocations.py b/planemo/commands/cmd_list_invocations.py index c8205bb88..62e36a811 100644 --- a/planemo/commands/cmd_list_invocations.py +++ b/planemo/commands/cmd_list_invocations.py @@ -1,4 +1,6 @@ -"""Module describing the planemo ``create_alias`` command.""" +"""Module describing the planemo ``list_invocations`` command.""" +import json + import click from planemo import options @@ -31,7 +33,6 @@ def cli(ctx, workflow_id, **kwds): profile = profiles.ensure_profile(ctx, kwds.get('profile')) invocations = get_invocations(url=profile['galaxy_url'], key=profile['galaxy_admin_key'] or profile['galaxy_user_key'], workflow_id=workflow_id) - if tabulate: state_colors = { 'ok': '\033[92m', # green @@ -45,14 +46,16 @@ def cli(ctx, workflow_id, **kwds): } print(tabulate({ "Invocation ID": invocations.keys(), - "Jobs status": [', '.join(['{}{} jobs {}\033[0m'.format(state_colors[k], v, k) for k, v in inv_states.items()] - ) for inv_states in invocations.values()], - "Invocation report URL": ['{}/api/invocations/{}/report.pdf'.format(profile['galaxy_url'].strip('/'), inv_id - ) for inv_id in invocations] + "Jobs status": [', '.join(['{}{} jobs {}\033[0m'.format(state_colors[k], v, k) for k, v in inv['states'].items()] + ) for inv in invocations.values()], + "Invocation report URL": ['{}/workflows/invocations/report?id={}'.format(profile['galaxy_url'].strip('/'), inv_id + ) for inv_id in invocations], + "History URL": ['{}/histories/view?id={}'.format(profile['galaxy_url'].strip('/'), invocations[inv_id]['history_id'] + ) for inv_id in invocations] }, headers="keys")) else: error("The tabulate package is not installed, invocations could not be listed correctly.") - print(invocations) + print(json.dumps(invocations, indent=4, sort_keys=True)) info("{} invocations found.".format(len(invocations))) return diff --git a/planemo/galaxy/api.py b/planemo/galaxy/api.py index ea1e807c7..f2cd822c8 100644 --- a/planemo/galaxy/api.py +++ b/planemo/galaxy/api.py @@ -114,7 +114,10 @@ def summarize_history(ctx, gi, history_id): def get_invocations(url, key, workflow_id): inv_gi = gi(None, url, key) invocations = inv_gi.workflows.get_invocations(workflow_id) - return {invocation['id']: inv_gi.invocations.get_invocation_summary(invocation['id'])['states'] for invocation in invocations} + return {invocation['id']: { + 'states': inv_gi.invocations.get_invocation_summary(invocation['id'])['states'], + 'history_id': invocation['history_id']} + for invocation in invocations} def _format_for_summary(blob, empty_message, prefix="| "): From abcfb81608f40a45e2f754a897686ed511222f44 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 21 Oct 2020 18:16:44 +0200 Subject: [PATCH 21/23] small fix to list_invocations test --- tests/test_external_galaxy_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_external_galaxy_commands.py b/tests/test_external_galaxy_commands.py index 79679daa3..b5c8fad29 100644 --- a/tests/test_external_galaxy_commands.py +++ b/tests/test_external_galaxy_commands.py @@ -53,7 +53,7 @@ def test_plain_init(self): assert 'Run failed' not in result.output result = self._check_exit_code(list_invocs_cmd) assert '1 invocations found.' in result.output - assert '1 jobs ok' in result.output or "{'ok': 1}" in result.output # so it passes regardless if tabulate is installed or not + assert '1 jobs ok' in result.output or '"ok": 1' in result.output # so it passes regardless if tabulate is installed or not # test alias and profile deletion result = self._check_exit_code(alias_delete_cmd) From 693da3525eb6f5cc3e50f9f33d83936389085ab2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Nov 2020 13:36:46 -0500 Subject: [PATCH 22/23] Runnable URIs instead of IDs. I prefer these abstractions. --- planemo/commands/cmd_workflow_edit.py | 2 +- planemo/galaxy/activity.py | 11 ++++------ planemo/galaxy/config.py | 9 ++++++++- planemo/galaxy/workflows.py | 19 +++++++++++++++--- planemo/runnable.py | 29 ++++++++++++++++++--------- 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/planemo/commands/cmd_workflow_edit.py b/planemo/commands/cmd_workflow_edit.py index 12afca024..7d8904f7a 100644 --- a/planemo/commands/cmd_workflow_edit.py +++ b/planemo/commands/cmd_workflow_edit.py @@ -36,7 +36,7 @@ def cli(ctx, workflow_identifier, output=None, force=False, **kwds): with engine_context(ctx, **kwds) as galaxy_engine: with galaxy_engine.ensure_runnables_served([runnable]) as config: - workflow_id = runnable.workflow_id or config.workflow_id(runnable.path) + workflow_id = config.workflow_id_for_runnable(runnable) url = "%s/workflow/editor?id=%s" % (config.galaxy_url, workflow_id) click.launch(url) if kwds["engine"] != "external_galaxy": diff --git a/planemo/galaxy/activity.py b/planemo/galaxy/activity.py index 77dc66051..03c9e47dc 100644 --- a/planemo/galaxy/activity.py +++ b/planemo/galaxy/activity.py @@ -29,6 +29,7 @@ from six.moves.urllib.parse import urljoin from planemo.galaxy.api import summarize_history +from planemo.galaxy.workflows import remote_runnable_to_workflow_id from planemo.io import wait_on from planemo.runnable import ( ErrorRunResponse, @@ -153,12 +154,8 @@ def _execute(ctx, config, runnable, job_path, **kwds): summarize_history(ctx, user_gi, history_id) elif runnable.type in [RunnableType.galaxy_workflow, RunnableType.cwl_workflow]: response_class = GalaxyWorkflowRunResponse - if runnable.workflow_id: - runnable.workflow_dict = get_dict_from_workflow(user_gi, runnable.workflow_id) - workflow_id = runnable.workflow_id - else: - workflow_id = config.workflow_id(runnable.path) - ctx.vlog("Found Galaxy workflow ID [%s] for path [%s]" % (workflow_id, runnable.path)) + workflow_id = config.workflow_id_for_runnable(runnable) + ctx.vlog("Found Galaxy workflow ID [%s] for URI [%s]" % (workflow_id, runnable.uri)) # TODO: Use the following when BioBlend 0.14 is released # invocation = user_gi.worklfows.invoke_workflow( # workflow_id, @@ -348,7 +345,7 @@ def get_dataset(dataset_details, filename=None): ctx.vlog("collecting outputs to directory %s" % output_directory) - for runnable_output in get_outputs(self._runnable): + for runnable_output in get_outputs(self._runnable, gi=self._user_gi): output_id = runnable_output.get_id() if not output_id: ctx.vlog("Workflow output identified without an ID (label), skipping") diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 44a519635..fee405d79 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -757,7 +757,7 @@ def ready(): def install_workflows(self): for runnable in self.runnables: - if runnable.type.name in ["galaxy_workflow", "cwl_workflow"] and not runnable.workflow_id: + if runnable.type.name in ["galaxy_workflow", "cwl_workflow"] and not runnable.is_remote_workflow_uri: self._install_workflow(runnable) def _install_workflow(self, runnable): @@ -778,6 +778,13 @@ def _install_workflow(self, runnable): ) self._workflow_ids[runnable.path] = workflow["id"] + def workflow_id_for_runnable(self, runnable): + if runnable.is_remote_workflow_uri: + workflow_id = remote_runnable_to_workflow_id(runnable) + else: + workflow_id = self.workflow_id(runnable.path) + return workflow_id + def workflow_id(self, path): return self._workflow_ids[path] diff --git a/planemo/galaxy/workflows.py b/planemo/galaxy/workflows.py index 8cde4d648..378e19830 100644 --- a/planemo/galaxy/workflows.py +++ b/planemo/galaxy/workflows.py @@ -14,6 +14,7 @@ from planemo.io import warn FAILED_REPOSITORIES_MESSAGE = "Failed to install one or more repositories." +GALAXY_WORKFLOWS_PREFIX = "gxprofile://workflows/" def load_shed_repos(runnable): @@ -106,10 +107,18 @@ def register_tool_ids(tool_ids, workflow): WorkflowOutput = namedtuple("WorkflowOutput", ["order_index", "output_name", "label"]) -def describe_outputs(runnable): +def remote_runnable_to_workflow_id(runnable): + assert runnable.is_remote_workflow_uri + workflow_id = runnable.uri[len("GALAXY_WORKFLOWS_PREFIX"):] + return workflow_id + + +def describe_outputs(runnable, gi=None): """Return a list of :class:`WorkflowOutput` objects for target workflow.""" - if runnable.workflow_dict: - workflow = runnable.workflow_dict + if runnable.path.startswith(GALAXY_WORKFLOWS_PREFIX): + workflow_id = remote_runnable_to_workflow_id(runnable) + assert gi is not None + workflow = get_dict_from_workflow(gi, workflow_id) else: workflow = _raw_dict(runnable.path) @@ -226,6 +235,10 @@ def new_workflow_associated_path(workflow_path, suffix="tests"): return base + sep + suffix + "." + ext +def get_dict_from_workflow(gi, workflow_id): + return gi.workflows.export_workflow_dict(workflow_id) + + __all__ = ( "import_workflow", "describe_outputs", diff --git a/planemo/runnable.py b/planemo/runnable.py index c3fd44140..f3c94ee6f 100644 --- a/planemo/runnable.py +++ b/planemo/runnable.py @@ -24,7 +24,7 @@ ) from planemo.exit_codes import EXIT_CODE_UNKNOWN_FILE_TYPE, ExitCodeException -from planemo.galaxy.workflows import describe_outputs +from planemo.galaxy.workflows import describe_outputs, GALAXY_WORKFLOWS_PREFIX from planemo.io import error from planemo.test import check_output, for_collections @@ -64,15 +64,20 @@ def is_galaxy_artifact(runnable_type): return "galaxy" in runnable_type.name -_Runnable = collections.namedtuple("Runnable", ["path", "type"]) +_Runnable = collections.namedtuple("Runnable", ["uri", "type"]) class Runnable(_Runnable): """Abstraction describing tools and workflows.""" - def __init__(self, path, type): - self.workflow_id = None - self.workflow_dict = None + @property + def path(self): + assert not self.is_remote_workflow_uri + return self.uri + + @property + def is_remote_workflow_uri(self): + return self.uri.startswith(GALAXY_WORKFLOWS_PREFIX) @property def test_data_search_path(self): @@ -177,8 +182,8 @@ def for_paths(paths, temp_path=None): def for_id(runnable_id): """Produce a class:`Runnable` for supplied Galaxy workflow ID.""" - runnable = Runnable(None, RunnableType.galaxy_workflow) - runnable.workflow_id = runnable_id + uri = GALAXY_WORKFLOWS_PREFIX + runnable_id + runnable = Runnable(uri, RunnableType.galaxy_workflow) return runnable @@ -435,8 +440,12 @@ def _tests_path(runnable): return None -def get_outputs(runnable): - """Return a list of :class:`RunnableOutput` objects for this runnable.""" +def get_outputs(runnable, gi=None): + """Return a list of :class:`RunnableOutput` objects for this runnable. + + Supply bioblend user Galaxy instance object (as gi) if additional context + needed to resolve workflow details. + """ if not runnable.is_single_artifact: raise NotImplementedError("Cannot generate outputs for a directory.") if runnable.type in [RunnableType.galaxy_tool, RunnableType.cwl_tool]: @@ -446,7 +455,7 @@ def get_outputs(runnable): outputs = [ToolOutput(o) for o in output_datasets.values()] return outputs elif runnable.type == RunnableType.galaxy_workflow: - workflow_outputs = describe_outputs(runnable) + workflow_outputs = describe_outputs(runnable, gi=gi) return [GalaxyWorkflowOutput(o) for o in workflow_outputs] elif runnable.type == RunnableType.cwl_workflow: workflow = workflow_proxy(runnable.path, strict_cwl_validation=False) From bcc02f2388e2fa77b139aa07ff1033f16d2c8cf4 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Fri, 20 Nov 2020 12:32:33 +0100 Subject: [PATCH 23/23] small fixes --- planemo/galaxy/activity.py | 1 - planemo/galaxy/config.py | 1 + planemo/galaxy/workflows.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/planemo/galaxy/activity.py b/planemo/galaxy/activity.py index 03c9e47dc..e0fe708d5 100644 --- a/planemo/galaxy/activity.py +++ b/planemo/galaxy/activity.py @@ -29,7 +29,6 @@ from six.moves.urllib.parse import urljoin from planemo.galaxy.api import summarize_history -from planemo.galaxy.workflows import remote_runnable_to_workflow_id from planemo.io import wait_on from planemo.runnable import ( ErrorRunResponse, diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index fee405d79..43b6b94de 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -24,6 +24,7 @@ from planemo.config import OptionSource from planemo.deps import ensure_dependency_resolvers_conf_configured from planemo.docker import docker_host_args +from planemo.galaxy.workflows import remote_runnable_to_workflow_id from planemo.io import ( communicate, kill_pid_file, diff --git a/planemo/galaxy/workflows.py b/planemo/galaxy/workflows.py index 378e19830..78ed1066e 100644 --- a/planemo/galaxy/workflows.py +++ b/planemo/galaxy/workflows.py @@ -109,13 +109,13 @@ def register_tool_ids(tool_ids, workflow): def remote_runnable_to_workflow_id(runnable): assert runnable.is_remote_workflow_uri - workflow_id = runnable.uri[len("GALAXY_WORKFLOWS_PREFIX"):] + workflow_id = runnable.uri[len(GALAXY_WORKFLOWS_PREFIX):] return workflow_id def describe_outputs(runnable, gi=None): """Return a list of :class:`WorkflowOutput` objects for target workflow.""" - if runnable.path.startswith(GALAXY_WORKFLOWS_PREFIX): + if runnable.uri.startswith(GALAXY_WORKFLOWS_PREFIX): workflow_id = remote_runnable_to_workflow_id(runnable) assert gi is not None workflow = get_dict_from_workflow(gi, workflow_id)