From 6851af38d3e27355f466b804fd32f11ea0620cbb Mon Sep 17 00:00:00 2001 From: Delphine-L Date: Thu, 29 Feb 2024 09:40:05 -0600 Subject: [PATCH 1/8] Add the option output_metadata --- planemo/commands/cmd_run.py | 6 ++++++ planemo/options.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index 5700a0768..183931fa0 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -27,6 +27,7 @@ @options.run_history_tags_option() @options.run_output_directory_option() @options.run_output_json_option() +@options.run_output_metadata_option() @options.run_download_outputs_option() @options.engine_options() @options.test_options() @@ -50,6 +51,11 @@ def cli(ctx, runnable_identifier, job_path, **kwds): kwds["engine"] = "galaxy" with engine_context(ctx, **kwds) as engine: run_result = engine.run([runnable], [job_path])[0] + run_infos ={"history_id": run_result._history_id, "invocation_id": run_result._invocation_id, "workflow_id": run_result._workflow_id} + output_metadata = kwds.get("output_metadata", None) + if output_metadata: + with open(output_metadata, "w") as f: + json.dump(run_infos, f) if not run_result.was_successful: warn("Run failed [%s]" % unicodify(run_result)) diff --git a/planemo/options.py b/planemo/options.py index c17069787..0b7ad8fed 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -273,6 +273,18 @@ def run_output_directory_option(): help=("Where to store outputs of a 'run' task."), ) +def run_output_metadata_option(): + return planemo_option( + "output_metadata", + "--output_metadata", + type=click.Path( + file_okay=True, + dir_okay=False, + resolve_path=True, + ), + default=None, + help=("Where to store JSON dictionary describing the metadata of " "a 'run' task."), + ) def run_output_json_option(): return planemo_option( From b50f855e6721ac42deb6cb4c4e4f2cd3ea8d5402 Mon Sep 17 00:00:00 2001 From: Delphine-L Date: Thu, 29 Feb 2024 14:43:15 -0600 Subject: [PATCH 2/8] add description of the output_metadata option --- docs/commands/run.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/commands/run.rst b/docs/commands/run.rst index 94c1b4bee..fc7a3f562 100644 --- a/docs/commands/run.rst +++ b/docs/commands/run.rst @@ -168,6 +168,8 @@ Planemo command for running tools and jobs. Where to store outputs of a 'run' task. --output_json FILE Where to store JSON dictionary describing outputs of a 'run' task. + --output_metadata FILE Where to store JSON dictionary describing + the metadata of a 'run' task. --download_outputs / --no_download_outputs After tool or workflow runs are complete, download the output files to the location From c41091aab19f1d096eb333096cf1019e4d3a719e Mon Sep 17 00:00:00 2001 From: Delphine-L Date: Tue, 5 Mar 2024 11:52:30 -0500 Subject: [PATCH 3/8] correct linting errors --- planemo/commands/cmd_run.py | 6 +++++- planemo/options.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index 183931fa0..3da702940 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -51,7 +51,11 @@ def cli(ctx, runnable_identifier, job_path, **kwds): kwds["engine"] = "galaxy" with engine_context(ctx, **kwds) as engine: run_result = engine.run([runnable], [job_path])[0] - run_infos ={"history_id": run_result._history_id, "invocation_id": run_result._invocation_id, "workflow_id": run_result._workflow_id} + run_infos = { + "history_id": run_result._history_id, + "invocation_id": run_result._invocation_id, + "workflow_id": run_result._workflow_id, + } output_metadata = kwds.get("output_metadata", None) if output_metadata: with open(output_metadata, "w") as f: diff --git a/planemo/options.py b/planemo/options.py index 0b7ad8fed..cea6f0e97 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -273,6 +273,7 @@ def run_output_directory_option(): help=("Where to store outputs of a 'run' task."), ) + def run_output_metadata_option(): return planemo_option( "output_metadata", @@ -286,6 +287,7 @@ def run_output_metadata_option(): help=("Where to store JSON dictionary describing the metadata of " "a 'run' task."), ) + def run_output_json_option(): return planemo_option( "output_json", From f594f79fb31f54285aef69dae27336be976c329f Mon Sep 17 00:00:00 2001 From: Delphine-L Date: Tue, 5 Mar 2024 13:07:19 -0500 Subject: [PATCH 4/8] create run info only for galaxy external and galaxy engines --- planemo/commands/cmd_run.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index 3da702940..cab2a0858 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -51,15 +51,16 @@ def cli(ctx, runnable_identifier, job_path, **kwds): kwds["engine"] = "galaxy" with engine_context(ctx, **kwds) as engine: run_result = engine.run([runnable], [job_path])[0] - run_infos = { - "history_id": run_result._history_id, - "invocation_id": run_result._invocation_id, - "workflow_id": run_result._workflow_id, - } - output_metadata = kwds.get("output_metadata", None) - if output_metadata: - with open(output_metadata, "w") as f: - json.dump(run_infos, f) + if kwds["engine"] == "galaxy" or kwds["engine"] == "external_galaxy": + run_infos = { + "history_id": run_result._history_id, + "invocation_id": run_result._invocation_id, + "workflow_id": run_result._workflow_id, + } + output_metadata = kwds.get("output_metadata", None) + if output_metadata: + with open(output_metadata, "w") as f: + json.dump(run_infos, f) if not run_result.was_successful: warn("Run failed [%s]" % unicodify(run_result)) From 5a0eb3185a02f71f6312b680b67e8bf0deae7b8d Mon Sep 17 00:00:00 2001 From: Delphine-L Date: Wed, 6 Mar 2024 07:15:16 -0500 Subject: [PATCH 5/8] move the creation of object inside the condition --- planemo/commands/cmd_run.py | 10 +++++----- planemo/options.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index cab2a0858..bd35ee08c 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -52,13 +52,13 @@ def cli(ctx, runnable_identifier, job_path, **kwds): with engine_context(ctx, **kwds) as engine: run_result = engine.run([runnable], [job_path])[0] if kwds["engine"] == "galaxy" or kwds["engine"] == "external_galaxy": - run_infos = { - "history_id": run_result._history_id, - "invocation_id": run_result._invocation_id, - "workflow_id": run_result._workflow_id, - } output_metadata = kwds.get("output_metadata", None) if output_metadata: + run_infos = { + "history_id": run_result._history_id, + "invocation_id": run_result._invocation_id, + "workflow_id": run_result._workflow_id, + } with open(output_metadata, "w") as f: json.dump(run_infos, f) diff --git a/planemo/options.py b/planemo/options.py index cea6f0e97..694237ae4 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -284,7 +284,7 @@ def run_output_metadata_option(): resolve_path=True, ), default=None, - help=("Where to store JSON dictionary describing the metadata of " "a 'run' task."), + help=("Where to store JSON dictionary describing the metadata of " "a workflow 'run' task."), ) From a26d3bb4851f64a992e7d60e6ae4b68d9a7426ed Mon Sep 17 00:00:00 2001 From: Delphine Lariviere Date: Wed, 24 Apr 2024 08:17:18 -0400 Subject: [PATCH 6/8] don't wait for end of invocation to write file, and add tests --- planemo/commands/cmd_run.py | 10 ---- planemo/galaxy/activity.py | 11 ++++ tests/data/first_lines.ga | 100 ++++++++++++++++++++++++++++++++++++ tests/data/first_lines.yml | 3 ++ tests/test_run.py | 21 ++++++++ 5 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 tests/data/first_lines.ga create mode 100644 tests/data/first_lines.yml diff --git a/planemo/commands/cmd_run.py b/planemo/commands/cmd_run.py index bd35ee08c..e0306ba58 100644 --- a/planemo/commands/cmd_run.py +++ b/planemo/commands/cmd_run.py @@ -51,16 +51,6 @@ def cli(ctx, runnable_identifier, job_path, **kwds): kwds["engine"] = "galaxy" with engine_context(ctx, **kwds) as engine: run_result = engine.run([runnable], [job_path])[0] - if kwds["engine"] == "galaxy" or kwds["engine"] == "external_galaxy": - output_metadata = kwds.get("output_metadata", None) - if output_metadata: - run_infos = { - "history_id": run_result._history_id, - "invocation_id": run_result._invocation_id, - "workflow_id": run_result._workflow_id, - } - with open(output_metadata, "w") as f: - json.dump(run_infos, f) if not run_result.was_successful: warn("Run failed [%s]" % unicodify(run_result)) diff --git a/planemo/galaxy/activity.py b/planemo/galaxy/activity.py index 0eafdcffd..2964f2f2f 100644 --- a/planemo/galaxy/activity.py +++ b/planemo/galaxy/activity.py @@ -1,5 +1,6 @@ """Module provides generic interface to running Galaxy tools and workflows.""" +import json import os import sys import tempfile @@ -216,6 +217,16 @@ def _execute( # noqa C901 start_datetime=start_datetime, log=log_contents_str(config), ) + if kwds["engine"] == "galaxy" or kwds["engine"] == "external_galaxy": + output_metadata = kwds.get("output_metadata", None) + if output_metadata: + run_infos = { + "history_id": history_id, + "invocation_id": invocation["id"], + "workflow_id": workflow_id, + } + with open(output_metadata, "w") as f: + json.dump(run_infos, f) else: raise NotImplementedError() diff --git a/tests/data/first_lines.ga b/tests/data/first_lines.ga new file mode 100644 index 000000000..e08e948cf --- /dev/null +++ b/tests/data/first_lines.ga @@ -0,0 +1,100 @@ +{ + "a_galaxy_workflow": "true", + "annotation": "", + "comments": [], + "format-version": "0.1", + "name": "Testing", + "report": { + "markdown": "\n# Workflow Execution Report\n\n## Workflow Inputs\n```galaxy\ninvocation_inputs()\n```\n\n## Workflow Outputs\n```galaxy\ninvocation_outputs()\n```\n\r\n```galaxy\nhistory_dataset_peek(output=\"Truncated file\")\n```\r\n\n\n## Workflow\n```galaxy\nworkflow_display()\n```\n" + }, + "steps": { + "0": { + "annotation": "", + "content_id": null, + "errors": null, + "id": 0, + "input_connections": {}, + "inputs": [ + { + "description": "", + "name": "File to head" + } + ], + "label": "File to head", + "name": "Input dataset", + "outputs": [], + "position": { + "left": 0, + "top": 0 + }, + "tool_id": null, + "tool_state": "{\"optional\": false, \"tag\": null}", + "tool_version": null, + "type": "data_input", + "uuid": "6cada1bd-e96e-43d2-a41b-2f396c9ad2ca", + "when": null, + "workflow_outputs": [] + }, + "1": { + "annotation": "", + "content_id": "toolshed.g2.bx.psu.edu/repos/bgruening/text_processing/tp_head_tool/1.1.0", + "errors": null, + "id": 1, + "input_connections": { + "infile": { + "id": 0, + "output_name": "output" + } + }, + "inputs": [ + { + "description": "runtime parameter for tool Select first", + "name": "infile" + } + ], + "label": null, + "name": "Select first", + "outputs": [ + { + "name": "outfile", + "type": "input" + } + ], + "position": { + "left": 296.265625, + "top": 57.5703125 + }, + "post_job_actions": { + "RenameDatasetActionoutfile": { + "action_arguments": { + "newname": "Truncated_file" + }, + "action_type": "RenameDatasetAction", + "output_name": "outfile" + } + }, + "tool_id": "toolshed.g2.bx.psu.edu/repos/bgruening/text_processing/tp_head_tool/1.1.0", + "tool_shed_repository": { + "changeset_revision": "ddf54b12c295", + "name": "text_processing", + "owner": "bgruening", + "tool_shed": "toolshed.g2.bx.psu.edu" + }, + "tool_state": "{\"complement\": \"\", \"count\": \"3\", \"infile\": {\"__class__\": \"ConnectedValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}", + "tool_version": "1.1.0", + "type": "tool", + "uuid": "97a55f13-2d30-4ecd-a3fb-d8d433d3b3bb", + "when": null, + "workflow_outputs": [ + { + "label": "Truncated file", + "output_name": "outfile", + "uuid": "61afcb1c-d26e-4a98-988f-a73fbb074d0c" + } + ] + } + }, + "tags": [], + "uuid": "1649a361-0381-444c-bdb6-b099609f7f62", + "version": 4 +} \ No newline at end of file diff --git a/tests/data/first_lines.yml b/tests/data/first_lines.yml new file mode 100644 index 000000000..a5dd1a03f --- /dev/null +++ b/tests/data/first_lines.yml @@ -0,0 +1,3 @@ +File to head: + class: File + path: gxfiles://genomeark/species/Taeniopygia_guttata/bTaeGut2/assembly_vgp_trio_2.0/evaluation/busco/bTaeGut2_trio.dip.hap1_busco_shortsummary.txt diff --git a/tests/test_run.py b/tests/test_run.py index 51ff42271..d00e91bf1 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -115,3 +115,24 @@ def test_run_output_directory(self): assert os.path.exists(output_path) with open(output_path) as fh: assert fh.read().startswith(" 16 198 1111") + + @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + def test_run_metadata(self): + with self._isolate() as f: + workflow_path = os.path.join(TEST_DATA_DIR, "first_lines.ga") + job_path = os.path.join(TEST_DATA_DIR, "first_lines.yml") + info_path = os.path.join(f, "run_info.json") + test_cmd = [ + "--verbose", + "run", + workflow_path, + job_path, + "--no_dependency_resolution", + "--galaxy_branch", + target_galaxy_branch(), + "--no_wait", + "--output_metadata", + info_path, + ] + self._check_exit_code(test_cmd) + assert os.path.exists(os.path.join(f, "run_info.json")) From 7dbe54cdf478f9a5b8d35bb71fae96f9f49eaf6f Mon Sep 17 00:00:00 2001 From: Delphine Lariviere Date: Wed, 24 Apr 2024 09:39:15 -0400 Subject: [PATCH 7/8] Add --test-data to the test command line --- tests/test_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_run.py b/tests/test_run.py index d00e91bf1..e62361e3d 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -133,6 +133,8 @@ def test_run_metadata(self): "--no_wait", "--output_metadata", info_path, + "--test_data", + TEST_DATA_DIR, ] self._check_exit_code(test_cmd) assert os.path.exists(os.path.join(f, "run_info.json")) From 9a5c9026a262e73df8b82424bc308be761fe35c6 Mon Sep 17 00:00:00 2001 From: Delphine Lariviere Date: Wed, 24 Apr 2024 12:25:30 -0400 Subject: [PATCH 8/8] change test file to local one --- tests/data/first_lines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/first_lines.yml b/tests/data/first_lines.yml index a5dd1a03f..4c428c501 100644 --- a/tests/data/first_lines.yml +++ b/tests/data/first_lines.yml @@ -1,3 +1,3 @@ File to head: class: File - path: gxfiles://genomeark/species/Taeniopygia_guttata/bTaeGut2/assembly_vgp_trio_2.0/evaluation/busco/bTaeGut2_trio.dip.hap1_busco_shortsummary.txt + path: hello.txt