From 2fc1b64742a03cb36594ad51c6cf4bbb824cabbe Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 10 Nov 2023 15:12:37 +0100 Subject: [PATCH 1/2] Fix copying metadata to copied job outputs Fixes https://github.com/galaxyproject/galaxy/issues/17003 --- lib/galaxy/model/metadata.py | 1 - lib/galaxy/model/store/__init__.py | 10 +++ .../test_dataset_copy_metadata_files.py | 75 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 test/integration/test_dataset_copy_metadata_files.py diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index a0b53a1527ae..bb126e219252 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -667,7 +667,6 @@ def from_external_value(self, value, parent, path_rewriter=None): # directory. Correct. file_name = path_rewriter(file_name) mf.update_from_file(file_name) - os.unlink(file_name) value = mf.id return value diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index e6195d07d0f5..3a93b98b25ed 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -455,6 +455,16 @@ def handle_dataset_object_edit(dataset_instance, dataset_attrs): self._attach_dataset_sources(dataset_attrs["dataset"], dataset_instance) if "id" in dataset_attrs["dataset"] and self.import_options.allow_edit: dataset_instance.dataset.id = dataset_attrs["dataset"]["id"] + for dataset_association in dataset_instance.dataset.history_associations: + if ( + dataset_association is not dataset_instance + and dataset_association.extension == dataset_instance.extension + ): + dataset_association.metadata = dataset_instance.metadata + dataset_association.blurb = dataset_instance.blurb + dataset_association.peek = dataset_instance.peek + dataset_association.info = dataset_instance.info + dataset_association.tool_version = dataset_instance.tool_version if job: dataset_instance.dataset.job_id = job.id diff --git a/test/integration/test_dataset_copy_metadata_files.py b/test/integration/test_dataset_copy_metadata_files.py new file mode 100644 index 000000000000..2d066ce741b9 --- /dev/null +++ b/test/integration/test_dataset_copy_metadata_files.py @@ -0,0 +1,75 @@ +from galaxy_test.api.test_workflows import RunsWorkflowFixtures +from galaxy_test.base.populators import ( + DatasetPopulator, + WorkflowPopulator, +) +from galaxy_test.driver.integration_util import IntegrationTestCase + + +class TestDirectoryStrategyMetadataFileIntegrationTestCase(IntegrationTestCase, RunsWorkflowFixtures): + dataset_populator: DatasetPopulator + workflow_populator: WorkflowPopulator + framework_tool_and_types = True + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["metadata_strategy"] = "directory" + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) + + def test_metadata_copied_to_copied_outputs(self, history_id): + summary = self.workflow_populator.run_workflow( + """ +class: GalaxyWorkflow +label: Tests metadata copied to copied outputs +inputs: + bam_file: + type: collection + collection_type: list +outputs: + copied_bam: + outputSource: extract/output +steps: + build_list: + tool_id: __BUILD_LIST__ + in: + datasets_0|input: bam_file + sleep: + tool_id: cat_data_and_sleep + tool_state: + sleep_time: 2 + in: + input1: + source: build_list/output + extract: + tool_id: __EXTRACT_DATASET__ + tool_state: + which: + which_dataset: first + in: + input: + source: sleep/out_file1 +test_data: + bam_file: + value: 1.bam + file_type: bam + type: File +""", + history_id=history_id, + wait=True, + assert_ok=True, + ) + invocation = self.workflow_populator.get_invocation(summary.invocation_id, step_details=True) + copied_bam = invocation["outputs"]["copied_bam"] + dataset = self.dataset_populator.get_history_dataset_details(history_id, content_id=copied_bam["id"]) + assert dataset["peek"] == "Binary bam alignments file" + assert len(dataset["meta_files"]) == 1 + + +class TestExtendedMetadataStrategyMetadataFileIntegrationTestCase(TestDirectoryStrategyMetadataFileIntegrationTestCase): + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["metadata_strategy"] = "directory" From 93d11dc04a0861ea8546744fe25d8bb927f9e42c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 10 Nov 2023 18:37:32 +0100 Subject: [PATCH 2/2] Update tar_to_directory dependency Should fix: ``` Traceback (most recent call last): File "", line 1, in File "/usr/local/lib/python3.7/site-packages/galaxy/util/compression_utils.py", line 114, in extract common_prefix_dir = self.common_prefix_dir File "/usr/local/lib/python3.7/site-packages/galaxy/util/compression_utils.py", line 103, in common_prefix_dir if len(common_prefix) >= 1 and not common_prefix.endswith(os.sep) and self.isdir(self.getmember(common_prefix)) \ File "/usr/local/lib/python3.7/site-packages/galaxy/util/compression_utils.py", line 186, in isdir return getattr(self, 'isdir_%s' % self.type)(member) File "/usr/local/lib/python3.7/site-packages/galaxy/util/compression_utils.py", line 189, in isdir_tar return member.isdir() AttributeError: 'NoneType' object has no attribute 'isdir' ``` seen on usegalaxy.eu with https://www.ncbi.nlm.nih.gov/geo/download/?acc=GSE176031&format=file --- lib/galaxy/datatypes/converters/tar_to_directory.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/converters/tar_to_directory.xml b/lib/galaxy/datatypes/converters/tar_to_directory.xml index c0c8714a5e51..59354b39b5fd 100644 --- a/lib/galaxy/datatypes/converters/tar_to_directory.xml +++ b/lib/galaxy/datatypes/converters/tar_to_directory.xml @@ -1,7 +1,7 @@ - galaxy-util + galaxy-util mkdir '$output1.files_path';