Skip to content

Commit

Permalink
Skip state filering in __MERGE_COLLECTION__ tool
Browse files Browse the repository at this point in the history
Fixes inputs getting skipped that are NEW, QUEUED, etc.

If you're after filtering out errored you should use the
`__FILTER_FAILED_DATASETS__` tool.
  • Loading branch information
mvdbeek committed Oct 27, 2023
1 parent d7beda0 commit 69ca24f
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 32 deletions.
51 changes: 19 additions & 32 deletions lib/galaxy/tools/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3376,40 +3376,27 @@ def produce_outputs(self, trans, out_data, output_collections, incoming, history
for copy, input_list in enumerate(input_lists):
for dce in input_list.collection.elements:
element = dce.element_object
valid = False

# dealing with a single element
if hasattr(element, "is_ok"):
if element.is_ok:
valid = True
elif hasattr(element, "dataset_instances"):
# we are probably a list:paired dataset, both need to be in non error state
forward_o, reverse_o = element.dataset_instances
if forward_o.is_ok and reverse_o.is_ok:
valid = True
element_identifier = dce.element_identifier
identifier_seen = element_identifier in new_element_structure
appearances = identifiers_map[element_identifier]
add_suffix = False
if dupl_actions == "suffix_every":
add_suffix = True
elif dupl_actions == "suffix_conflict" and len(appearances) > 1:
add_suffix = True
elif dupl_actions == "suffix_conflict_rest" and len(appearances) > 1 and appearances[0] != copy:
add_suffix = True

if dupl_actions == "keep_first" and identifier_seen:
continue

if valid:
element_identifier = dce.element_identifier
identifier_seen = element_identifier in new_element_structure
appearances = identifiers_map[element_identifier]
add_suffix = False
if dupl_actions == "suffix_every":
add_suffix = True
elif dupl_actions == "suffix_conflict" and len(appearances) > 1:
add_suffix = True
elif dupl_actions == "suffix_conflict_rest" and len(appearances) > 1 and appearances[0] != copy:
add_suffix = True

if dupl_actions == "keep_first" and identifier_seen:
continue

if add_suffix and suffix_pattern:
suffix = suffix_pattern.replace("#", str(copy + 1))
effective_identifer = f"{element_identifier}{suffix}"
else:
effective_identifer = element_identifier
if add_suffix and suffix_pattern:
suffix = suffix_pattern.replace("#", str(copy + 1))
effective_identifer = f"{element_identifier}{suffix}"
else:
effective_identifer = element_identifier

new_element_structure[effective_identifer] = element
new_element_structure[effective_identifer] = element

# Don't copy until we know everything is fine and we have the structure of the list ready to go.
new_elements = {}
Expand Down
41 changes: 41 additions & 0 deletions lib/galaxy_test/api/test_workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -2976,6 +2976,47 @@ def test_export_invocation_ro_crate(self):
workflow = crate.mainEntity
assert workflow

@skip_without_tool("__MERGE_COLLECTION__")
def test_merge_collection_scheduling(self, history_id):
summary = self._run_workflow(
"""
class: GalaxyWorkflow
inputs:
collection:
type: collection
collection_type: list
outputs:
merge_out:
outputSource: merge/output
steps:
sleep:
tool_id: cat_data_and_sleep
in:
input1: collection
state:
sleep_time: 5
merge:
tool_id: __MERGE_COLLECTION__
in:
inputs_1|input: sleep/out_file1
inputs_0|input: sleep/out_file1
test_data:
collection:
collection_type: list
elements:
- identifier: 1
content: A
""",
history_id=history_id,
wait=True,
assert_ok=True,
)
invocation = self.workflow_populator.get_invocation(summary.invocation_id, step_details=True)
merge_out_id = invocation["output_collections"]["merge_out"]["id"]
merge_out = self.dataset_populator.get_history_collection_details(history_id, content_id=merge_out_id)
assert merge_out["element_count"] == 1
assert merge_out["elements"][0]["object"]["state"] == "ok"

@skip_without_tool("__MERGE_COLLECTION__")
@skip_without_tool("cat_collection")
@skip_without_tool("head")
Expand Down

0 comments on commit 69ca24f

Please sign in to comment.