Skip to content

Commit

Permalink
Merge branch 'release_24.0' into dev
Browse files Browse the repository at this point in the history
  • Loading branch information
mvdbeek committed Apr 4, 2024
2 parents 4a4ead2 + d308444 commit 6f2a043
Show file tree
Hide file tree
Showing 26 changed files with 249 additions and 127 deletions.
6 changes: 4 additions & 2 deletions lib/galaxy/managers/workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,14 @@
visit_input_values,
)
from galaxy.tools.parameters.basic import (
ConnectedValue,
DataCollectionToolParameter,
DataToolParameter,
)
from galaxy.tools.parameters.workflow_utils import (
ConnectedValue,
RuntimeValue,
workflow_building_modes,
)
from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
from galaxy.util.hash_util import md5_hash_str
from galaxy.util.json import (
safe_dumps,
Expand Down
12 changes: 9 additions & 3 deletions lib/galaxy/tool_util/verify/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,12 @@
try:
from PIL import Image
except ImportError:
pass
Image = None # type: ignore[assignment]
try:
import tifffile
except ImportError:
pass
tifffile = None # type: ignore[assignment]


from galaxy.tool_util.parser.util import (
DEFAULT_DELTA,
Expand Down Expand Up @@ -193,7 +194,12 @@ def get_filename(filename: str) -> str:
elif compare == "contains":
files_contains(local_name, temp_name, attributes=attributes)
elif compare == "image_diff":
files_image_diff(local_name, temp_name, attributes=attributes)
if Image and tifffile:
files_image_diff(local_name, temp_name, attributes=attributes)
else:
raise Exception(
"pillow and tifffile are not installed, but required to compare files using the 'image_diff' method"
)
else:
raise Exception(f"Unimplemented Compare type: {compare}")
except AssertionError as err:
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/tools/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@
)
from galaxy.tools.parameters.input_translation import ToolInputTranslator
from galaxy.tools.parameters.meta import expand_meta_parameters
from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
from galaxy.tools.parameters.workflow_utils import workflow_building_modes
from galaxy.tools.parameters.wrapped_json import json_wrap
from galaxy.tools.test import parse_tests
from galaxy.util import (
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/tools/actions/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@
from galaxy.tools.parameters.basic import (
DataCollectionToolParameter,
DataToolParameter,
RuntimeValue,
SelectToolParameter,
)
from galaxy.tools.parameters.workflow_utils import RuntimeValue
from galaxy.tools.parameters.wrapped import (
LegacyUnprefixedDict,
WrappedParameters,
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/tools/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
on_text_for_names,
ToolExecutionCache,
)
from galaxy.tools.parameters.basic import is_runtime_value
from galaxy.tools.parameters.workflow_utils import is_runtime_value

if typing.TYPE_CHECKING:
from galaxy.tools import Tool
Expand Down
13 changes: 9 additions & 4 deletions lib/galaxy/tools/parameters/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@
from .basic import (
DataCollectionToolParameter,
DataToolParameter,
is_runtime_value,
ParameterValueError,
runtime_to_json,
SelectToolParameter,
ToolParameter,
)
Expand All @@ -29,6 +27,11 @@
Section,
UploadDataset,
)
from .workflow_utils import (
is_runtime_value,
runtime_to_json,
)
from .wrapped import flat_to_nested_state

REPLACE_ON_TRUTHY = object()

Expand Down Expand Up @@ -515,6 +518,8 @@ def populate_state(
def _populate_state_legacy(
request_context, inputs, incoming, state, errors, prefix="", context=None, check=True, simple_errors=True
):
if context is None:
context = flat_to_nested_state(incoming)
context = ExpressionContext(state, context)
for input in inputs.values():
state[input.name] = input.get_initial_value(request_context, context)
Expand Down Expand Up @@ -600,8 +605,8 @@ def _populate_state_legacy(
for upload_item in input.inputs.values():
new_state[upload_item.name] = upload_item.get_initial_value(request_context, context)
group_state.append(new_state)
for rep_state in group_state:
rep_index = rep_state["__index__"]
for rep_index, rep_state in enumerate(group_state):
rep_index = rep_state.get("__index__", rep_index)
rep_prefix = "%s_%d|" % (key, rep_index)
_populate_state_legacy(
request_context,
Expand Down
44 changes: 7 additions & 37 deletions lib/galaxy/tools/parameters/basic.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from galaxy.model.dataset_collections import builder
from galaxy.schema.fetch_data import FilesPayload
from galaxy.tool_util.parser import get_input_source as ensure_input_source
from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
from galaxy.tools.parameters.workflow_utils import workflow_building_modes
from galaxy.util import (
sanitize_param,
string_as_bool,
Expand All @@ -61,6 +61,12 @@
)
from .dataset_matcher import get_dataset_matcher_factory
from .sanitize import ToolParameterSanitizer
from .workflow_utils import (
is_runtime_value,
runtime_to_json,
runtime_to_object,
RuntimeValue,
)

if TYPE_CHECKING:
from sqlalchemy.orm import Session
Expand Down Expand Up @@ -92,12 +98,6 @@ def contains_workflow_parameter(value, search=False):
return False


def is_runtime_value(value):
return isinstance(value, RuntimeValue) or (
isinstance(value, MutableMapping) and value.get("__class__") in ["RuntimeValue", "ConnectedValue"]
)


def is_runtime_context(trans, other_values):
if trans.workflow_building_mode:
return True
Expand Down Expand Up @@ -2780,36 +2780,6 @@ def write_elements_to_collection(has_elements, collection_builder):
)


def runtime_to_json(runtime_value):
if isinstance(runtime_value, ConnectedValue) or (
isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
):
return {"__class__": "ConnectedValue"}
else:
return {"__class__": "RuntimeValue"}


def runtime_to_object(runtime_value):
if isinstance(runtime_value, ConnectedValue) or (
isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
):
return ConnectedValue()
else:
return RuntimeValue()


class RuntimeValue:
"""
Wrapper to note a value that is not yet set, but will be required at runtime.
"""


class ConnectedValue(RuntimeValue):
"""
Wrapper to note a value that is not yet set, but will be inferred from a connection.
"""


def history_item_dict_to_python(value, app, name):
if isinstance(value, MutableMapping) and "src" in value:
if value["src"] not in ("hda", "dce", "ldda", "hdca"):
Expand Down
31 changes: 20 additions & 11 deletions lib/galaxy/tools/parameters/dynamic_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@
User,
)
from galaxy.tools.expressions import do_eval
from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
from galaxy.tools.parameters.workflow_utils import (
is_runtime_value,
workflow_building_modes,
)
from galaxy.util import (
Element,
string_as_bool,
Expand Down Expand Up @@ -737,16 +740,20 @@ def get_fields(self, trans, other_values):
if not hasattr(dataset, "get_file_name"):
continue
# Ensure parsing dynamic options does not consume more than a megabyte worth memory.
path = dataset.get_file_name()
if os.path.getsize(path) < 1048576:
with open(path) as fh:
options += self.parse_file_fields(fh)
else:
# Pass just the first megabyte to parse_file_fields.
log.warning("Attempting to load options from large file, reading just first megabyte")
with open(path) as fh:
contents = fh.read(1048576)
options += self.parse_file_fields(StringIO(contents))
try:
path = dataset.get_file_name()
if os.path.getsize(path) < 1048576:
with open(path) as fh:
options += self.parse_file_fields(fh)
else:
# Pass just the first megabyte to parse_file_fields.
log.warning("Attempting to load options from large file, reading just first megabyte")
with open(path) as fh:
contents = fh.read(1048576)
options += self.parse_file_fields(StringIO(contents))
except Exception as e:
log.warning("Could not read contents from %s: %s", dataset, str(e))
continue
elif self.tool_data_table:
options = self.tool_data_table.get_fields()
if trans and trans.user and trans.workflow_building_mode != workflow_building_modes.ENABLED:
Expand Down Expand Up @@ -964,6 +971,8 @@ def _get_ref_data(other_values, ref_name):
list,
),
):
if is_runtime_value(ref):
return []
raise ValueError
if isinstance(ref, DatasetCollectionElement) and ref.hda:
ref = ref.hda
Expand Down
4 changes: 2 additions & 2 deletions lib/galaxy/tools/parameters/grouping.py
Original file line number Diff line number Diff line change
Expand Up @@ -618,8 +618,8 @@ def get_filenames(context):
writable_files = d_type.writable_files
writable_files_offset = 0
groups_incoming = [None for _ in range(file_count)]
for group_incoming in context.get(self.name, []):
i = int(group_incoming["__index__"])
for i, group_incoming in enumerate(context.get(self.name, [])):
i = int(group_incoming.get("__index__", i))
groups_incoming[i] = group_incoming
if d_type.composite_type is not None or force_composite:
# handle uploading of composite datatypes
Expand Down
26 changes: 1 addition & 25 deletions lib/galaxy/tools/parameters/meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
)
from galaxy.util import permutations
from . import visit_input_values
from .wrapped import process_key

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -153,31 +154,6 @@ def is_batch(value):
return WorkflowParameterExpansion(param_combinations, params_keys, input_combinations)


def process_key(incoming_key, incoming_value, d):
key_parts = incoming_key.split("|")
if len(key_parts) == 1:
# Regular parameter
if incoming_key in d and not incoming_value:
# In case we get an empty repeat after we already filled in a repeat element
return
d[incoming_key] = incoming_value
elif key_parts[0].rsplit("_", 1)[-1].isdigit():
# Repeat
input_name, index = key_parts[0].rsplit("_", 1)
index = int(index)
d.setdefault(input_name, [])
newlist = [{} for _ in range(index + 1)]
d[input_name].extend(newlist[len(d[input_name]) :])
subdict = d[input_name][index]
process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)
else:
# Section / Conditional
input_name = key_parts[0]
subdict = {}
d[input_name] = subdict
process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)


ExpandedT = Tuple[List[Dict[str, Any]], Optional[matching.MatchingCollections]]


Expand Down
4 changes: 0 additions & 4 deletions lib/galaxy/tools/parameters/workflow_building_modes.py

This file was deleted.

43 changes: 43 additions & 0 deletions lib/galaxy/tools/parameters/workflow_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
from collections.abc import MutableMapping


class workflow_building_modes:
DISABLED = False
ENABLED = True
USE_HISTORY = 1


def runtime_to_json(runtime_value):
if isinstance(runtime_value, ConnectedValue) or (
isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
):
return {"__class__": "ConnectedValue"}
else:
return {"__class__": "RuntimeValue"}


def runtime_to_object(runtime_value):
if isinstance(runtime_value, ConnectedValue) or (
isinstance(runtime_value, MutableMapping) and runtime_value["__class__"] == "ConnectedValue"
):
return ConnectedValue()
else:
return RuntimeValue()


class RuntimeValue:
"""
Wrapper to note a value that is not yet set, but will be required at runtime.
"""


class ConnectedValue(RuntimeValue):
"""
Wrapper to note a value that is not yet set, but will be inferred from a connection.
"""


def is_runtime_value(value):
return isinstance(value, RuntimeValue) or (
isinstance(value, MutableMapping) and value.get("__class__") in ["RuntimeValue", "ConnectedValue"]
)
40 changes: 38 additions & 2 deletions lib/galaxy/tools/parameters/wrapped.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
from collections import UserDict
from typing import Dict
from typing import (
Any,
Dict,
List,
)

from galaxy.tools.parameters.basic import (
DataCollectionToolParameter,
Expand Down Expand Up @@ -158,4 +162,36 @@ def make_list_copy(from_list):
return new_list


__all__ = ("LegacyUnprefixedDict", "WrappedParameters", "make_dict_copy")
def process_key(incoming_key: str, incoming_value: Any, d: Dict[str, Any]):
key_parts = incoming_key.split("|")
if len(key_parts) == 1:
# Regular parameter
if incoming_key in d and not incoming_value:
# In case we get an empty repeat after we already filled in a repeat element
return
d[incoming_key] = incoming_value
elif key_parts[0].rsplit("_", 1)[-1].isdigit():
# Repeat
input_name, _index = key_parts[0].rsplit("_", 1)
index = int(_index)
d.setdefault(input_name, [])
newlist: List[Dict[Any, Any]] = [{} for _ in range(index + 1)]
d[input_name].extend(newlist[len(d[input_name]) :])
subdict = d[input_name][index]
process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)
else:
# Section / Conditional
input_name = key_parts[0]
subdict = d.get(input_name, {})
d[input_name] = subdict
process_key("|".join(key_parts[1:]), incoming_value=incoming_value, d=subdict)


def flat_to_nested_state(incoming: Dict[str, Any]):
nested_state: Dict[str, Any] = {}
for key, value in incoming.items():
process_key(key, value, nested_state)
return nested_state


__all__ = ("LegacyUnprefixedDict", "WrappedParameters", "make_dict_copy", "process_key", "flat_to_nested_state")
2 changes: 1 addition & 1 deletion lib/galaxy/tools/recommendations.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import yaml

from galaxy.tools.parameters import populate_state
from galaxy.tools.parameters.workflow_building_modes import workflow_building_modes
from galaxy.tools.parameters.workflow_utils import workflow_building_modes
from galaxy.util import DEFAULT_SOCKET_TIMEOUT
from galaxy.workflow.modules import module_factory

Expand Down
Loading

0 comments on commit 6f2a043

Please sign in to comment.