diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 546085b26a2e..b7221cf0fef9 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -9,6 +9,7 @@ Dict, List, Optional, + Union, ) import sqlalchemy @@ -41,7 +42,10 @@ Safety, ) from galaxy.managers.collections import DatasetCollectionManager -from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.context import ( + ProvidesHistoryContext, + ProvidesUserContext, +) from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager @@ -68,6 +72,10 @@ ) from galaxy.security.idencoding import IdEncodingHelper from galaxy.structured_app import StructuredApp +from galaxy.tools._types import ( + ToolStateDumpedToJsonInternalT, + ToolStateJobInstancePopulatedT, +) from galaxy.util import ( defaultdict, ExecutionTimer, @@ -81,6 +89,9 @@ log = logging.getLogger(__name__) +JobStateT = str +JobStatesT = Union[JobStateT, List[JobStateT]] + class JobLock(BaseModel): active: bool = Field(title="Job lock status", description="If active, jobs will not dispatch") @@ -311,7 +322,15 @@ def __init__( self.ldda_manager = ldda_manager self.decode_id = id_encoding_helper.decode_id - def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state="ok"): + def by_tool_input( + self, + trans: ProvidesHistoryContext, + tool_id: str, + tool_version: Optional[str], + param: ToolStateJobInstancePopulatedT, + param_dump: ToolStateDumpedToJsonInternalT, + job_state: Optional[JobStatesT] = "ok", + ): """Search for jobs producing same results using the 'inputs' part of a tool POST.""" user = trans.user input_data = defaultdict(list) @@ -353,7 +372,14 @@ def populate_input_data_input_id(path, key, value): ) def __search( - self, tool_id, tool_version, user, input_data, job_state=None, param_dump=None, wildcard_param_dump=None + self, + tool_id: str, + tool_version: Optional[str], + user: model.User, + input_data, + job_state: Optional[JobStatesT], + param_dump: ToolStateDumpedToJsonInternalT, + wildcard_param_dump=None, ): search_timer = ExecutionTimer() @@ -462,7 +488,9 @@ def replace_dataset_ids(path, key, value): log.info("No equivalent jobs found %s", search_timer) return None - def _build_job_subquery(self, tool_id, user_id, tool_version, job_state, wildcard_param_dump): + def _build_job_subquery( + self, tool_id: str, user_id: int, tool_version: Optional[str], job_state, wildcard_param_dump + ): """Build subquery that selects a job with correct job parameters.""" stmt = select(model.Job.id).where( and_( diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 7193fd6ea618..6933b09d1809 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -120,6 +120,8 @@ check_param, params_from_strings, params_to_incoming, + params_to_json, + params_to_json_internal, params_to_strings, populate_state, visit_input_values, @@ -183,7 +185,19 @@ get_tool_shed_url_from_tool_shed_registry, ) from galaxy.version import VERSION_MAJOR -from galaxy.work.context import proxy_work_context_for_history +from galaxy.work.context import ( + proxy_work_context_for_history, + WorkRequestContext, +) +from ._types import ( + InputFormatT, + ParameterValidationErrorsT, + ToolRequestT, + ToolStateDumpedToJsonInternalT, + ToolStateDumpedToJsonT, + ToolStateJobInstancePopulatedT, + ToolStateJobInstanceT, +) from .execute import ( DatasetCollectionElementsSliceT, DEFAULT_JOB_CALLBACK, @@ -195,8 +209,6 @@ ExecutionSlice, JobCallbackT, MappingParameters, - ToolParameterRequestInstanceT, - ToolParameterRequestT, ) if TYPE_CHECKING: @@ -712,7 +724,7 @@ def encode(self, tool, app, nested=False): """ Convert the data to a string """ - value = params_to_strings(tool.inputs, self.inputs, app, nested=nested) + value = cast(Dict[str, Any], params_to_strings(tool.inputs, self.inputs, app, nested=nested)) value["__page__"] = self.page value["__rerun_remap_job_id__"] = self.rerun_remap_job_id return value @@ -1514,8 +1526,8 @@ def parse_input_elem( # Repeat group input_type = input_source.parse_input_type() if input_type == "repeat": - group_r = Repeat() - group_r.name = input_source.get("name") + repeat_name = input_source.get("name") + group_r = Repeat(repeat_name) group_r.title = input_source.get("title") group_r.help = input_source.get("help", None) page_source = input_source.parse_nested_inputs_source() @@ -1531,15 +1543,17 @@ def parse_input_elem( group_r.default = cast(int, min(max(group_r.default, group_r.min), group_r.max)) rval[group_r.name] = group_r elif input_type == "conditional": - group_c = Conditional() - group_c.name = input_source.get("name") - group_c.value_ref = input_source.get("value_ref", None) + cond_name = input_source.get("name") + group_c = Conditional(cond_name) + value_ref = input_source.get("value_ref", None) + group_c.value_ref = value_ref group_c.value_ref_in_group = input_source.get_bool("value_ref_in_group", True) value_from = input_source.get("value_from", None) if value_from: value_from = value_from.split(":") temp_value_from = locals().get(value_from[0]) - group_c.test_param = rval[group_c.value_ref] + assert value_ref + group_c.test_param = rval[value_ref] assert isinstance(group_c.test_param, ToolParameter) group_c.test_param.refresh_on_change = True for attr in value_from[1].split("."): @@ -1600,8 +1614,8 @@ def parse_input_elem( group_c.cases.append(case) rval[group_c.name] = group_c elif input_type == "section": - group_s = Section() - group_s.name = input_source.get("name") + section_name = input_source.get("name") + group_s = Section(section_name) group_s.title = input_source.get("title") group_s.help = input_source.get("help", None) group_s.expanded = input_source.get_bool("expanded", False) @@ -1610,8 +1624,8 @@ def parse_input_elem( rval[group_s.name] = group_s elif input_type == "upload_dataset": elem = input_source.elem() - group_u = UploadDataset() - group_u.name = elem.get("name") + upload_name = elem.get("name") + group_u = UploadDataset(upload_name) group_u.title = elem.get("title") group_u.file_type_name = elem.get("file_type_name", group_u.file_type_name) group_u.default_file_type = elem.get("default_file_type", group_u.default_file_type) @@ -1796,22 +1810,22 @@ def visit_inputs(self, values, callback): if self.check_values: visit_input_values(self.inputs, values, callback) - def expand_incoming(self, trans, incoming, request_context, input_format="legacy"): - rerun_remap_job_id = None - if "rerun_remap_job_id" in incoming: - try: - rerun_remap_job_id = trans.app.security.decode_id(incoming["rerun_remap_job_id"]) - except Exception as exception: - log.error(str(exception)) - raise exceptions.MessageException( - "Failure executing tool with id '%s' (attempting to rerun invalid job).", self.id - ) - + def expand_incoming( + self, request_context: WorkRequestContext, incoming: ToolRequestT, input_format: InputFormatT = "legacy" + ) -> Tuple[ + List[ToolStateJobInstancePopulatedT], + List[ToolStateJobInstancePopulatedT], + Optional[int], + Optional[MatchingCollections], + ]: + rerun_remap_job_id = _rerun_remap_job_id(request_context, incoming, self.id) set_dataset_matcher_factory(request_context, self) # Fixed set of input parameters may correspond to any number of jobs. # Expand these out to individual parameters for given jobs (tool executions). - expanded_incomings, collection_info = expand_meta_parameters(trans, self, incoming) + expanded_incomings: List[ToolStateJobInstanceT] + collection_info: Optional[MatchingCollections] + expanded_incomings, collection_info = expand_meta_parameters(request_context, self, incoming) # Remapping a single job to many jobs doesn't make sense, so disable # remap if multi-runs of tools are being used. @@ -1831,39 +1845,11 @@ def expand_incoming(self, trans, incoming, request_context, input_format="legacy "internals.galaxy.tools.validation", "Validated and populated state for tool request", ) - all_errors = [] - all_params = [] + all_errors: List[ParameterValidationErrorsT] = [] + all_params: List[ToolStateJobInstancePopulatedT] = [] + for expanded_incoming in expanded_incomings: - params = {} - errors: Dict[str, str] = {} - if self.input_translator: - self.input_translator.translate(expanded_incoming) - if not self.check_values: - # If `self.check_values` is false we don't do any checking or - # processing on input This is used to pass raw values - # through to/from external sites. - params = expanded_incoming - else: - # Update state for all inputs on the current page taking new - # values from `incoming`. - populate_state( - request_context, - self.inputs, - expanded_incoming, - params, - errors, - simple_errors=False, - input_format=input_format, - ) - # If the tool provides a `validate_input` hook, call it. - validate_input = self.get_hook("validate_input") - if validate_input: - # hooks are so terrible ... this is specifically for https://github.com/galaxyproject/tools-devteam/blob/main/tool_collections/gops/basecoverage/operation_filter.py - legacy_non_dce_params = { - k: v.hda if isinstance(v, model.DatasetCollectionElement) and v.hda else v - for k, v in params.items() - } - validate_input(request_context, errors, legacy_non_dce_params, self.inputs) + params, errors = self._populate(request_context, expanded_incoming, input_format) all_errors.append(errors) all_params.append(params) unset_dataset_matcher_factory(request_context) @@ -1871,14 +1857,74 @@ def expand_incoming(self, trans, incoming, request_context, input_format="legacy log.info(validation_timer) return all_params, all_errors, rerun_remap_job_id, collection_info + def _populate( + self, request_context, expanded_incoming: ToolStateJobInstanceT, input_format: InputFormatT + ) -> Tuple[ToolStateJobInstancePopulatedT, ParameterValidationErrorsT]: + """Validate expanded parameters for a job to replace references with model objects. + + So convert a ToolStateJobInstanceT to a ToolStateJobInstancePopulatedT. + """ + params: ToolStateJobInstancePopulatedT = {} + errors: ParameterValidationErrorsT = {} + if self.input_translator: + self.input_translator.translate(expanded_incoming) + if not self.check_values: + # If `self.check_values` is false we don't do any checking or + # processing on input This is used to pass raw values + # through to/from external sites. + params = cast(ToolStateJobInstancePopulatedT, expanded_incoming) + else: + # Update state for all inputs on the current page taking new + # values from `incoming`. + populate_state( + request_context, + self.inputs, + expanded_incoming, + params, + errors, + simple_errors=False, + input_format=input_format, + ) + # If the tool provides a `validate_input` hook, call it. + validate_input = self.get_hook("validate_input") + if validate_input: + # hooks are so terrible ... this is specifically for https://github.com/galaxyproject/tools-devteam/blob/main/tool_collections/gops/basecoverage/operation_filter.py + legacy_non_dce_params = { + k: v.hda if isinstance(v, model.DatasetCollectionElement) and v.hda else v + for k, v in params.items() + } + validate_input(request_context, errors, legacy_non_dce_params, self.inputs) + return params, errors + + def completed_jobs( + self, trans, use_cached_job: bool, all_params: List[ToolStateJobInstancePopulatedT] + ) -> Dict[int, Optional[model.Job]]: + completed_jobs: Dict[int, Optional[model.Job]] = {} + for i, param in enumerate(all_params): + if use_cached_job: + tool_id = self.id + assert tool_id + param_dump: ToolStateDumpedToJsonInternalT = params_to_json_internal(self.inputs, param, self.app) + completed_jobs[i] = self.job_search.by_tool_input( + trans=trans, + tool_id=tool_id, + tool_version=self.version, + param=param, + param_dump=param_dump, + job_state=None, + ) + else: + completed_jobs[i] = None + return completed_jobs + def handle_input( self, trans, - incoming: ToolParameterRequestT, + incoming: ToolRequestT, history: Optional[model.History] = None, use_cached_job: bool = DEFAULT_USE_CACHED_JOB, preferred_object_store_id: Optional[str] = DEFAULT_PREFERRED_OBJECT_STORE_ID, - input_format: str = "legacy", + input_format: InputFormatT = "legacy", ): """ Process incoming parameters for this tool from the dict `incoming`, @@ -1887,26 +1933,17 @@ def handle_input( there were no errors). """ request_context = proxy_work_context_for_history(trans, history=history) - all_params, all_errors, rerun_remap_job_id, collection_info = self.expand_incoming( - trans=trans, incoming=incoming, request_context=request_context, input_format=input_format - ) + expanded = self.expand_incoming(request_context, incoming=incoming, input_format=input_format) + all_params: List[ToolStateJobInstancePopulatedT] = expanded[0] + all_errors: List[ParameterValidationErrorsT] = expanded[1] + rerun_remap_job_id: Optional[int] = expanded[2] + collection_info: Optional[MatchingCollections] = expanded[3] + # If there were errors, we stay on the same page and display them self.handle_incoming_errors(all_errors) mapping_params = MappingParameters(incoming, all_params) - completed_jobs: Dict[int, Optional[model.Job]] = {} - for i, param in enumerate(all_params): - if use_cached_job: - completed_jobs[i] = self.job_search.by_tool_input( - trans=trans, - tool_id=self.id, - tool_version=self.version, - param=param, - param_dump=self.params_to_strings(param, self.app, nested=True), - job_state=None, - ) - else: - completed_jobs[i] = None + completed_jobs: Dict[int, Optional[model.Job]] = self.completed_jobs(trans, use_cached_job, all_params) execution_tracker = execute_job( trans, self, @@ -1935,7 +1972,7 @@ def handle_input( implicit_collections=execution_tracker.implicit_collections, ) - def handle_incoming_errors(self, all_errors): + def handle_incoming_errors(self, all_errors: List[ParameterValidationErrorsT]) -> None: if any(all_errors): # simple param_key -> message string for tool form. err_data = {key: unicodify(value) for d in all_errors for (key, value) in d.items()} @@ -2060,7 +2097,7 @@ def get_static_param_values(self, trans): def execute( self, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[model.History] = None, set_output_hid: bool = DEFAULT_SET_OUTPUT_HID, flush_job: bool = True, @@ -2086,7 +2123,7 @@ def execute( def _execute( self, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[model.History] = None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, execution_cache: Optional[ToolExecutionCache] = None, @@ -2128,7 +2165,7 @@ def _execute( log.error("Tool execution failed for job: %s", job_id) raise - def params_to_strings(self, params, app, nested=False): + def params_to_strings(self, params: ToolStateJobInstancePopulatedT, app, nested=False): return params_to_strings(self.inputs, params, app, nested) def params_from_strings(self, params, app, ignore_errors=False): @@ -2559,7 +2596,7 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo set_dataset_matcher_factory(request_context, self) # create tool state state_inputs: Dict[str, str] = {} - state_errors: Dict[str, str] = {} + state_errors: ParameterValidationErrorsT = {} populate_state(request_context, self.inputs, params.__dict__, state_inputs, state_errors) # create tool model @@ -2581,6 +2618,8 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo else: action = self.app.url_for(self.action) + state_inputs_json: ToolStateDumpedToJsonT = params_to_json(self.inputs, state_inputs, self.app) + # update tool model tool_model.update( { @@ -2594,7 +2633,7 @@ def to_json(self, trans, kwd=None, job=None, workflow_building_mode=False, histo "requirements": [{"name": r.name, "version": r.version} for r in self.requirements], "errors": state_errors, "tool_errors": self.tool_errors, - "state_inputs": params_to_strings(self.inputs, state_inputs, self.app, use_security=True, nested=True), + "state_inputs": state_inputs_json, "job_id": trans.security.encode_id(job.id) if job else None, "job_remap": job.remappable() if job else None, "history_id": trans.security.encode_id(history.id) if history else None, @@ -4110,6 +4149,21 @@ def produce_outputs(self, trans, out_data, output_collections, incoming, history # ---- Utility classes to be factored out ----------------------------------- + + +def _rerun_remap_job_id(trans, incoming, tool_id: Optional[str]) -> Optional[int]: + rerun_remap_job_id = None + if "rerun_remap_job_id" in incoming: + try: + rerun_remap_job_id = trans.app.security.decode_id(incoming["rerun_remap_job_id"]) + except Exception as exception: + log.error(str(exception)) + raise exceptions.MessageException( + "Failure executing tool with id '%s' (attempting to rerun invalid job).", tool_id + ) + return rerun_remap_job_id + + class TracksterConfig: """Trackster configuration encapsulation.""" diff --git a/lib/galaxy/tools/_types.py b/lib/galaxy/tools/_types.py new file mode 100644 index 000000000000..635a86cf459d --- /dev/null +++ b/lib/galaxy/tools/_types.py @@ -0,0 +1,65 @@ +""" +Tool state goes through several different iterations that are difficult to follow I think, +hopefully datatypes can be used as markers to describe what has been done - even if they don't +provide strong traditional typing semantics. + ++--------------------------------+------------+---------------------------------+------------+-----------+ +| Python Type | State for? | Object References | Validated? | xref | ++================================+============+=================================+============+===========+ +| ToolRequestT | request | src dicts of encoded ids | nope | | +| ToolStateJobInstanceT | a job | src dicts of encoded ids | nope | | +| ToolStateJobInstancePopulatedT | a job | model objs loaded from db | check_param | | +| ToolStateDumpedToJsonT | a job | src dicts of encoded ids | " | | +| | | (normalized into values attr) | " | | +| ToolStateDumpedToJsonInternalT | a job | src dicts of decoded ids | " | | +| | | (normalized into values attr) | " | | +| ToolStateDumpedToStringsT | a job | src dicts dumped to strs | " | | +| | | (normalized into values attr) | " | | ++--------------------------------+------------+---------------------------------+-------------+----------+ +""" + +from typing import ( + Any, + Dict, + Union, +) + +from typing_extensions import Literal + +# Input dictionary from the API, may include map/reduce instructions. Objects are referenced by "src" +# dictionaries and encoded IDS. +ToolRequestT = Dict[str, Any] + +# Input dictionary extracted from a tool request for running a tool individually as a single job. Objects are referenced +# by "src" dictionaries with encoded IDs still but batch instructions have been pulled out. Parameters have not +# been "checked" (check_param has not been called). +ToolStateJobInstanceT = Dict[str, Any] + +# Input dictionary for an individual job where objects are their model objects and parameters have been +# "checked" (check_param has been called). +ToolStateJobInstancePopulatedT = Dict[str, Any] + +# Input dictionary for an individual where the state has been valiated and populated but then converted back down +# to json. Object references are unified in the format of {"values": List["src" dictionary]} where the src dictionaries. +# are decoded ids (ints). +# See comments on galaxy.tools.parameters.params_to_strings for more information. +ToolStateDumpedToJsonInternalT = Dict[str, Any] + +# Input dictionary for an individual where the state has been valiated and populated but then converted back down +# to json. Object references are unified in the format of {"values": List["src" dictionary]} where src dictonaries +# are encoded (ids). See comments on galaxy.tools.parameters.params_to_strings for more information. +ToolStateDumpedToJsonT = Dict[str, Any] + +# Input dictionary for an individual where the state has been valiated and populated but then converted back down +# to json. Object references are unified in the format of {"values": List["src" dictionary]} but dumped into +# strings. See comments on galaxy.tools.parameters.params_to_strings for more information. This maybe should be +# broken into separate types for encoded and decoded IDs in subsequent type refinements if both are used, it not +# this comment should be updated to indicate which is used exclusively. +ToolStateDumpedToStringsT = Dict[str, str] + +# A dictionary of error messages that occur while attempting to validate a ToolStateJobInstanceT and transform it +# into a ToolStateJobInstancePopulatedT with model objects populated. Tool errors indicate the job should not be +# further processed. +ParameterValidationErrorsT = Dict[str, Union["ParameterValidationErrorsT", str, Exception]] + +InputFormatT = Literal["legacy", "21.01"] diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 989ab5a6ad45..bddc224e99ae 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -37,6 +37,7 @@ from galaxy.model.dataset_collections.matching import MatchingCollections from galaxy.model.none_like import NoneDataset from galaxy.objectstore import ObjectStorePopulator +from galaxy.tools._types import ToolStateJobInstancePopulatedT from galaxy.tools.execute import ( DatasetCollectionElementsSliceT, DEFAULT_DATASET_COLLECTION_ELEMENTS, @@ -45,7 +46,6 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, JobCallbackT, - ToolParameterRequestInstanceT, ) from galaxy.tools.execution_helpers import ( filter_output, @@ -88,7 +88,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, @@ -104,6 +104,21 @@ def execute( ) -> ToolActionExecuteResult: """Perform target tool action.""" + @abstractmethod + def get_output_name( + self, + output, + dataset=None, + tool=None, + on_text=None, + trans=None, + incoming=None, + history=None, + params=None, + job_params=None, + ) -> str: + """Get name to assign a tool output.""" + class DefaultToolAction(ToolAction): """Default tool action is to run an external command""" @@ -401,7 +416,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, @@ -950,7 +965,7 @@ def get_output_name( history=None, params=None, job_params=None, - ): + ) -> str: if output.label: params["tool"] = tool params["on_string"] = on_text diff --git a/lib/galaxy/tools/actions/data_manager.py b/lib/galaxy/tools/actions/data_manager.py index c24e86fd0afb..d8786ad5a921 100644 --- a/lib/galaxy/tools/actions/data_manager.py +++ b/lib/galaxy/tools/actions/data_manager.py @@ -7,6 +7,7 @@ ) from galaxy.model.base import transaction from galaxy.model.dataset_collections.matching import MatchingCollections +from galaxy.tools._types import ToolStateJobInstancePopulatedT from galaxy.tools.execute import ( DatasetCollectionElementsSliceT, DEFAULT_DATASET_COLLECTION_ELEMENTS, @@ -15,7 +16,6 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, JobCallbackT, - ToolParameterRequestInstanceT, ) from galaxy.tools.execution_helpers import ToolExecutionCache from . import ( @@ -33,7 +33,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py index 848995c61dac..de502f3a11d7 100644 --- a/lib/galaxy/tools/actions/history_imp_exp.py +++ b/lib/galaxy/tools/actions/history_imp_exp.py @@ -11,6 +11,7 @@ ) from galaxy.model.base import transaction from galaxy.model.dataset_collections.matching import MatchingCollections +from galaxy.tools._types import ToolStateJobInstancePopulatedT from galaxy.tools.actions import ( ToolAction, ToolActionExecuteResult, @@ -23,7 +24,6 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, JobCallbackT, - ToolParameterRequestInstanceT, ) from galaxy.tools.execution_helpers import ToolExecutionCache from galaxy.tools.imp_exp import ( @@ -44,7 +44,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, @@ -121,7 +121,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index f7d6ce844a9d..2b46c6060ccd 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -16,6 +16,7 @@ ) from galaxy.model.base import transaction from galaxy.model.dataset_collections.matching import MatchingCollections +from galaxy.tools._types import ToolStateJobInstancePopulatedT from galaxy.tools.execute import ( DatasetCollectionElementsSliceT, DEFAULT_DATASET_COLLECTION_ELEMENTS, @@ -24,7 +25,6 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, JobCallbackT, - ToolParameterRequestInstanceT, ) from galaxy.tools.execution_helpers import ToolExecutionCache from galaxy.util import asbool @@ -43,7 +43,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, diff --git a/lib/galaxy/tools/actions/model_operations.py b/lib/galaxy/tools/actions/model_operations.py index 1b18adcf39f6..fdfecfb9c65e 100644 --- a/lib/galaxy/tools/actions/model_operations.py +++ b/lib/galaxy/tools/actions/model_operations.py @@ -10,6 +10,7 @@ ) from galaxy.model.dataset_collections.matching import MatchingCollections from galaxy.objectstore import ObjectStorePopulator +from galaxy.tools._types import ToolStateJobInstancePopulatedT from galaxy.tools.actions import ( DefaultToolAction, OutputCollections, @@ -24,7 +25,6 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, JobCallbackT, - ToolParameterRequestInstanceT, ) from galaxy.tools.execution_helpers import ToolExecutionCache @@ -52,7 +52,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index b85bba71a0d2..c758c96b2ae7 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -11,6 +11,7 @@ from galaxy.model.base import transaction from galaxy.model.dataset_collections.matching import MatchingCollections from galaxy.model.dataset_collections.structure import UninitializedTree +from galaxy.tools._types import ToolStateJobInstancePopulatedT from galaxy.tools.actions import upload_common from galaxy.tools.execute import ( DatasetCollectionElementsSliceT, @@ -20,7 +21,6 @@ DEFAULT_RERUN_REMAP_JOB_ID, DEFAULT_SET_OUTPUT_HID, JobCallbackT, - ToolParameterRequestInstanceT, ) from galaxy.tools.execution_helpers import ToolExecutionCache from galaxy.util import ExecutionTimer @@ -40,7 +40,7 @@ def execute( self, tool, trans, - incoming: Optional[ToolParameterRequestInstanceT] = None, + incoming: Optional[ToolStateJobInstancePopulatedT] = None, history: Optional[History] = None, job_params=None, rerun_remap_job_id: Optional[int] = DEFAULT_RERUN_REMAP_JOB_ID, diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index aef512f61f37..2ad41a75493b 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -15,6 +15,8 @@ List, NamedTuple, Optional, + Tuple, + Union, ) from boltons.iterutils import remap @@ -35,6 +37,10 @@ ToolExecutionCache, ) from galaxy.tools.parameters.workflow_utils import is_runtime_value +from ._types import ( + ToolRequestT, + ToolStateJobInstancePopulatedT, +) if typing.TYPE_CHECKING: from galaxy.tools import Tool @@ -48,10 +54,6 @@ CompletedJobsT = Dict[int, Optional[model.Job]] JobCallbackT = Callable WorkflowResourceParametersT = Dict[str, Any] -# Input dictionary from the API, may include map/reduce instructions -ToolParameterRequestT = Dict[str, Any] -# Input dictionary extracted from a tool request for running a tool individually -ToolParameterRequestInstanceT = Dict[str, Any] DatasetCollectionElementsSliceT = Dict[str, model.DatasetCollectionElement] DEFAULT_USE_CACHED_JOB = False DEFAULT_PREFERRED_OBJECT_STORE_ID: Optional[str] = None @@ -67,8 +69,8 @@ def __init__(self, execution_tracker: "ExecutionTracker"): class MappingParameters(NamedTuple): - param_template: ToolParameterRequestT - param_combinations: List[ToolParameterRequestInstanceT] + param_template: ToolRequestT + param_combinations: List[ToolStateJobInstancePopulatedT] def execute( @@ -242,14 +244,14 @@ def execute_single_job(execution_slice: "ExecutionSlice", completed_job: Optiona class ExecutionSlice: job_index: int - param_combination: ToolParameterRequestInstanceT + param_combination: ToolStateJobInstancePopulatedT dataset_collection_elements: Optional[DatasetCollectionElementsSliceT] history: Optional[model.History] def __init__( self, job_index: int, - param_combination: ToolParameterRequestInstanceT, + param_combination: ToolStateJobInstancePopulatedT, dataset_collection_elements: Optional[DatasetCollectionElementsSliceT] = DEFAULT_DATASET_COLLECTION_ELEMENTS, ): self.job_index = job_index @@ -258,7 +260,16 @@ def __init__( self.history = None +ExecutionErrorsT = Union[str, Exception] + + class ExecutionTracker: + execution_errors: List[ExecutionErrorsT] + successful_jobs: List[model.Job] + output_datasets: List[model.HistoryDatasetAssociation] + output_collections: List[Tuple[str, model.HistoryDatasetCollectionAssociation]] + implicit_collections: Dict[str, model.HistoryDatasetCollectionAssociation] + def __init__( self, trans, @@ -312,8 +323,9 @@ def record_error(self, error): @property def on_text(self): - if self._on_text is None: - collection_names = ["collection %d" % c.hid for c in self.collection_info.collections.values()] + collection_info = self.collection_info + if self._on_text is None and collection_info is not None: + collection_names = ["collection %d" % c.hid for c in collection_info.collections.values()] self._on_text = on_text_for_names(collection_names) return self._on_text @@ -371,19 +383,23 @@ def find_collection(input_dict, input_name): ) subcollection_mapping_type = None if self.is_implicit_input(input_name): - subcollection_mapping_type = self.collection_info.subcollection_mapping_type(input_name) + collection_info = self.collection_info + assert collection_info + subcollection_mapping_type = collection_info.subcollection_mapping_type(input_name) return get_structure( input_collection, collection_type_description, leaf_subcollection_type=subcollection_mapping_type ) def _structure_for_output(self, trans, tool_output): - structure = self.collection_info.structure + collection_info = self.collection_info + assert collection_info + structure = collection_info.structure if hasattr(tool_output, "default_identifier_source"): # Switch the structure for outputs if the output specified a default_identifier_source collection_type_descriptions = trans.app.dataset_collection_manager.collection_type_descriptions - source_collection = self.collection_info.collections.get(tool_output.default_identifier_source) + source_collection = collection_info.collections.get(tool_output.default_identifier_source) if source_collection: collection_type_description = collection_type_descriptions.for_collection_type( source_collection.collection.collection_type @@ -423,10 +439,10 @@ def precreate_output_collections(self, history, params): # collection replaced with a specific dataset. Need to replace this # with the collection and wrap everything up so can evaluate output # label. + collection_info = self.collection_info + assert collection_info trans = self.trans - params.update( - self.collection_info.collections - ) # Replace datasets with source collections for labelling outputs. + params.update(collection_info.collections) # Replace datasets with source collections for labelling outputs. collection_instances = {} implicit_inputs = self.implicit_inputs @@ -498,6 +514,7 @@ def finalize_dataset_collections(self, trans): completed_collections = {} if ( self.completed_jobs + and self.completed_jobs[0] and self.implicit_collection_jobs and len(self.completed_jobs) == len(self.successful_jobs) ): @@ -518,13 +535,16 @@ def finalize_dataset_collections(self, trans): implicit_collection_jobs = implicit_collection.implicit_collection_jobs implicit_collection_jobs.populated_state = "ok" trans.sa_session.add(implicit_collection_jobs) + collection_info = self.collection_info + assert collection_info implicit_collection.collection.finalize( - collection_type_description=self.collection_info.structure.collection_type_description + collection_type_description=collection_info.structure.collection_type_description ) # Mark implicit HDCA as copied - completed_implicit_collection = implicit_collection and completed_collections.get( - implicit_collection.implicit_output_name + implicit_output_name = implicit_collection.implicit_output_name + completed_implicit_collection = ( + implicit_collection and implicit_output_name and completed_collections.get(implicit_output_name) ) if completed_implicit_collection: implicit_collection.copied_from_history_dataset_collection_association_id = ( @@ -538,14 +558,20 @@ def finalize_dataset_collections(self, trans): @property def implicit_inputs(self): - implicit_inputs = list(self.collection_info.collections.items()) + collection_info = self.collection_info + assert collection_info + implicit_inputs = list(collection_info.collections.items()) return implicit_inputs def is_implicit_input(self, input_name): - return input_name in self.collection_info.collections + collection_info = self.collection_info + assert collection_info + return input_name in collection_info.collections def walk_implicit_collections(self): - return self.collection_info.structure.walk_collections(self.implicit_collections) + collection_info = self.collection_info + assert collection_info + return collection_info.structure.walk_collections(self.implicit_collections) def new_execution_slices(self): if self.collection_info is None: @@ -594,7 +620,9 @@ def __init__( # New to track these things for tool output API response in the tool case, # in the workflow case we just write stuff to the database and forget about # it. - self.outputs_by_output_name = collections.defaultdict(list) + self.outputs_by_output_name: Dict[str, List[Union[model.DatasetInstance, model.DatasetCollection]]] = ( + collections.defaultdict(list) + ) def record_success(self, execution_slice, job, outputs): super().record_success(execution_slice, job, outputs) diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index 7e72bf3a9eb1..25d6775446f9 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -4,7 +4,9 @@ from json import dumps from typing import ( + cast, Dict, + Optional, Union, ) @@ -32,12 +34,23 @@ runtime_to_json, ) from .wrapped import flat_to_nested_state +from .._types import ( + InputFormatT, + ParameterValidationErrorsT, + ToolStateDumpedToJsonInternalT, + ToolStateDumpedToJsonT, + ToolStateDumpedToStringsT, + ToolStateJobInstancePopulatedT, + ToolStateJobInstanceT, +) REPLACE_ON_TRUTHY = object() # Some tools use the code tag and access the code base, expecting certain tool parameters to be available here. __all__ = ("DataCollectionToolParameter", "DataToolParameter", "SelectToolParameter") +ToolInputsT = Dict[str, Union[Group, ToolParameter]] + def visit_input_values( inputs, @@ -63,21 +76,20 @@ def visit_input_values( >>> from galaxy.tools.parameters.basic import TextToolParameter, BooleanToolParameter >>> from galaxy.tools.parameters.grouping import Repeat >>> a = TextToolParameter(None, XML('')) - >>> b = Repeat() + >>> b = Repeat('b') >>> c = TextToolParameter(None, XML('')) - >>> d = Repeat() + >>> d = Repeat('d') >>> e = TextToolParameter(None, XML('')) - >>> f = Conditional() + >>> f = Conditional('f') >>> g = BooleanToolParameter(None, XML('')) >>> h = TextToolParameter(None, XML('')) >>> i = TextToolParameter(None, XML('')) >>> j = TextToolParameter(None, XML('')) - >>> b.name = b.title = 'b' + >>> b.title = 'b' >>> b.inputs = dict([ ('c', c), ('d', d) ]) - >>> d.name = d.title = 'd' + >>> d.title = 'd' >>> d.inputs = dict([ ('e', e), ('f', f) ]) >>> f.test_param = g - >>> f.name = 'f' >>> f.cases = [Bunch(value='true', inputs= {'h': h}), Bunch(value='false', inputs= { 'i': i })] >>> >>> def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs): @@ -253,15 +265,37 @@ def check_param(trans, param, incoming_value, param_values, simple_errors=True): return value, error +def params_to_json_internal( + params: ToolInputsT, param_values: ToolStateJobInstancePopulatedT, app +) -> ToolStateDumpedToJsonInternalT: + """Return ToolStateDumpedToJsonT for supplied validated and populated parameters.""" + return cast( + ToolStateDumpedToJsonInternalT, params_to_strings(params, param_values, app, nested=True, use_security=False) + ) + + +def params_to_json(params: ToolInputsT, param_values: ToolStateJobInstancePopulatedT, app) -> ToolStateDumpedToJsonT: + """Return ToolStateDumpedToJsonT for supplied validated and populated parameters.""" + return cast(ToolStateDumpedToJsonT, params_to_strings(params, param_values, app, nested=True, use_security=True)) + + def params_to_strings( - params: Dict[str, Union[Group, ToolParameter]], param_values: Dict, app, nested=False, use_security=False -) -> Dict: + params: ToolInputsT, + param_values: ToolStateJobInstancePopulatedT, + app, + nested=False, + use_security=False, +) -> Union[ToolStateDumpedToJsonT, ToolStateDumpedToJsonInternalT, ToolStateDumpedToStringsT]: """ Convert a dictionary of parameter values to a dictionary of strings suitable for persisting. The `value_to_basic` method of each parameter is called to convert its value to basic types, the result of which is then json encoded (this allowing complex nested parameters and - such). + such). If `nested` this will remain as a sort of JSON-ifiable dictionary + (ToolStateDumpedToJsonT), otherwise these will dumped into strings of the + JSON (ToolStateDumpedToStringsT). If use_security is False, this will return + object references with decoded (integer) IDs, otherwise they will be encoded + strings. """ rval = {} for key, value in param_values.items(): @@ -344,14 +378,14 @@ def replace_dataset_ids(path, key, value): def populate_state( request_context, - inputs, - incoming, - state, - errors=None, + inputs: ToolInputsT, + incoming: ToolStateJobInstanceT, + state: ToolStateJobInstancePopulatedT, + errors: Optional[ParameterValidationErrorsT] = None, context=None, check=True, simple_errors=True, - input_format="legacy", + input_format: InputFormatT = "legacy", ): """ Populates nested state dict from incoming parameter values. @@ -361,24 +395,21 @@ def populate_state( >>> from galaxy.tools.parameters.grouping import Repeat >>> trans = Bunch(workflow_building_mode=False) >>> a = TextToolParameter(None, XML('')) - >>> b = Repeat() + >>> b = Repeat('b') >>> b.min = 0 >>> b.max = 1 >>> c = TextToolParameter(None, XML('')) - >>> d = Repeat() + >>> d = Repeat('d') >>> d.min = 0 >>> d.max = 1 >>> e = TextToolParameter(None, XML('')) - >>> f = Conditional() + >>> f = Conditional('f') >>> g = BooleanToolParameter(None, XML('')) >>> h = TextToolParameter(None, XML('')) >>> i = TextToolParameter(None, XML('')) - >>> b.name = 'b' >>> b.inputs = dict([('c', c), ('d', d)]) - >>> d.name = 'd' >>> d.inputs = dict([('e', e), ('f', f)]) >>> f.test_param = g - >>> f.name = 'f' >>> f.cases = [Bunch(value='true', inputs= { 'h': h }), Bunch(value='false', inputs= { 'i': i })] >>> inputs = dict([('a',a),('b',b)]) >>> flat = dict([('a', 1), ('b_0|c', 2), ('b_0|d_0|e', 3), ('b_0|d_0|f|h', 4), ('b_0|d_0|f|g', True)]) @@ -426,77 +457,84 @@ def populate_state( state[input.name] = input.get_initial_value(request_context, context) group_state = state[input.name] if input.type == "repeat": - if len(incoming[input.name]) > input.max or len(incoming[input.name]) < input.min: - errors[input.name] = "The number of repeat elements is outside the range specified by the tool." + repeat_input = cast(Repeat, input) + if ( + len(incoming[repeat_input.name]) > repeat_input.max + or len(incoming[repeat_input.name]) < repeat_input.min + ): + errors[repeat_input.name] = ( + "The number of repeat elements is outside the range specified by the tool." + ) else: del group_state[:] - for rep in incoming[input.name]: - new_state = {} + for rep in incoming[repeat_input.name]: + new_state: ToolStateJobInstancePopulatedT = {} group_state.append(new_state) - new_errors = {} + repeat_errors: ParameterValidationErrorsT = {} populate_state( request_context, - input.inputs, + repeat_input.inputs, rep, new_state, - new_errors, + repeat_errors, context=context, check=check, simple_errors=simple_errors, input_format=input_format, ) - if new_errors: - errors[input.name] = new_errors + if repeat_errors: + errors[repeat_input.name] = repeat_errors elif input.type == "conditional": - test_param_value = incoming.get(input.name, {}).get(input.test_param.name) + conditional_input = cast(Conditional, input) + test_param = cast(ToolParameter, conditional_input.test_param) + test_param_value = incoming.get(conditional_input.name, {}).get(test_param.name) value, error = ( - check_param( - request_context, input.test_param, test_param_value, context, simple_errors=simple_errors - ) + check_param(request_context, test_param, test_param_value, context, simple_errors=simple_errors) if check else [test_param_value, None] ) if error: - errors[input.test_param.name] = error + errors[test_param.name] = error else: try: - current_case = input.get_current_case(value) - group_state = state[input.name] = {} - new_errors = {} + current_case = conditional_input.get_current_case(value) + group_state = state[conditional_input.name] = {} + cast_errors: ParameterValidationErrorsT = {} populate_state( request_context, - input.cases[current_case].inputs, - incoming.get(input.name), + conditional_input.cases[current_case].inputs, + cast(ToolStateJobInstanceT, incoming.get(conditional_input.name)), group_state, - new_errors, + cast_errors, context=context, check=check, simple_errors=simple_errors, input_format=input_format, ) - if new_errors: - errors[input.name] = new_errors + if cast_errors: + errors[conditional_input.name] = cast_errors group_state["__current_case__"] = current_case except Exception: - errors[input.test_param.name] = "The selected case is unavailable/invalid." - group_state[input.test_param.name] = value + errors[test_param.name] = "The selected case is unavailable/invalid." + group_state[test_param.name] = value elif input.type == "section": - new_errors = {} + section_input = cast(Section, input) + section_errors: ParameterValidationErrorsT = {} populate_state( request_context, - input.inputs, - incoming.get(input.name), + section_input.inputs, + cast(ToolStateJobInstanceT, incoming.get(section_input.name)), group_state, - new_errors, + section_errors, context=context, check=check, simple_errors=simple_errors, input_format=input_format, ) - if new_errors: - errors[input.name] = new_errors + if section_errors: + errors[section_input.name] = section_errors elif input.type == "upload_dataset": raise NotImplementedError @@ -516,7 +554,15 @@ def populate_state( def _populate_state_legacy( - request_context, inputs, incoming, state, errors, prefix="", context=None, check=True, simple_errors=True + request_context, + inputs: ToolInputsT, + incoming: ToolStateJobInstanceT, + state: ToolStateJobInstancePopulatedT, + errors, + prefix="", + context=None, + check=True, + simple_errors=True, ): if context is None: context = flat_to_nested_state(incoming) @@ -527,22 +573,23 @@ def _populate_state_legacy( group_state = state[input.name] group_prefix = f"{key}|" if input.type == "repeat": + repeat_input = cast(Repeat, input) rep_index = 0 del group_state[:] while True: rep_prefix = "%s_%d" % (key, rep_index) - rep_min_default = input.default if input.default > input.min else input.min + rep_min_default = repeat_input.default if repeat_input.default > repeat_input.min else repeat_input.min if ( not any(incoming_key.startswith(rep_prefix) for incoming_key in incoming.keys()) and rep_index >= rep_min_default ): break - if rep_index < input.max: - new_state = {"__index__": rep_index} + if rep_index < repeat_input.max: + new_state: ToolStateJobInstancePopulatedT = {"__index__": rep_index} group_state.append(new_state) _populate_state_legacy( request_context, - input.inputs, + repeat_input.inputs, incoming, new_state, errors, @@ -553,13 +600,21 @@ def _populate_state_legacy( ) rep_index += 1 elif input.type == "conditional": - if input.value_ref and not input.value_ref_in_group: - test_param_key = prefix + input.test_param.name + conditional_input = cast(Conditional, input) + test_param = cast(ToolParameter, conditional_input.test_param) + if conditional_input.value_ref and not conditional_input.value_ref_in_group: + test_param_key = prefix + test_param.name else: - test_param_key = group_prefix + input.test_param.name - test_param_value = incoming.get(test_param_key, group_state.get(input.test_param.name)) + test_param_key = group_prefix + test_param.name + test_param_value = incoming.get(test_param_key, group_state.get(test_param.name)) value, error = ( - check_param(request_context, input.test_param, test_param_value, context, simple_errors=simple_errors) + check_param( + request_context, + test_param, + test_param_value, + context, + simple_errors=simple_errors, + ) if check else [test_param_value, None] ) @@ -567,11 +622,11 @@ def _populate_state_legacy( errors[test_param_key] = error else: try: - current_case = input.get_current_case(value) - group_state = state[input.name] = {} + current_case = conditional_input.get_current_case(value) + group_state = state[conditional_input.name] = cast(ToolStateJobInstancePopulatedT, {}) _populate_state_legacy( request_context, - input.cases[current_case].inputs, + conditional_input.cases[current_case].inputs, incoming, group_state, errors, @@ -583,11 +638,12 @@ def _populate_state_legacy( group_state["__current_case__"] = current_case except Exception: errors[test_param_key] = "The selected case is unavailable/invalid." - group_state[input.test_param.name] = value + group_state[test_param.name] = value elif input.type == "section": + section_input = cast(Section, input) _populate_state_legacy( request_context, - input.inputs, + section_input.inputs, incoming, group_state, errors, @@ -597,20 +653,21 @@ def _populate_state_legacy( simple_errors=simple_errors, ) elif input.type == "upload_dataset": - file_count = input.get_file_count(request_context, context) + dataset_input = cast(UploadDataset, input) + file_count = dataset_input.get_file_count(request_context, context) while len(group_state) > file_count: del group_state[-1] while file_count > len(group_state): - new_state = {"__index__": len(group_state)} - for upload_item in input.inputs.values(): - new_state[upload_item.name] = upload_item.get_initial_value(request_context, context) - group_state.append(new_state) + new_state_upload: ToolStateJobInstancePopulatedT = {"__index__": len(group_state)} + for upload_item in dataset_input.inputs.values(): + new_state_upload[upload_item.name] = upload_item.get_initial_value(request_context, context) + group_state.append(new_state_upload) for rep_index, rep_state in enumerate(group_state): rep_index = rep_state.get("__index__", rep_index) rep_prefix = "%s_%d|" % (key, rep_index) _populate_state_legacy( request_context, - input.inputs, + dataset_input.inputs, incoming, rep_state, errors, diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 5eeba8aaecaa..9669b62771e9 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -174,6 +174,7 @@ class ToolParameter(UsesDictVisibleKeys): >>> assert sorted(p.to_dict(trans).items()) == [('argument', '--parameter-name'), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'ToolParameter'), ('name', 'parameter_name'), ('optional', False), ('refresh_on_change', False), ('type', 'text'), ('value', None)] """ + name: str dict_collection_visible_keys = ["name", "argument", "type", "label", "help", "refresh_on_change"] def __init__(self, tool, input_source, context=None): diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 9962e61cf624..26b4e171c0de 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -6,6 +6,7 @@ import logging import os import unicodedata +from math import inf from typing import ( Any, Callable, @@ -35,6 +36,7 @@ if TYPE_CHECKING: from galaxy.tools import Tool from galaxy.tools.parameter.basic import ToolParameter + from galaxy.tools.parameters import ToolInputsT log = logging.getLogger(__name__) URI_PREFIXES = [ @@ -58,9 +60,10 @@ class Group(UsesDictVisibleKeys): dict_collection_visible_keys = ["name", "type"] type: str + name: str - def __init__(self): - self.name = None + def __init__(self, name: str): + self.name = name @property def visible(self): @@ -94,15 +97,18 @@ def to_dict(self, trans): class Repeat(Group): dict_collection_visible_keys = ["name", "type", "title", "help", "default", "min", "max"] type = "repeat" + inputs: "ToolInputsT" + min: int + max: float - def __init__(self): - Group.__init__(self) + def __init__(self, name: str): + Group.__init__(self, name) self._title = None - self.inputs = None + self.inputs = {} self.help = None self.default = 0 - self.min = None - self.max = None + self.min = 0 + self.max = inf @property def title(self): @@ -186,11 +192,12 @@ def input_to_dict(input): class Section(Group): dict_collection_visible_keys = ["name", "type", "title", "help", "expanded"] type = "section" + inputs: "ToolInputsT" - def __init__(self): - Group.__init__(self) + def __init__(self, name: str): + Group.__init__(self, name) self.title = None - self.inputs = None + self.inputs = {} self.help = None self.expanded = False @@ -266,11 +273,12 @@ class Dataset(Bunch): class UploadDataset(Group): type = "upload_dataset" + inputs: "ToolInputsT" - def __init__(self): - Group.__init__(self) + def __init__(self, name: str): + Group.__init__(self, name) self.title = None - self.inputs = None + self.inputs = {} self.file_type_name = "file_type" self.default_file_type = "txt" self.file_type_to_ext = {"auto": self.default_file_type} @@ -735,12 +743,13 @@ def get_filenames(context): class Conditional(Group): type = "conditional" value_from: Callable[[ExpressionContext, "Conditional", "Tool"], Mapping[str, str]] + cases: List["ConditionalWhen"] - def __init__(self): - Group.__init__(self) + def __init__(self, name: str): + Group.__init__(self, name) self.test_param: Optional[ToolParameter] = None self.cases = [] - self.value_ref = None + self.value_ref: Optional[str] = None self.value_ref_in_group = True # When our test_param is not part of the conditional Group, this is False @property @@ -761,7 +770,7 @@ def get_current_case(self, value): def value_to_basic(self, value, app, use_security=False): if self.test_param is None: raise Exception("Must set 'test_param' attribute to use.") - rval = {} + rval: Dict[str, Any] = {} rval[self.test_param.name] = self.test_param.value_to_basic(value[self.test_param.name], app) current_case = rval["__current_case__"] = self.get_current_case(value[self.test_param.name]) for input in self.cases[current_case].inputs.values(): diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py index fee9f6d6079e..f2d8ba1a68d1 100644 --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -22,6 +22,10 @@ from galaxy.util import permutations from . import visit_input_values from .wrapped import process_key +from .._types import ( + ToolRequestT, + ToolStateJobInstanceT, +) log = logging.getLogger(__name__) @@ -154,10 +158,10 @@ def is_batch(value): return WorkflowParameterExpansion(param_combinations, params_keys, input_combinations) -ExpandedT = Tuple[List[Dict[str, Any]], Optional[matching.MatchingCollections]] +ExpandedT = Tuple[List[ToolStateJobInstanceT], Optional[matching.MatchingCollections]] -def expand_meta_parameters(trans, tool, incoming) -> ExpandedT: +def expand_meta_parameters(trans, tool, incoming: ToolRequestT) -> ExpandedT: """ Take in a dictionary of raw incoming parameters and expand to a list of expanded incoming parameters (one set of parameters per tool diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 6aebebe5ec3c..9eb5efb40938 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -71,7 +71,7 @@ JobIndexViewEnum, JobsService, ) -from galaxy.work.context import WorkRequestContext +from galaxy.work.context import proxy_work_context_for_history log = logging.getLogger(__name__) @@ -478,10 +478,8 @@ def search( for k, v in payload.__annotations__.items(): if k.startswith("files_") or k.startswith("__files_"): inputs[k] = v - request_context = WorkRequestContext(app=trans.app, user=trans.user, history=trans.history) - all_params, all_errors, _, _ = tool.expand_incoming( - trans=trans, incoming=inputs, request_context=request_context - ) + request_context = proxy_work_context_for_history(trans) + all_params, all_errors, _, _ = tool.expand_incoming(request_context, incoming=inputs) if any(all_errors): return [] params_dump = [tool.params_to_strings(param, trans.app, nested=True) for param in all_params] diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 5de6e9d9b47c..e63063f871a4 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -83,6 +83,7 @@ from galaxy.structured_app import StructuredApp from galaxy.tool_shed.galaxy_install.install_manager import InstallRepositoryManager from galaxy.tools import recommendations +from galaxy.tools._types import ParameterValidationErrorsT from galaxy.tools.parameters import populate_state from galaxy.tools.parameters.workflow_utils import workflow_building_modes from galaxy.web import ( @@ -537,7 +538,7 @@ def build_module(self, trans: GalaxyWebTransaction, payload=None): module = module_factory.from_dict(trans, payload, from_tool_form=True) if "tool_state" not in payload: module_state: Dict[str, Any] = {} - errors: Dict[str, str] = {} + errors: ParameterValidationErrorsT = {} populate_state(trans, module.get_inputs(), inputs, module_state, errors=errors, check=True) module.recover_state(module_state, from_tool_form=True) module.check_and_update_state() diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index 025fb7aac920..81db0f7e1c6c 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -175,7 +175,7 @@ def set_history(self, history): def proxy_work_context_for_history( trans: ProvidesHistoryContext, history: Optional[History] = None, workflow_building_mode=False -): +) -> WorkRequestContext: """Create a WorkContext for supplied context with potentially different history. This provides semi-structured access to a transaction/work context with a supplied target diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 9a35e4f9b07b..c6a6af2f9090 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -30,6 +30,7 @@ ) from galaxy.job_execution.actions.post import ActionBox from galaxy.model import ( + Job, PostJobAction, Workflow, WorkflowInvocationStep, @@ -1209,8 +1210,7 @@ def get_inputs(self): option[2] = True input_parameter_type.static_options[i] = tuple(option) - parameter_type_cond = Conditional() - parameter_type_cond.name = "parameter_definition" + parameter_type_cond = Conditional("parameter_definition") parameter_type_cond.test_param = input_parameter_type cases = [] @@ -1262,8 +1262,7 @@ def get_inputs(self): input_default_value = ColorToolParameter(None, default_source) optional_value = optional_param(optional) - optional_cond = Conditional() - optional_cond.name = "optional" + optional_cond = Conditional("optional") optional_cond.test_param = optional_value when_this_type = ConditionalWhen() @@ -1276,8 +1275,7 @@ def get_inputs(self): name="specify_default", label="Specify a default value", type="boolean", checked=specify_default_checked ) specify_default = BooleanToolParameter(None, specify_default_source) - specify_default_cond = Conditional() - specify_default_cond.name = "specify_default" + specify_default_cond = Conditional("specify_default") specify_default_cond.test_param = specify_default when_specify_default_true = ConditionalWhen() @@ -1350,9 +1348,8 @@ def get_inputs(self): "selected": restrict_how_value == "staticSuggestions", }, ] - restrictions_cond = Conditional() + restrictions_cond = Conditional("restrictions") restrictions_how = SelectToolParameter(None, restrict_how_source) - restrictions_cond.name = "restrictions" restrictions_cond.test_param = restrictions_how when_restrict_none = ConditionalWhen() @@ -2290,19 +2287,7 @@ def callback(input, prefixed_name: str, **kwargs): param_combinations.append(execution_state.inputs) complete = False - completed_jobs = {} - for i, param in enumerate(param_combinations): - if use_cached_job: - completed_jobs[i] = tool.job_search.by_tool_input( - trans=trans, - tool_id=tool.id, - tool_version=tool.version, - param=param, - param_dump=tool.params_to_strings(param, trans.app, nested=True), - job_state=None, - ) - else: - completed_jobs[i] = None + completed_jobs: Dict[int, Optional[Job]] = tool.completed_jobs(trans, use_cached_job, param_combinations) try: mapping_params = MappingParameters(tool_state.inputs, param_combinations) max_num_jobs = progress.maximum_jobs_to_schedule_or_none diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index 52d25566b638..e571b5fd8898 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -57,8 +57,7 @@ def test_simple_evaluation(self): assert command_line == "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat" def test_repeat_evaluation(self): - repeat = Repeat() - repeat.name = "r" + repeat = Repeat("r") repeat.inputs = {"thresh": self.tool.test_thresh_param()} self.tool.set_params({"r": repeat}) self.job.parameters = [ @@ -85,8 +84,7 @@ def test_conditional_evaluation(self): select_xml = XML("""""") parameter = SelectToolParameter(self.tool, select_xml) - conditional = Conditional() - conditional.name = "c" + conditional = Conditional("c") conditional.test_param = parameter when = ConditionalWhen() when.inputs = {"thresh": self.tool.test_thresh_param()}