diff --git a/.gitignore b/.gitignore index 1a6b7d7f2a07..c907531198ef 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,7 @@ doc/build doc/schema.md doc/source/admin/config_logging_default_yaml.rst doc/source/dev/schema.md +doc/source/dev/plantuml.jar client/docs/dist # Webpack stats diff --git a/doc/source/dev/image.Makefile b/doc/source/dev/image.Makefile new file mode 100644 index 000000000000..75cb5e1910e4 --- /dev/null +++ b/doc/source/dev/image.Makefile @@ -0,0 +1,11 @@ +MINDMAPS := $(wildcard *.mindmap.yml) +INPUTS := $(wildcard *.plantuml.txt) +OUTPUTS := $(INPUTS:.txt=.svg) + +all: plantuml.jar $(MINDMAPS) $(OUTPUTS) + +$(OUTPUTS): $(INPUTS) $(MINDMAPS) + java -jar plantuml.jar -c plantuml_options.txt -tsvg $(INPUTS) + +plantuml.jar: + wget http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar || curl --output plantuml.jar http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar diff --git a/doc/source/dev/plantuml_options.txt b/doc/source/dev/plantuml_options.txt new file mode 100644 index 000000000000..70424ef26736 --- /dev/null +++ b/doc/source/dev/plantuml_options.txt @@ -0,0 +1,51 @@ +' skinparam handwritten true +' skinparam roundcorner 20 + +skinparam class { + ArrowFontColor DarkOrange + BackgroundColor #FFEFD5 + ArrowColor Orange + BorderColor DarkOrange +} + +skinparam object { + ArrowFontColor DarkOrange + BackgroundColor #FFEFD5 + BackgroundColor #FFEFD5 + ArrowColor Orange + BorderColor DarkOrange +} + +skinparam ComponentBackgroundColor #FFEFD5 +skinparam ComponentBorderColor DarkOrange + +skinparam DatabaseBackgroundColor #FFEFD5 +skinparam DatabaseBorderColor DarkOrange + +skinparam StorageBackgroundColor #FFEFD5 +skinparam StorageBorderColor DarkOrange + +skinparam QueueBackgroundColor #FFEFD5 +skinparam QueueBorderColor DarkOrange + +skinparam note { + BackgroundColor #FFEFD5 + BorderColor #BF5700 +} + +skinparam sequence { + ArrowColor Orange + ArrowFontColor DarkOrange + ActorBorderColor DarkOrange + ActorBackgroundColor #FFEFD5 + + ParticipantBorderColor DarkOrange + ParticipantBackgroundColor #FFEFD5 + + LifeLineBorderColor DarkOrange + LifeLineBackgroundColor #FFEFD5 + + DividerBorderColor DarkOrange + GroupBorderColor DarkOrange +} + diff --git a/doc/source/dev/plantuml_style.txt b/doc/source/dev/plantuml_style.txt new file mode 100644 index 000000000000..18911d622b75 --- /dev/null +++ b/doc/source/dev/plantuml_style.txt @@ -0,0 +1,9 @@ + diff --git a/doc/source/dev/tool_state.md b/doc/source/dev/tool_state.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/doc/source/dev/tool_state_api.plantuml.txt b/doc/source/dev/tool_state_api.plantuml.txt new file mode 100644 index 000000000000..060b4c5bdb9e --- /dev/null +++ b/doc/source/dev/tool_state_api.plantuml.txt @@ -0,0 +1,17 @@ +@startuml +'!include plantuml_options.txt +participant "API Request" as apireq +boundary "Jobs API" as api +participant "Job Service" as service +database Database as database +queue TaskQueue as queue +apireq -> api : HTTP JSON +api -> service : To boundary +service -> service : Build RequestToolState +service -> service : Validate RequestToolState (pydantic) +service -> service : decode() RequestToolState \ninto RequestInternalToolState +service -> database : Serialize RequestInternalToolState +service -> queue : Queue QueueJobs with reference to\npersisted RequestInternalToolState +service -> api : JobCreateResponse\n (pydantic model) +api -> apireq : JobCreateResponse\n (as json) +@enduml diff --git a/doc/source/dev/tool_state_state_classes.plantuml.txt b/doc/source/dev/tool_state_state_classes.plantuml.txt new file mode 100644 index 000000000000..612c13d8e683 --- /dev/null +++ b/doc/source/dev/tool_state_state_classes.plantuml.txt @@ -0,0 +1,41 @@ +@startuml +!include plantuml_options.txt + +package galaxy.tool_util.parameters.state { + +class ToolState { +state_representation: str +input_state: Dict[str, Any] ++ validate(input_models: ToolParameterBundle) ++ {abstract} _to_base_model(input_models: ToolParameterBundle): Optional[Type[BaseModel]] +} + +class RequestToolState { +state_representation = "request" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form \n{src: "hda", id: }.\n Allow mapping/reduce constructs. + +class RequestInternalToolState { +state_representation = "request_internal" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form \n{src: "hda", id: }.\n Allow mapping/reduce constructs. + +class JobInternalToolState { +state_representation = "job_internal" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] + +} +note bottom: Object references of the form \n{src: "hda", id: }.\n Mapping constructs expanded out.\n (Defaults are inserted?) + +ToolState <|-- RequestToolState +ToolState <|-- RequestInternalToolState +ToolState <|-- JobInternalToolState + +RequestToolState - RequestInternalToolState : decode > + +RequestInternalToolState o-- JobInternalToolState : expand > + +} +@enduml \ No newline at end of file diff --git a/lib/galaxy/config/schemas/tool_shed_config_schema.yml b/lib/galaxy/config/schemas/tool_shed_config_schema.yml index 42ea164d9d5f..2a1eee2b533f 100644 --- a/lib/galaxy/config/schemas/tool_shed_config_schema.yml +++ b/lib/galaxy/config/schemas/tool_shed_config_schema.yml @@ -102,6 +102,13 @@ mapping: the repositories and tools within the Tool Shed given that you specify the following two config options. + tool_state_cache_dir: + type: str + default: database/tool_state_cache + required: false + desc: | + Cache directory for tool state. + repo_name_boost: type: float default: 0.9 diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py index 674b3abcc357..61669314ae26 100644 --- a/lib/galaxy/tool_util/cwl/parser.py +++ b/lib/galaxy/tool_util/cwl/parser.py @@ -144,6 +144,10 @@ def galaxy_id(self) -> str: tool_id = tool_id[1:] return tool_id + @abstractmethod + def input_fields(self) -> list: + """Return InputInstance objects describing mapping to Galaxy inputs.""" + @abstractmethod def input_instances(self): """Return InputInstance objects describing mapping to Galaxy inputs.""" @@ -236,7 +240,7 @@ def label(self): else: return "" - def input_fields(self): + def input_fields(self) -> list: input_records_schema = self._eval_schema(self._tool.inputs_record_schema) if input_records_schema["type"] != "record": raise Exception("Unhandled CWL tool input structure") diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py new file mode 100644 index 000000000000..048bc546fb20 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/__init__.py @@ -0,0 +1,99 @@ +from .convert import ( + decode, + encode, +) +from .factory import ( + from_input_source, + input_models_for_pages, + input_models_for_tool_source, + input_models_from_json, + tool_parameter_bundle_from_json, +) +from .json import to_json_schema_string +from .models import ( + BooleanParameterModel, + ColorParameterModel, + ConditionalParameterModel, + ConditionalWhen, + CwlBooleanParameterModel, + CwlDirectoryParameterModel, + CwlFileParameterModel, + CwlFloatParameterModel, + CwlIntegerParameterModel, + CwlNullParameterModel, + CwlStringParameterModel, + CwlUnionParameterModel, + DataCollectionParameterModel, + DataParameterModel, + FloatParameterModel, + HiddenParameterModel, + IntegerParameterModel, + LabelValue, + RepeatParameterModel, + RulesParameterModel, + SelectParameterModel, + TextParameterModel, + ToolParameterBundle, + ToolParameterBundleModel, + ToolParameterModel, + ToolParameterT, + validate_against_model, + validate_internal_request, + validate_request, + validate_test_case, +) +from .state import ( + JobInternalToolState, + RequestInternalToolState, + RequestToolState, + TestCaseToolState, + ToolState, +) +from .visitor import visit_input_values + +__all__ = ( + "from_input_source", + "input_models_for_pages", + "input_models_for_tool_source", + "tool_parameter_bundle_from_json", + "input_models_from_json", + "JobInternalToolState", + "ToolParameterBundle", + "ToolParameterBundleModel", + "ToolParameterModel", + "IntegerParameterModel", + "BooleanParameterModel", + "CwlFileParameterModel", + "CwlFloatParameterModel", + "CwlIntegerParameterModel", + "CwlStringParameterModel", + "CwlNullParameterModel", + "CwlUnionParameterModel", + "CwlBooleanParameterModel", + "CwlDirectoryParameterModel", + "TextParameterModel", + "FloatParameterModel", + "HiddenParameterModel", + "ColorParameterModel", + "RulesParameterModel", + "DataParameterModel", + "DataCollectionParameterModel", + "LabelValue", + "SelectParameterModel", + "ConditionalParameterModel", + "ConditionalWhen", + "RepeatParameterModel", + "validate_against_model", + "validate_internal_request", + "validate_request", + "validate_test_case", + "ToolState", + "TestCaseToolState", + "ToolParameterT", + "to_json_schema_string", + "RequestToolState", + "RequestInternalToolState", + "visit_input_values", + "decode", + "encode", +) diff --git a/lib/galaxy/tool_util/parameters/_types.py b/lib/galaxy/tool_util/parameters/_types.py new file mode 100644 index 000000000000..59d42279df72 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/_types.py @@ -0,0 +1,53 @@ +"""Type utilities for building pydantic models for tool parameters. + +Lots of mypy exceptions in here - this code is all well tested and the exceptions +are fine otherwise because we're using the typing system to interact with pydantic +and build runtime models not to use mypy to type check static code. +""" + +from typing import ( + Any, + cast, + Generic, + List, + Optional, + Type, + Union, +) + +# https://stackoverflow.com/questions/56832881/check-if-a-field-is-typing-optional +# Python >= 3.8 +try: + from typing import get_args # type: ignore[attr-defined,unused-ignore] + from typing import get_origin # type: ignore[attr-defined,unused-ignore] +# Compatibility +except ImportError: + + def get_args(tp: Any) -> tuple: + return getattr(tp, "__args__", ()) if tp is not Generic else Generic # type: ignore[return-value,assignment,unused-ignore] + + def get_origin(tp: Any) -> Optional[Any]: # type: ignore[no-redef,unused-ignore] + return getattr(tp, "__origin__", None) + + +def optional_if_needed(type: Type, is_optional: bool) -> Type: + return_type: Type = type + if is_optional: + return_type = Optional[type] # type: ignore[assignment] + return return_type + + +def union_type(args: List[Type]) -> Type: + return Union[tuple(args)] # type: ignore[return-value] + + +def list_type(arg: Type) -> Type: + return List[arg] # type: ignore[valid-type] + + +def cast_as_type(arg) -> Type: + return cast(Type, arg) + + +def is_optional(field) -> bool: + return get_origin(field) is Union and type(None) in get_args(field) diff --git a/lib/galaxy/tool_util/parameters/case.py b/lib/galaxy/tool_util/parameters/case.py new file mode 100644 index 000000000000..036c2866a9c9 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/case.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +from typing import ( + Any, + Dict, + List, +) + +from .models import ( + DataCollectionParameterModel, + DataParameterModel, + FloatParameterModel, + IntegerParameterModel, + parameters_by_name, + ToolParameterBundle, + ToolParameterT, +) +from .state import TestCaseToolState + + +@dataclass +class TestCaseStateAndWarnings: + tool_state: TestCaseToolState + warnings: List[str] + + +def legacy_from_string(parameter: ToolParameterT, value: str, warnings: List[str], profile: str) -> Any: + """Convert string values in XML test cases into typed variants. + + This should only be used when parsing XML test cases into a TestCaseToolState object. + We have to maintain backward compatibility on these for older Galaxy tool profile versions. + """ + is_string = isinstance(value, str) + result_value: Any = value + if is_string and isinstance(parameter, (IntegerParameterModel,)): + warnings.append( + f"Implicitly converted {parameter.name} to an integer from a string value, please use 'value_json' to define this test input parameter value instead." + ) + result_value = int(value) + elif is_string and isinstance(parameter, (FloatParameterModel,)): + warnings.append( + f"Implicitly converted {parameter.name} to a floating point number from a string value, please use 'value_json' to define this test input parameter value instead." + ) + result_value = float(value) + return result_value + + +def test_case_state( + test_dict: Dict[str, Any], tool_parameter_bundle: ToolParameterBundle, profile: str +) -> TestCaseStateAndWarnings: + warnings: List[str] = [] + inputs = test_dict["inputs"] + state = {} + by_name = parameters_by_name(tool_parameter_bundle) + for input in inputs: + input_name = input["name"] + if input_name not in by_name: + raise Exception(f"Cannot find tool parameter for {input_name}") + tool_parameter_model = by_name[input_name] + input_value = input["value"] + input_value = legacy_from_string(tool_parameter_model, input_value, warnings, profile) + if isinstance(tool_parameter_model, (DataParameterModel,)): + pass + elif isinstance(tool_parameter_model, (DataCollectionParameterModel,)): + pass + + state[input_name] = input_value + + tool_state = TestCaseToolState(state) + tool_state.validate(tool_parameter_bundle) + return TestCaseStateAndWarnings(tool_state, warnings) diff --git a/lib/galaxy/tool_util/parameters/convert.py b/lib/galaxy/tool_util/parameters/convert.py new file mode 100644 index 000000000000..14caed47e92c --- /dev/null +++ b/lib/galaxy/tool_util/parameters/convert.py @@ -0,0 +1,73 @@ +"""Utilities for converting between request states. +""" + +from typing import ( + Any, + Callable, +) + +from .models import ( + ToolParameterBundle, + ToolParameterT, +) +from .state import ( + RequestInternalToolState, + RequestToolState, +) +from .visitor import ( + visit_input_values, + VISITOR_NO_REPLACEMENT, +) + + +def decode( + external_state: RequestToolState, input_models: ToolParameterBundle, decode_id: Callable[[str], int] +) -> RequestInternalToolState: + """Prepare an external representation of tool state (request) for storing in the database (request_internal).""" + + external_state.validate(input_models) + + def decode_callback(parameter: ToolParameterT, value: Any): + if parameter.parameter_type == "gx_data": + assert isinstance(value, dict), str(value) + assert "id" in value + decoded_dict = value.copy() + decoded_dict["id"] = decode_id(value["id"]) + return decoded_dict + else: + return VISITOR_NO_REPLACEMENT + + internal_state_dict = visit_input_values( + input_models, + external_state, + decode_callback, + ) + + internal_request_state = RequestInternalToolState(internal_state_dict) + internal_request_state.validate(input_models) + return internal_request_state + + +def encode( + external_state: RequestInternalToolState, input_models: ToolParameterBundle, encode_id: Callable[[int], str] +) -> RequestToolState: + """Prepare an external representation of tool state (request) for storing in the database (request_internal).""" + + def encode_callback(parameter: ToolParameterT, value: Any): + if parameter.parameter_type == "gx_data": + assert isinstance(value, dict), str(value) + assert "id" in value + encoded_dict = value.copy() + encoded_dict["id"] = encode_id(value["id"]) + return encoded_dict + else: + return VISITOR_NO_REPLACEMENT + + request_state_dict = visit_input_values( + input_models, + external_state, + encode_callback, + ) + request_state = RequestToolState(request_state_dict) + request_state.validate(input_models) + return request_state diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py new file mode 100644 index 000000000000..db11c301473c --- /dev/null +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -0,0 +1,296 @@ +from typing import ( + Any, + Dict, + List, + Optional, +) + +from galaxy.tool_util.parser.cwl import CwlInputSource +from galaxy.tool_util.parser.interface import ( + InputSource, + PageSource, + PagesSource, + ToolSource, +) +from .models import ( + BooleanParameterModel, + ColorParameterModel, + ConditionalParameterModel, + ConditionalWhen, + CwlBooleanParameterModel, + CwlDirectoryParameterModel, + CwlFileParameterModel, + CwlFloatParameterModel, + CwlIntegerParameterModel, + CwlNullParameterModel, + CwlStringParameterModel, + CwlUnionParameterModel, + DataCollectionParameterModel, + DataParameterModel, + FloatParameterModel, + HiddenParameterModel, + IntegerParameterModel, + LabelValue, + RepeatParameterModel, + RulesParameterModel, + SelectParameterModel, + SectionParameterModel, + TextParameterModel, + ToolParameterBundle, + ToolParameterBundleModel, + ToolParameterT, +) + + +class ParameterDefinitionError(Exception): + pass + + +def get_color_value(input_source: InputSource) -> str: + return input_source.get("value", "#000000") + + +def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: + input_type = input_source.parse_input_type() + if input_type == "param": + param_type = input_source.get("type") + if param_type == "integer": + optional = input_source.parse_optional() + value = input_source.get("value") + int_value: Optional[int] + if value: + int_value = int(value) + elif optional: + int_value = None + else: + raise ParameterDefinitionError() + return IntegerParameterModel(name=input_source.parse_name(), optional=optional, value=int_value) + elif param_type == "boolean": + nullable = input_source.parse_optional() + checked = input_source.get_bool("checked", None if nullable else False) + return BooleanParameterModel( + name=input_source.parse_name(), + optional=nullable, + value=checked, + ) + elif param_type == "text": + optional = input_source.parse_optional() + return TextParameterModel( + name=input_source.parse_name(), + optional=optional, + ) + elif param_type == "float": + optional = input_source.parse_optional() + value = input_source.get("value") + float_value: Optional[float] + if value: + float_value = float(value) + elif optional: + float_value = None + else: + raise ParameterDefinitionError() + return FloatParameterModel( + name=input_source.parse_name(), + optional=optional, + value=float_value, + ) + elif param_type == "hidden": + optional = input_source.parse_optional() + return HiddenParameterModel( + name=input_source.parse_name(), + optional=optional, + ) + elif param_type == "color": + optional = input_source.parse_optional() + return ColorParameterModel( + name=input_source.parse_name(), + optional=optional, + value=get_color_value(input_source), + ) + elif param_type == "rules": + return RulesParameterModel( + name=input_source.parse_name(), + ) + elif param_type == "data": + optional = input_source.parse_optional() + multiple = input_source.get_bool("multiple", False) + return DataParameterModel( + name=input_source.parse_name(), + optional=optional, + multiple=multiple, + ) + elif param_type == "data_collection": + optional = input_source.parse_optional() + return DataCollectionParameterModel( + name=input_source.parse_name(), + optional=optional, + ) + elif param_type == "select": + # Function... example in devteam cummeRbund. + optional = input_source.parse_optional() + dynamic_options = input_source.get("dynamic_options", None) + dynamic_options_elem = input_source.parse_dynamic_options_elem() + multiple = input_source.get_bool("multiple", False) + is_static = dynamic_options is None and dynamic_options_elem is None + options: Optional[List[LabelValue]] = None + if is_static: + options = [] + for option_label, option_value, selected in input_source.parse_static_options(): + options.append(LabelValue(label=option_label, value=option_value, selected=selected)) + return SelectParameterModel( + name=input_source.parse_name(), + optional=optional, + options=options, + multiple=multiple, + ) + else: + raise Exception(f"Unknown Galaxy parameter type {param_type}") + elif input_type == "conditional": + test_param_input_source = input_source.parse_test_input_source() + test_parameter = _from_input_source_galaxy(test_param_input_source) + whens = [] + default_value = object() + if isinstance(test_parameter, BooleanParameterModel): + default_value = test_parameter.value + # TODO: handle select parameter model... + for value, case_inputs_sources in input_source.parse_when_input_sources(): + if isinstance(test_parameter, BooleanParameterModel): + # TODO: investigate truevalue/falsevalue when... + from galaxy.util import string_as_bool + + typed_value = string_as_bool(value) + else: + typed_value = value + + tool_parameter_models = input_models_for_page(case_inputs_sources) + is_default_when = False + if typed_value == default_value: + is_default_when = True + whens.append( + ConditionalWhen(discriminator=value, parameters=tool_parameter_models, is_default_when=is_default_when) + ) + return ConditionalParameterModel( + name=input_source.parse_name(), + test_parameter=test_parameter, + whens=whens, + ) + elif input_type == "repeat": + # TODO: min/max + name = input_source.get("name") + # title = input_source.get("title") + # help = input_source.get("help", None) + instance_sources = input_source.parse_nested_inputs_source() + instance_tool_parameter_models = input_models_for_page(instance_sources) + min_raw = input_source.get("min", None) + max_raw = input_source.get("max", None) + min = int(min_raw) if min_raw is not None else None + max = int(max_raw) if max_raw is not None else None + return RepeatParameterModel( + name=name, + parameters=instance_tool_parameter_models, + min=min, + max=max, + ) + elif input_type == "section": + name = input_source.get("name") + instance_sources = input_source.parse_nested_inputs_source() + instance_tool_parameter_models = input_models_for_page(instance_sources) + return SectionParameterModel( + name=name, + parameters=instance_tool_parameter_models, + ) + else: + raise Exception( + f"Cannot generate tool parameter model for supplied tool source - unknown input_type {input_type}" + ) + + +def _simple_cwl_type_to_model(simple_type: str, input_source: CwlInputSource): + if simple_type == "int": + return CwlIntegerParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "float": + return CwlFloatParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "null": + return CwlNullParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "string": + return CwlStringParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "boolean": + return CwlBooleanParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "org.w3id.cwl.cwl.File": + return CwlFileParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "org.w3id.cwl.cwl.Directory": + return CwlDirectoryParameterModel( + name=input_source.parse_name(), + ) + raise NotImplementedError( + f"Cannot generate tool parameter model for this CWL artifact yet - contains unknown type {simple_type}." + ) + + +def _from_input_source_cwl(input_source: CwlInputSource) -> ToolParameterT: + schema_salad_field = input_source.field + if schema_salad_field is None: + raise NotImplementedError("Cannot generate tool parameter model for this CWL artifact yet.") + if "type" not in schema_salad_field: + raise NotImplementedError("Cannot generate tool parameter model for this CWL artifact yet.") + schema_salad_type = schema_salad_field["type"] + if isinstance(schema_salad_type, str): + return _simple_cwl_type_to_model(schema_salad_type, input_source) + elif isinstance(schema_salad_type, list): + return CwlUnionParameterModel( + name=input_source.parse_name(), + parameters=[_simple_cwl_type_to_model(t, input_source) for t in schema_salad_type], + ) + else: + raise NotImplementedError("Cannot generate tool parameter model for this CWL artifact yet.") + + +def input_models_from_json(json: List[Dict[str, Any]]) -> ToolParameterBundle: + return ToolParameterBundleModel(input_models=json) + + +def tool_parameter_bundle_from_json(json: Dict[str, Any]) -> ToolParameterBundleModel: + return ToolParameterBundleModel(**json) + + +def input_models_for_tool_source(tool_source: ToolSource) -> ToolParameterBundleModel: + pages = tool_source.parse_input_pages() + return ToolParameterBundleModel(input_models=input_models_for_pages(pages)) + + +def input_models_for_pages(pages: PagesSource) -> List[ToolParameterT]: + input_models = [] + if pages.inputs_defined: + for page_source in pages.page_sources: + input_models.extend(input_models_for_page(page_source)) + + return input_models + + +def input_models_for_page(page_source: PageSource) -> List[ToolParameterT]: + input_models = [] + for input_source in page_source.parse_input_sources(): + tool_parameter_model = from_input_source(input_source) + input_models.append(tool_parameter_model) + return input_models + + +def from_input_source(input_source: InputSource) -> ToolParameterT: + tool_parameter: ToolParameterT + if isinstance(input_source, CwlInputSource): + tool_parameter = _from_input_source_cwl(input_source) + else: + tool_parameter = _from_input_source_galaxy(input_source) + return tool_parameter diff --git a/lib/galaxy/tool_util/parameters/json.py b/lib/galaxy/tool_util/parameters/json.py new file mode 100644 index 000000000000..6796353a4fb7 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/json.py @@ -0,0 +1,27 @@ +import json +from typing import ( + Any, + Dict, +) + +from pydantic.json_schema import GenerateJsonSchema +from typing_extensions import Literal + +MODE = Literal["validation", "serialization"] +DEFAULT_JSON_SCHEMA_MODE: MODE = "validation" + + +class CustomGenerateJsonSchema(GenerateJsonSchema): + + def generate(self, schema, mode: MODE = DEFAULT_JSON_SCHEMA_MODE): + json_schema = super().generate(schema, mode=mode) + json_schema["$schema"] = self.schema_dialect + return json_schema + + +def to_json_schema(model, mode: MODE = DEFAULT_JSON_SCHEMA_MODE) -> Dict[str, Any]: + return model.model_json_schema(schema_generator=CustomGenerateJsonSchema, mode=mode) + + +def to_json_schema_string(model, mode: MODE = DEFAULT_JSON_SCHEMA_MODE) -> str: + return json.dumps(to_json_schema(model, mode=mode), indent=4) diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py new file mode 100644 index 000000000000..87982d06a199 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/models.py @@ -0,0 +1,928 @@ +# attempt to model requires_value... +# conditional can descend... +from abc import abstractmethod +from typing import ( + Any, + Callable, + cast, + Dict, + Iterable, + List, + Mapping, + NamedTuple, + Optional, + Type, + Union, +) + +from pydantic import ( + BaseModel, + ConfigDict, + create_model, + Discriminator, + Field, + field_validator, + RootModel, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + Tag, + ValidationError, +) +from typing_extensions import ( + Annotated, + Literal, + Protocol, +) + +from galaxy.exceptions import RequestParameterInvalidException +from ._types import ( + cast_as_type, + is_optional, + list_type, + optional_if_needed, + union_type, +) + +# TODO: +# - implement job vs request... +# - drill down +# - implement data_ref on rules and implement some cross model validation +# - Optional conditionals... work through that? +# - Sections - fight that battle again... + +# + request: Return info needed to build request pydantic model at runtime. +# + request_internal: This is a pydantic model to validate what Galaxy expects to find in the database, +# in particular dataset and collection references should be decoded integers. +StateRepresentationT = Literal["request", "request_internal", "job_internal", "test_case"] + + +# could be made more specific - validators need to be classmethod +ValidatorDictT = Dict[str, Callable] + + +class DynamicModelInformation(NamedTuple): + name: str + definition: tuple + validators: ValidatorDictT + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +def allow_batching(job_template: DynamicModelInformation, batch_type: Optional[Type] = None) -> DynamicModelInformation: + job_py_type: Type = job_template.definition[0] + default_value = job_template.definition[1] + batch_type = batch_type or job_py_type + + class BatchRequest(StrictModel): + meta_class: Literal["Batch"] = Field(..., alias="__class__") + values: List[batch_type] # type: ignore[valid-type] + + request_type = union_type([job_py_type, BatchRequest]) + + return DynamicModelInformation( + job_template.name, + (request_type, default_value), + {}, # should we modify these somehow? + ) + + +class Validators: + def validate_not_none(cls, v): + assert v is not None, "null is an invalid value for attribute" + return v + + +class ParamModel(Protocol): + @property + def name(self) -> str: ... + + @property + def request_requires_value(self) -> bool: + # if this is a non-optional type and no default is defined - an + # input value MUST be specified. + ... + + +def dynamic_model_information_from_py_type(param_model: ParamModel, py_type: Type): + name = param_model.name + initialize = ... if param_model.request_requires_value else None + py_type_is_optional = is_optional(py_type) + if not py_type_is_optional and not param_model.request_requires_value: + validators = {"not_null": field_validator(name)(Validators.validate_not_none)} + else: + validators = {} + + return DynamicModelInformation( + name, + (py_type, initialize), + validators, + ) + + +# We probably need incoming (parameter def) and outgoing (parameter value as transmitted) models, +# where value in the incoming model means "default value" and value in the outgoing model is the actual +# value a user has set. (incoming/outgoing from the client perspective). +class BaseToolParameterModelDefinition(BaseModel): + name: str + parameter_type: str + + @abstractmethod + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + """Return info needed to build Pydantic model at runtime for validation.""" + + +class BaseGalaxyToolParameterModelDefinition(BaseToolParameterModelDefinition): + hidden: bool = False + label: Optional[str] = None + help: Optional[str] = None + argument: Optional[str] = None + refresh_on_change: bool = False + is_dynamic: bool = False + optional: bool = False + + +class LabelValue(BaseModel): + label: str + value: str + selected: bool + + +class TextParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_text"] = "gx_text" + area: bool = False + default_value: Optional[str] = Field(default=None, alias="value") + default_options: List[LabelValue] = [] + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictStr, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return False + + +class IntegerParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_integer"] = "gx_integer" + optional: bool + value: Optional[int] = None + min: Optional[int] = None + max: Optional[int] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictInt, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return False + + +class FloatParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_float"] = "gx_float" + value: Optional[float] = None + min: Optional[float] = None + max: Optional[float] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(union_type([StrictInt, StrictFloat]), self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return False + + +DataSrcT = Literal["hda", "ldda"] +MultiDataSrcT = Literal["hda", "ldda", "hdca"] +CollectionStrT = Literal["hdca"] + +TestCaseDataSrcT = Literal["File"] + + +class DataRequest(StrictModel): + src: DataSrcT + id: StrictStr + + +class BatchDataInstance(StrictModel): + src: MultiDataSrcT + id: StrictStr + + +class MultiDataInstance(StrictModel): + src: MultiDataSrcT + id: StrictStr + + +MultiDataRequest: Type = union_type([MultiDataInstance, List[MultiDataInstance]]) + + +class DataRequestInternal(StrictModel): + src: DataSrcT + id: StrictInt + + +class BatchDataInstanceInternal(StrictModel): + src: MultiDataSrcT + id: StrictInt + + +class MultiDataInstanceInternal(StrictModel): + src: MultiDataSrcT + id: StrictInt + + +class DataTestCaseValue(StrictModel): + src: TestCaseDataSrcT + path: str + + +class MultipleDataTestCaseValue(RootModel): + root: List[DataTestCaseValue] + + +MultiDataRequestInternal: Type = union_type([MultiDataInstanceInternal, List[MultiDataInstanceInternal]]) + + +class DataParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_data"] = "gx_data" + extensions: List[str] = ["data"] + multiple: bool = False + min: Optional[int] = None + max: Optional[int] = None + + @property + def py_type(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequest + else: + base_model = DataRequest + return optional_if_needed(base_model, self.optional) + + @property + def py_type_internal(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequestInternal + else: + base_model = DataRequestInternal + return optional_if_needed(base_model, self.optional) + + @property + def py_type_test_case(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequestInternal + else: + base_model = DataTestCaseValue + return optional_if_needed(base_model, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + if state_representation == "request": + return allow_batching(dynamic_model_information_from_py_type(self, self.py_type), BatchDataInstance) + elif state_representation == "request_internal": + return allow_batching( + dynamic_model_information_from_py_type(self, self.py_type_internal), BatchDataInstanceInternal + ) + elif state_representation == "job_internal": + return dynamic_model_information_from_py_type(self, self.py_type_internal) + elif state_representation == "test_case": + return dynamic_model_information_from_py_type(self, self.py_type_test_case) + + @property + def request_requires_value(self) -> bool: + return not self.optional + + +class DataCollectionRequest(StrictModel): + src: CollectionStrT + id: StrictStr + + +class DataCollectionRequestInternal(StrictModel): + src: CollectionStrT + id: StrictInt + + +class DataCollectionParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_data_collection"] = "gx_data_collection" + collection_type: Optional[str] = None + extensions: List[str] = ["data"] + + @property + def py_type(self) -> Type: + return optional_if_needed(DataCollectionRequest, self.optional) + + @property + def py_type_internal(self) -> Type: + return optional_if_needed(DataCollectionRequestInternal, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + if state_representation == "request": + return allow_batching(dynamic_model_information_from_py_type(self, self.py_type)) + elif state_representation == "request_internal": + return allow_batching(dynamic_model_information_from_py_type(self, self.py_type_internal)) + else: + raise NotImplementedError("...") + + @property + def request_requires_value(self) -> bool: + return not self.optional + + +class HiddenParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_hidden"] = "gx_hidden" + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictStr, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return not self.optional + + +def ensure_color_valid(value: Optional[Any]): + if value is None: + return + if not isinstance(value, str): + raise ValueError(f"Invalid color value type {value.__class__} encountered.") + value_str: str = value + message = f"Invalid color value string format {value_str} encountered." + if len(value_str) != 7: + raise ValueError(message + "0") + if value_str[0] != "#": + raise ValueError(message + "1") + for byte_str in value_str[1:]: + if byte_str not in "0123456789abcdef": + raise ValueError(message + "2") + + +class ColorParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_color"] = "gx_color" + value: Optional[str] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictStr, self.optional) + + @staticmethod + def validate_color_str(value) -> str: + ensure_color_valid(value) + return value + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + validators = {"color_format": field_validator(self.name)(ColorParameterModel.validate_color_str)} + return DynamicModelInformation( + self.name, + (self.py_type, ...), + validators, + ) + + @property + def request_requires_value(self) -> bool: + return False + + +class BooleanParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_boolean"] = "gx_boolean" + value: Optional[bool] = False + truevalue: Optional[str] = None + falsevalue: Optional[str] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictBool, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + # these parameters always have an implicit default - either None if + # if it is optional or 'checked' if not (itself defaulting to False). + return False + + +class DirectoryUriParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_directory_uri"] + value: Optional[str] + + @property + def request_requires_value(self) -> bool: + return True + + +class RulesMapping(StrictModel): + type: str + columns: List[StrictInt] + + +class RulesModel(StrictModel): + rules: List[Dict[str, Any]] + mappings: List[RulesMapping] + + +class RulesParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_rules"] = "gx_rules" + + @property + def py_type(self) -> Type: + return RulesModel + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + +class SelectParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_select"] = "gx_select" + options: Optional[List[LabelValue]] = None + multiple: bool + + @property + def py_type(self) -> Type: + if self.options is not None: + literal_options: List[Type] = [cast_as_type(Literal[o.value]) for o in self.options] + py_type = union_type(literal_options) + else: + py_type = StrictStr + if self.multiple: + py_type = list_type(py_type) + return optional_if_needed(py_type, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def has_selected_static_option(self): + return self.options is not None and any(o.selected for o in self.options) + + @property + def request_requires_value(self) -> bool: + return not self.optional and not self.has_selected_static_option + + +DiscriminatorType = Union[bool, str] + + +class ConditionalWhen(StrictModel): + discriminator: DiscriminatorType + parameters: List["ToolParameterT"] + is_default_when: bool + + +class ConditionalParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_conditional"] = "gx_conditional" + test_parameter: Union[BooleanParameterModel, SelectParameterModel] + whens: List[ConditionalWhen] + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + test_param_name = self.test_parameter.name + test_info = self.test_parameter.pydantic_template(state_representation) + extra_validators = test_info.validators + test_parameter_requires_value = self.test_parameter.request_requires_value + when_types: List[Type[BaseModel]] = [] + default_type = None + for when in self.whens: + discriminator = when.discriminator + parameters = when.parameters + if test_parameter_requires_value: + initialize_test = ... + else: + initialize_test = None + + extra_kwd = {test_param_name: (Union[str, bool], initialize_test)} + when_types.append( + cast( + Type[BaseModel], + Annotated[ + create_field_model( + parameters, + f"When_{test_param_name}_{discriminator}", + state_representation, + extra_kwd=extra_kwd, + extra_validators=extra_validators, + ), + Tag(str(discriminator)), + ], + ) + ) + if when.is_default_when: + extra_kwd = {} + default_type = create_field_model( + parameters, + f"When_{test_param_name}___absent", + state_representation, + extra_kwd=extra_kwd, + extra_validators={}, + ) + when_types.append(cast(Type[BaseModel], Annotated[default_type, Tag("__absent__")])) + + def model_x_discriminator(v: Any) -> str: + if test_param_name not in v: + return "__absent__" + else: + test_param_val = v[test_param_name] + if test_param_val is True: + return "true" + elif test_param_val is False: + return "false" + else: + return str(test_param_val) + + cond_type = union_type(when_types) + + class ConditionalType(RootModel): + root: cond_type = Field(..., discriminator=Discriminator(model_x_discriminator)) # type: ignore[valid-type] + + if default_type is not None: + initialize_cond = None + else: + initialize_cond = ... + + py_type = ConditionalType + + return DynamicModelInformation( + self.name, + (py_type, initialize_cond), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return False # TODO + + +class RepeatParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_repeat"] = "gx_repeat" + parameters: List["ToolParameterT"] + min: Optional[int] = None + max: Optional[int] = None + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + # Maybe validators for min and max... + instance_class: Type[BaseModel] = create_field_model( + self.parameters, f"Repeat_{self.name}", state_representation + ) + + field_params = {} + if self.min is not None: + field_params["min_length"] = self.min + if self.max is not None: + field_params["max_length"] = self.max + + class RepeatType(RootModel): + root: List[instance_class] = Field(..., **field_params) # type: ignore[valid-type] + + return DynamicModelInformation( + self.name, + (RepeatType, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True # TODO: + + +class SectionParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_section"] = "gx_section" + parameters: List["ToolParameterT"] + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + instance_class: Type[BaseModel] = create_field_model( + self.parameters, f"Section_{self.name}", state_representation + ) + if self.request_requires_value: + initialize_section = ... + else: + initialize_section = None + return DynamicModelInformation( + self.name, + (instance_class, initialize_section), + {}, + ) + + @property + def request_requires_value(self) -> bool: + any_request_parameters_required = False + for parameter in self.parameters: + if parameter.request_requires_value: + any_request_parameters_required = True + break + return any_request_parameters_required + + +LiteralNone: Type = Literal[None] # type: ignore[assignment] + + +class CwlNullParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_null"] = "cwl_null" + + @property + def py_type(self) -> Type: + return LiteralNone + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return False + + +class CwlStringParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_string"] = "cwl_string" + + @property + def py_type(self) -> Type: + return StrictStr + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlIntegerParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_integer"] = "cwl_integer" + + @property + def py_type(self) -> Type: + return StrictInt + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlFloatParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_float"] = "cwl_float" + + @property + def py_type(self) -> Type: + return union_type([StrictFloat, StrictInt]) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlBooleanParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_boolean"] = "cwl_boolean" + + @property + def py_type(self) -> Type: + return StrictBool + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlUnionParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_union"] = "cwl_union" + parameters: List["CwlParameterT"] + + @property + def py_type(self) -> Type: + union_of_cwl_types: List[Type] = [] + for parameter in self.parameters: + union_of_cwl_types.append(parameter.py_type) + return union_type(union_of_cwl_types) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return False # TODO: + + +class CwlFileParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["cwl_file"] = "cwl_file" + + @property + def py_type(self) -> Type: + return DataRequest + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlDirectoryParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["cwl_directory"] = "cwl_directory" + + @property + def py_type(self) -> Type: + return DataRequest + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + +CwlParameterT = Union[ + CwlIntegerParameterModel, + CwlFloatParameterModel, + CwlStringParameterModel, + CwlBooleanParameterModel, + CwlNullParameterModel, + CwlFileParameterModel, + CwlDirectoryParameterModel, + CwlUnionParameterModel, +] + +GalaxyParameterT = Union[ + TextParameterModel, + IntegerParameterModel, + FloatParameterModel, + BooleanParameterModel, + HiddenParameterModel, + SelectParameterModel, + DataParameterModel, + DataCollectionParameterModel, + DirectoryUriParameterModel, + RulesParameterModel, + ColorParameterModel, + ConditionalParameterModel, + RepeatParameterModel, +] + +ToolParameterT = Union[ + CwlParameterT, + GalaxyParameterT, +] + + +class ToolParameterModel(RootModel): + root: ToolParameterT = Field(..., discriminator="parameter_type") + + +ConditionalWhen.model_rebuild() +ConditionalParameterModel.model_rebuild() +RepeatParameterModel.model_rebuild() +CwlUnionParameterModel.model_rebuild() + + +class ToolParameterBundle(Protocol): + """An object having a dictionary of input models (i.e. a 'Tool')""" + + # TODO: rename to parameters to align with ConditionalWhen and Repeat. + input_models: List[ToolParameterT] + + +class ToolParameterBundleModel(BaseModel): + input_models: List[ToolParameterT] + + +def parameters_by_name(tool_parameter_bundle: ToolParameterBundle) -> Dict[str, ToolParameterT]: + as_dict = {} + for input_model in simple_input_models(tool_parameter_bundle.input_models): + as_dict[input_model.name] = input_model + return as_dict + + +def to_simple_model(input_parameter: Union[ToolParameterModel, ToolParameterT]) -> ToolParameterT: + if input_parameter.__class__ == ToolParameterModel: + assert isinstance(input_parameter, ToolParameterModel) + return cast(ToolParameterT, input_parameter.root) + else: + return cast(ToolParameterT, input_parameter) + + +def simple_input_models( + input_models: Union[List[ToolParameterModel], List[ToolParameterT]] +) -> Iterable[ToolParameterT]: + return [to_simple_model(m) for m in input_models] + + +def create_model_strict(*args, **kwd) -> Type[BaseModel]: + model_config = ConfigDict(extra="forbid") + + return create_model(*args, __config__=model_config, **kwd) + + +def create_request_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "request") + + +def create_request_internal_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "request_internal") + + +def create_test_case_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "test_case") + + +def create_field_model( + tool_parameter_models: Union[List[ToolParameterModel], List[ToolParameterT]], + name: str, + state_representation: StateRepresentationT, + extra_kwd: Optional[Mapping[str, tuple]] = None, + extra_validators: Optional[ValidatorDictT] = None, +) -> Type[BaseModel]: + kwd: Dict[str, tuple] = {} + if extra_kwd: + kwd.update(extra_kwd) + model_validators = (extra_validators or {}).copy() + + for input_model in tool_parameter_models: + input_model = to_simple_model(input_model) + input_name = input_model.name + pydantic_request_template = input_model.pydantic_template(state_representation) + kwd[input_name] = pydantic_request_template.definition + input_validators = pydantic_request_template.validators + for validator_name, validator_callable in input_validators.items(): + model_validators[f"{input_name}_{validator_name}"] = validator_callable + + pydantic_model = create_model_strict(name, __validators__=model_validators, **kwd) + return pydantic_model + + +def validate_against_model(pydantic_model: Type[BaseModel], parameter_state: Dict[str, Any]) -> None: + try: + pydantic_model(**parameter_state) + except ValidationError as e: + # TODO: Improve this or maybe add a handler for this in the FastAPI exception + # handler. + raise RequestParameterInvalidException(str(e)) + + +def validate_request(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_request_model(tool) + validate_against_model(pydantic_model, request) + + +def validate_internal_request(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_request_internal_model(tool) + validate_against_model(pydantic_model, request) + + +def validate_test_case(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_test_case_model(tool) + validate_against_model(pydantic_model, request) diff --git a/lib/galaxy/tool_util/parameters/state.py b/lib/galaxy/tool_util/parameters/state.py new file mode 100644 index 000000000000..52a929a19383 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/state.py @@ -0,0 +1,82 @@ +from abc import ( + ABC, + abstractmethod, +) +from typing import ( + Any, + Dict, + Optional, + Type, +) + +from pydantic import BaseModel +from typing_extensions import Literal + +from .models import ( + create_request_internal_model, + create_request_model, + StateRepresentationT, + ToolParameterBundle, + validate_against_model, +) + + +class ToolState(ABC): + input_state: Dict[str, Any] + + def __init__(self, input_state: Dict[str, Any]): + self.input_state = input_state + + def _validate(self, pydantic_model: Type[BaseModel]) -> None: + validate_against_model(pydantic_model, self.input_state) + + def validate(self, input_models: ToolParameterBundle) -> None: + base_model = self.parameter_model_for(input_models) + if base_model is None: + raise NotImplementedError( + f"Validating tool state against state representation {self.state_representation} is not implemented." + ) + self._validate(base_model) + + @property + @abstractmethod + def state_representation(self) -> StateRepresentationT: + """Get state representation of the inputs.""" + + @classmethod + def parameter_model_for(cls, input_models: ToolParameterBundle) -> Optional[Type[BaseModel]]: + return None + + +class RequestToolState(ToolState): + state_representation: Literal["request"] = "request" + + @classmethod + def parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + return create_request_model(input_models) + + +class RequestInternalToolState(ToolState): + state_representation: Literal["request_internal"] = "request_internal" + + @classmethod + def parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + return create_request_internal_model(input_models) + + +class JobInternalToolState(ToolState): + state_representation: Literal["job_internal"] = "job_internal" + + @classmethod + def parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + # implement a job model... + return create_request_internal_model(input_models) + + +class TestCaseToolState(ToolState): + state_representation: Literal["test_case"] = "test_case" + + @classmethod + def parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + # implement a test case model... + return create_request_internal_model(input_models) diff --git a/lib/galaxy/tool_util/parameters/visitor.py b/lib/galaxy/tool_util/parameters/visitor.py new file mode 100644 index 000000000000..7b68afa4a0aa --- /dev/null +++ b/lib/galaxy/tool_util/parameters/visitor.py @@ -0,0 +1,56 @@ +from typing import ( + Any, + Dict, + Iterable, +) + +from typing_extensions import Protocol + +from .models import ( + simple_input_models, + ToolParameterBundle, + ToolParameterT, +) +from .state import ToolState + +VISITOR_NO_REPLACEMENT = object() +VISITOR_UNDEFINED = object() + + +class Callback(Protocol): + def __call__(self, parameter: ToolParameterT, value: Any): + pass + + +def visit_input_values( + input_models: ToolParameterBundle, + tool_state: ToolState, + callback: Callback, + no_replacement_value=VISITOR_NO_REPLACEMENT, +) -> Dict[str, Any]: + return _visit_input_values( + simple_input_models(input_models.input_models), + tool_state.input_state, + callback=callback, + no_replacement_value=no_replacement_value, + ) + + +def _visit_input_values( + input_models: Iterable[ToolParameterT], + input_values: Dict[str, Any], + callback: Callback, + no_replacement_value=VISITOR_NO_REPLACEMENT, +) -> Dict[str, Any]: + new_input_values = {} + for model in input_models: + name = model.name + input_value = input_values.get(name, VISITOR_UNDEFINED) + replacement = callback(model, input_value) + if replacement != no_replacement_value: + new_input_values[name] = replacement + elif replacement is VISITOR_UNDEFINED: + pass + else: + new_input_values[name] = input_value + return new_input_values diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py index 1647c39aa701..45a4634ae82f 100644 --- a/lib/galaxy/tool_util/parser/cwl.py +++ b/lib/galaxy/tool_util/parser/cwl.py @@ -179,18 +179,41 @@ def to_string(self): return json.dumps(self.tool_proxy.to_persistent_representation()) +class CwlInputSource(YamlInputSource): + def __init__(self, as_dict, as_field): + super().__init__(as_dict) + self._field = as_field + + @property + def field(self): + return self._field + + class CwlPageSource(PageSource): def __init__(self, tool_proxy): cwl_instances = tool_proxy.input_instances() - self._input_list = list(map(self._to_input_source, cwl_instances)) + input_fields = tool_proxy.input_fields() + input_list = [] + for cwl_instance in cwl_instances: + name = cwl_instance.name + input_field = None + for field in input_fields: + if field["name"] == name: + input_field = field + input_list.append(CwlInputSource(cwl_instance.to_dict(), input_field)) + + self._input_list = input_list def _to_input_source(self, input_instance): as_dict = input_instance.to_dict() - return YamlInputSource(as_dict) + return CwlInputSource(as_dict) def parse_input_sources(self): return self._input_list + def input_fields(self): + return self._input_fields + __all__ = ( "CwlToolSource", diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 9fa0bdcfdb10..51097c3a3d96 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -11,6 +11,7 @@ Iterable, List, Optional, + Tuple, ) from packaging.version import Version @@ -1263,7 +1264,7 @@ def parse_dynamic_options_elem(self): options_elem = self.input_elem.find("options") return options_elem - def parse_static_options(self): + def parse_static_options(self) -> List[Tuple[str, str, bool]]: """ >>> from galaxy.util import parse_xml_string_to_etree >>> xml = '' diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index c2b5f95e4be7..b2f293e3cfe2 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -4,6 +4,7 @@ Dict, List, Optional, + Tuple, ) import packaging.version @@ -364,7 +365,7 @@ def parse_when_input_sources(self): sources.append((value, case_page_source)) return sources - def parse_static_options(self): + def parse_static_options(self) -> List[Tuple[str, str, bool]]: static_options = [] input_dict = self.input_dict for option in input_dict.get("options", {}): diff --git a/lib/galaxy/tool_util/unittest_utils/parameters.py b/lib/galaxy/tool_util/unittest_utils/parameters.py new file mode 100644 index 000000000000..d3be68b7cca2 --- /dev/null +++ b/lib/galaxy/tool_util/unittest_utils/parameters.py @@ -0,0 +1,46 @@ +import os +from typing import List + +from galaxy.tool_util.parameters import ( + from_input_source, + ToolParameterBundle, + ToolParameterT, +) +from galaxy.tool_util.parser.factory import get_tool_source +from galaxy.util import galaxy_directory + + +class ParameterBundle(ToolParameterBundle): + input_models: List[ToolParameterT] + + def __init__(self, parameter: ToolParameterT): + self.input_models = [parameter] + + +def parameter_bundle(parameter: ToolParameterT) -> ParameterBundle: + return ParameterBundle(parameter) + + +def parameter_bundle_for_file(filename: str) -> ParameterBundle: + return parameter_bundle(tool_parameter(filename)) + + +def tool_parameter(filename: str) -> ToolParameterT: + return from_input_source(parameter_source(filename)) + + +def parameter_source(filename: str): + tool_source = parameter_tool_source(filename) + input_sources = tool_source.parse_input_pages().page_sources[0].parse_input_sources() + assert len(input_sources) == 1 + return input_sources[0] + + +def parameter_tool_source(basename: str): + path_prefix = os.path.join(galaxy_directory(), "test/functional/tools/parameters", basename) + if os.path.exists(f"{path_prefix}.xml"): + path = f"{path_prefix}.xml" + else: + path = f"{path_prefix}.cwl" + tool_source = get_tool_source(path, macro_paths=[]) + return tool_source diff --git a/lib/tool_shed/managers/tool_state_cache.py b/lib/tool_shed/managers/tool_state_cache.py new file mode 100644 index 000000000000..010ab288a334 --- /dev/null +++ b/lib/tool_shed/managers/tool_state_cache.py @@ -0,0 +1,42 @@ +import json +import os +from typing import ( + Any, + Dict, + Optional, +) + +RAW_CACHED_JSON = Dict[str, Any] + + +class ToolStateCache: + _cache_directory: str + + def __init__(self, cache_directory: str): + if not os.path.exists(cache_directory): + os.makedirs(cache_directory) + self._cache_directory = cache_directory + + def _cache_target(self, tool_id: str, tool_version: str): + # consider breaking this into multiple directories... + cache_target = os.path.join(self._cache_directory, tool_id, tool_version) + return cache_target + + def get_cache_entry_for(self, tool_id: str, tool_version: str) -> Optional[RAW_CACHED_JSON]: + cache_target = self._cache_target(tool_id, tool_version) + if not os.path.exists(cache_target): + return None + with open(cache_target) as f: + return json.load(f) + + def has_cached_entry_for(self, tool_id: str, tool_version: str) -> bool: + cache_target = self._cache_target(tool_id, tool_version) + return os.path.exists(cache_target) + + def insert_cache_entry_for(self, tool_id: str, tool_version: str, entry: RAW_CACHED_JSON) -> None: + cache_target = self._cache_target(tool_id, tool_version) + parent_directory = os.path.dirname(cache_target) + if not os.path.exists(parent_directory): + os.makedirs(parent_directory) + with open(cache_target, "w") as f: + json.dump(entry, f) diff --git a/lib/tool_shed/managers/tools.py b/lib/tool_shed/managers/tools.py index bd648d4903a9..d252fafbadbb 100644 --- a/lib/tool_shed/managers/tools.py +++ b/lib/tool_shed/managers/tools.py @@ -1,8 +1,40 @@ +import os +import tempfile from collections import namedtuple +from typing import ( + List, + Optional, + Tuple, +) from galaxy import exceptions -from tool_shed.context import SessionRequestContext +from galaxy.exceptions import ( + InternalServerError, + ObjectNotFound, +) +from galaxy.tool_shed.metadata.metadata_generator import RepositoryMetadataToolDict +from galaxy.tool_shed.util.basic_util import remove_dir +from galaxy.tool_shed.util.hg_util import ( + clone_repository, + get_changectx_for_changeset, +) +from galaxy.tool_util.parameters import ( + input_models_for_tool_source, + tool_parameter_bundle_from_json, + ToolParameterBundleModel, +) +from galaxy.tool_util.parser import ( + get_tool_source, + ToolSource, +) +from tool_shed.context import ( + ProvidesRepositoriesContext, + SessionRequestContext, +) +from tool_shed.util.common_util import generate_clone_url_for +from tool_shed.webapp.model import RepositoryMetadata from tool_shed.webapp.search.tool_search import ToolSearch +from .trs import trs_tool_id_to_repository_metadata def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict: @@ -42,3 +74,62 @@ def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = results = tool_search.search(trans.app, search_term, page, page_size, boosts) results["hostname"] = trans.repositories_hostname return results + + +def get_repository_metadata_tool_dict( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str +) -> Tuple[RepositoryMetadata, RepositoryMetadataToolDict]: + name, owner, tool_id = trs_tool_id.split("~", 3) + repository, metadata_by_version = trs_tool_id_to_repository_metadata(trans, trs_tool_id) + if tool_version not in metadata_by_version: + raise ObjectNotFound() + tool_version_repository_metadata: RepositoryMetadata = metadata_by_version[tool_version] + raw_metadata = tool_version_repository_metadata.metadata + tool_dicts: List[RepositoryMetadataToolDict] = raw_metadata.get("tools", []) + for tool_dict in tool_dicts: + if tool_dict["id"] != tool_id or tool_dict["version"] != tool_version: + continue + return tool_version_repository_metadata, tool_dict + raise ObjectNotFound() + + +def tool_input_models_cached_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolParameterBundleModel: + tool_state_cache = trans.app.tool_state_cache + raw_json = tool_state_cache.get_cache_entry_for(trs_tool_id, tool_version) + if raw_json is not None: + return tool_parameter_bundle_from_json(raw_json) + bundle = tool_input_models_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) + tool_state_cache.insert_cache_entry_for(trs_tool_id, tool_version, bundle.dict()) + return bundle + + +def tool_input_models_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolParameterBundleModel: + tool_source = tool_source_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) + return input_models_for_tool_source(tool_source) + + +def tool_source_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolSource: + rval = get_repository_metadata_tool_dict(trans, trs_tool_id, tool_version) + repository_metadata, tool_version_metadata = rval + tool_config = tool_version_metadata["tool_config"] + + repo = repository_metadata.repository.hg_repo + ctx = get_changectx_for_changeset(repo, repository_metadata.changeset_revision) + work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-tool_source") + if repository_clone_url is None: + repository_clone_url = generate_clone_url_for(trans, repository_metadata.repository) + try: + cloned_ok, error_message = clone_repository(repository_clone_url, work_dir, str(ctx.rev())) + if error_message: + raise InternalServerError("Failed to materialize target repository revision") + path_to_tool = os.path.join(work_dir, tool_config) + tool_source = get_tool_source(path_to_tool) + return tool_source + finally: + remove_dir(work_dir) diff --git a/lib/tool_shed/managers/trs.py b/lib/tool_shed/managers/trs.py index d77fdc7334ee..ebb74220ccd3 100644 --- a/lib/tool_shed/managers/trs.py +++ b/lib/tool_shed/managers/trs.py @@ -74,10 +74,14 @@ def tool_classes() -> List[ToolClass]: return [ToolClass(id="galaxy_tool", name="Galaxy Tool", description="Galaxy XML Tools")] -def trs_tool_id_to_repository(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> Repository: +def trs_tool_id_to_guid(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> str: guid = decode_identifier(trans.repositories_hostname, trs_tool_id) guid = remove_protocol_and_user_from_clone_url(guid) - return guid_to_repository(trans.app, guid) + return guid + + +def trs_tool_id_to_repository(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> Repository: + return guid_to_repository(trans.app, trs_tool_id_to_guid(trans, trs_tool_id)) def get_repository_metadata_by_tool_version( @@ -104,7 +108,7 @@ def get_tools_for(repository_metadata: RepositoryMetadata) -> List[Dict[str, Any def trs_tool_id_to_repository_metadata( trans: ProvidesRepositoriesContext, trs_tool_id: str -) -> Optional[Tuple[Repository, Dict[str, RepositoryMetadata]]]: +) -> Tuple[Repository, Dict[str, RepositoryMetadata]]: tool_guid = decode_identifier(trans.repositories_hostname, trs_tool_id) tool_guid = remove_protocol_and_user_from_clone_url(tool_guid) _, tool_id = tool_guid.rsplit("/", 1) diff --git a/lib/tool_shed/structured_app.py b/lib/tool_shed/structured_app.py index deb0b0ece6be..c3eee0c94299 100644 --- a/lib/tool_shed/structured_app.py +++ b/lib/tool_shed/structured_app.py @@ -3,6 +3,7 @@ from galaxy.structured_app import BasicSharedApp if TYPE_CHECKING: + from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.repository_registry import Registry as RepositoryRegistry from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry from tool_shed.util.hgweb_config import HgWebConfigManager @@ -16,3 +17,4 @@ class ToolShedApp(BasicSharedApp): repository_registry: "RepositoryRegistry" hgweb_config_manager: "HgWebConfigManager" security_agent: "CommunityRBACAgent" + tool_state_cache: "ToolStateCache" diff --git a/lib/tool_shed/test/functional/test_shed_tools.py b/lib/tool_shed/test/functional/test_shed_tools.py index c54dd825e479..5c2a9ea4389e 100644 --- a/lib/tool_shed/test/functional/test_shed_tools.py +++ b/lib/tool_shed/test/functional/test_shed_tools.py @@ -62,9 +62,17 @@ def test_trs_tool_list(self): repository = populator.setup_column_maker_repo(prefix="toolstrsindex") tool_id = populator.tool_guid(self, repository, "Add_a_column1") tool_shed_base, encoded_tool_id = encode_identifier(tool_id) - print(encoded_tool_id) url = f"ga4gh/trs/v2/tools/{encoded_tool_id}" - print(url) tool_response = self.api_interactor.get(url) tool_response.raise_for_status() assert Tool(**tool_response.json()) + + @skip_if_api_v1 + def test_trs_tool_parameter_json_schema(self): + populator = self.populator + repository = populator.setup_column_maker_repo(prefix="toolsparameterschema") + tool_id = populator.tool_guid(self, repository, "Add_a_column1") + tool_shed_base, encoded_tool_id = encode_identifier(tool_id) + url = f"tools/{encoded_tool_id}/versions/1.1.0/parameter_request_schema" + tool_response = self.api_interactor.get(url) + tool_response.raise_for_status() diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index bd48db71ce01..486a88730909 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -4,10 +4,19 @@ from fastapi import ( Path, Request, + Response, ) +from galaxy.tool_util.parameters import ( + RequestToolState, + to_json_schema_string, + ToolParameterBundleModel, +) from tool_shed.context import SessionRequestContext -from tool_shed.managers.tools import search +from tool_shed.managers.tools import ( + search, + tool_input_models_cached_for, +) from tool_shed.managers.trs import ( get_tool, service_info, @@ -41,6 +50,17 @@ description="See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids", ) +TOOL_VERSION_PATH_PARAM: str = Path( + ..., + title="Galaxy Tool Wrapper Version", + description="The full version string defined on the Galaxy tool wrapper.", +) + + +def json_schema_response(pydantic_model) -> Response: + json_str = to_json_schema_string(pydantic_model) + return Response(content=json_str, media_type="application/json") + @router.cbv class FastAPITools: @@ -122,3 +142,32 @@ def trs_get_versions( tool_id: str = TOOL_ID_PATH_PARAM, ) -> List[ToolVersion]: return get_tool(trans, tool_id).versions + + @router.get( + "/api/tools/{tool_id}/versions/{tool_version}/parameter_model", + operation_id="tools__parameter_model", + summary="Return Galaxy's meta model description of the tool's inputs", + ) + def tool_parameters_meta_model( + self, + trans: SessionRequestContext = DependsOnTrans, + tool_id: str = TOOL_ID_PATH_PARAM, + tool_version: str = TOOL_VERSION_PATH_PARAM, + ) -> ToolParameterBundleModel: + return tool_input_models_cached_for(trans, tool_id, tool_version) + + @router.get( + "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema", + operation_id="tools__parameter_request_model", + summary="Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point", + description="The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution.", + ) + def tool_state( + self, + trans: SessionRequestContext = DependsOnTrans, + tool_id: str = TOOL_ID_PATH_PARAM, + tool_version: str = TOOL_VERSION_PATH_PARAM, + ) -> Response: + return json_schema_response( + RequestToolState.parameter_model_for(tool_input_models_cached_for(trans, tool_id, tool_version)) + ) diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index 58ccf206596c..4083674241a4 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -33,6 +33,7 @@ from galaxy.structured_app import BasicSharedApp from galaxy.web_stack import application_stack_instance from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager +from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.structured_app import ToolShedApp from tool_shed.util.hgweb_config import hgweb_config_manager from tool_shed.webapp.model.migrations import verify_database @@ -83,6 +84,7 @@ def __init__(self, **kwd) -> None: self._register_singleton(SharedModelMapping, model) self._register_singleton(mapping.ToolShedModelMapping, model) self._register_singleton(scoped_session, self.model.context) + self.tool_state_cache = ToolStateCache(self.config.tool_state_cache_dir) self.user_manager = self._register_singleton(UserManager, UserManager(self, app_type="tool_shed")) self.api_keys_manager = self._register_singleton(ApiKeyManager) # initialize the Tool Shed tag handler. diff --git a/lib/tool_shed/webapp/frontend/src/schema/schema.ts b/lib/tool_shed/webapp/frontend/src/schema/schema.ts index 738ad268432e..547f47f9e0be 100644 --- a/lib/tool_shed/webapp/frontend/src/schema/schema.ts +++ b/lib/tool_shed/webapp/frontend/src/schema/schema.ts @@ -168,6 +168,17 @@ export interface paths { */ put: operations["tools__build_search_index"] } + "/api/tools/{tool_id}/versions/{tool_version}/parameter_model": { + /** Return Galaxy's meta model description of the tool's inputs */ + get: operations["tools__parameter_model"] + } + "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema": { + /** + * Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point + * @description The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution. + */ + get: operations["tools__parameter_request_model"] + } "/api/users": { /** * Index @@ -259,6 +270,53 @@ export interface components { /** Files */ files?: string[] | null } + /** BooleanParameterModel */ + BooleanParameterModel: { + /** Argument */ + argument?: string | null + /** Falsevalue */ + falsevalue?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_boolean + * @constant + * @enum {string} + */ + parameter_type?: "gx_boolean" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Truevalue */ + truevalue?: string | null + /** + * Value + * @default false + */ + value?: boolean | null + } /** BuildSearchIndexResponse */ BuildSearchIndexResponse: { /** Repositories Indexed */ @@ -295,6 +353,121 @@ export interface components { */ type: string } + /** ColorParameterModel */ + ColorParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_color + * @constant + * @enum {string} + */ + parameter_type?: "gx_color" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Value */ + value?: string | null + } + /** ConditionalParameterModel */ + ConditionalParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_conditional + * @constant + * @enum {string} + */ + parameter_type?: "gx_conditional" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Test Parameter */ + test_parameter: + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["SelectParameterModel"] + /** Whens */ + whens: components["schemas"]["ConditionalWhen"][] + } + /** ConditionalWhen */ + ConditionalWhen: { + /** Discriminator */ + discriminator: boolean | string + /** Is Default When */ + is_default_when: boolean + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + )[] + } /** CreateCategoryRequest */ CreateCategoryRequest: { /** Description */ @@ -332,6 +505,266 @@ export interface components { /** Username */ username: string } + /** CwlBooleanParameterModel */ + CwlBooleanParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_boolean + * @constant + * @enum {string} + */ + parameter_type?: "cwl_boolean" + } + /** CwlDirectoryParameterModel */ + CwlDirectoryParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default cwl_directory + * @constant + * @enum {string} + */ + parameter_type?: "cwl_directory" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } + /** CwlFileParameterModel */ + CwlFileParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default cwl_file + * @constant + * @enum {string} + */ + parameter_type?: "cwl_file" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } + /** CwlFloatParameterModel */ + CwlFloatParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_float + * @constant + * @enum {string} + */ + parameter_type?: "cwl_float" + } + /** CwlIntegerParameterModel */ + CwlIntegerParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_integer + * @constant + * @enum {string} + */ + parameter_type?: "cwl_integer" + } + /** CwlNullParameterModel */ + CwlNullParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_null + * @constant + * @enum {string} + */ + parameter_type?: "cwl_null" + } + /** CwlStringParameterModel */ + CwlStringParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_string + * @constant + * @enum {string} + */ + parameter_type?: "cwl_string" + } + /** CwlUnionParameterModel */ + CwlUnionParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_union + * @constant + * @enum {string} + */ + parameter_type?: "cwl_union" + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + )[] + } + /** DataCollectionParameterModel */ + DataCollectionParameterModel: { + /** Argument */ + argument?: string | null + /** Collection Type */ + collection_type?: string | null + /** + * Extensions + * @default [ + * "data" + * ] + */ + extensions?: string[] + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_data_collection + * @constant + * @enum {string} + */ + parameter_type?: "gx_data_collection" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } + /** DataParameterModel */ + DataParameterModel: { + /** Argument */ + argument?: string | null + /** + * Extensions + * @default [ + * "data" + * ] + */ + extensions?: string[] + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** + * Multiple + * @default false + */ + multiple?: boolean + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_data + * @constant + * @enum {string} + */ + parameter_type?: "gx_data" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } /** * DescriptorType * @enum {string} @@ -375,16 +808,137 @@ export interface components { /** User Id */ user_id: string } + /** DirectoryUriParameterModel */ + DirectoryUriParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @constant + * @enum {string} + */ + parameter_type: "gx_directory_uri" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Value */ + value: string | null + } /** FailedRepositoryUpdateMessage */ FailedRepositoryUpdateMessage: { /** Err Msg */ err_msg: string } + /** FloatParameterModel */ + FloatParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_float + * @constant + * @enum {string} + */ + parameter_type?: "gx_float" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Value */ + value?: number | null + } /** HTTPValidationError */ HTTPValidationError: { /** Detail */ detail?: components["schemas"]["ValidationError"][] } + /** HiddenParameterModel */ + HiddenParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_hidden + * @constant + * @enum {string} + */ + parameter_type?: "gx_hidden" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } /** ImageData */ ImageData: { /** @@ -424,6 +978,56 @@ export interface components { metadata_info?: components["schemas"]["RepositoryMetadataInstallInfo"] | null repo_info?: components["schemas"]["RepositoryExtraInstallInfo"] | null } + /** IntegerParameterModel */ + IntegerParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** Name */ + name: string + /** Optional */ + optional: boolean + /** + * Parameter Type + * @default gx_integer + * @constant + * @enum {string} + */ + parameter_type?: "gx_integer" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Value */ + value?: number | null + } + /** LabelValue */ + LabelValue: { + /** Label */ + label: string + /** Selected */ + selected: boolean + /** Value */ + value: string + } /** Organization */ Organization: { /** @@ -438,6 +1042,68 @@ export interface components { */ url: string } + /** RepeatParameterModel */ + RepeatParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_repeat + * @constant + * @enum {string} + */ + parameter_type?: "gx_repeat" + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + )[] + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } /** RepositoriesByCategory */ RepositoriesByCategory: { /** Description */ @@ -693,6 +1359,86 @@ export interface components { /** Stop Time */ stop_time: string } + /** RulesParameterModel */ + RulesParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_rules + * @constant + * @enum {string} + */ + parameter_type?: "gx_rules" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } + /** SelectParameterModel */ + SelectParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Multiple */ + multiple: boolean + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** Options */ + options?: components["schemas"]["LabelValue"][] | null + /** + * Parameter Type + * @default gx_select + * @constant + * @enum {string} + */ + parameter_type?: "gx_select" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + } /** Service */ Service: { /** @@ -762,6 +1508,56 @@ export interface components { */ version: string } + /** TextParameterModel */ + TextParameterModel: { + /** + * Area + * @default false + */ + area?: boolean + /** Argument */ + argument?: string | null + /** + * Default Options + * @default [] + */ + default_options?: components["schemas"]["LabelValue"][] + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_text + * @constant + * @enum {string} + */ + parameter_type?: "gx_text" + /** + * Refresh On Change + * @default false + */ + refresh_on_change?: boolean + /** Value */ + value?: string | null + } /** Tool */ Tool: { /** @@ -837,6 +1633,33 @@ export interface components { */ name?: string | null } + /** ToolParameterBundleModel */ + ToolParameterBundleModel: { + /** Input Models */ + input_models: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + )[] + } /** ToolVersion */ ToolVersion: { /** @@ -1768,6 +2591,59 @@ export interface operations { } } } + tools__parameter_model: { + /** Return Galaxy's meta model description of the tool's inputs */ + parameters: { + /** @description See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids */ + /** @description The full version string defined on the Galaxy tool wrapper. */ + path: { + tool_id: string + tool_version: string + } + } + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["ToolParameterBundleModel"] + } + } + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"] + } + } + } + } + tools__parameter_request_model: { + /** + * Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point + * @description The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution. + */ + parameters: { + /** @description See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids */ + /** @description The full version string defined on the Galaxy tool wrapper. */ + path: { + tool_id: string + tool_version: string + } + } + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": Record + } + } + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"] + } + } + } + } users__index: { /** * Index diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index 859b8fe2b096..a31ab4861f4a 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -644,6 +644,8 @@ def __str__(self): class RepositoryMetadata(Dictifiable): + repository: "Repository" + # Once the class has been mapped, all Column items in this table will be available # as instrumented class attributes on RepositoryMetadata. table = Table( diff --git a/test/functional/tools/parameters/cwl_boolean.cwl b/test/functional/tools/parameters/cwl_boolean.cwl new file mode 100644 index 000000000000..be6150d48920 --- /dev/null +++ b/test/functional/tools/parameters/cwl_boolean.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: boolean + +outputs: + output: boolean + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_boolean_optional.cwl b/test/functional/tools/parameters/cwl_boolean_optional.cwl new file mode 100644 index 000000000000..f05516dd5d80 --- /dev/null +++ b/test/functional/tools/parameters/cwl_boolean_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: boolean? + +outputs: + output: boolean? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_directory.cwl b/test/functional/tools/parameters/cwl_directory.cwl new file mode 100644 index 000000000000..66d3c5353079 --- /dev/null +++ b/test/functional/tools/parameters/cwl_directory.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: Directory + +outputs: + output: Directory + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_file.cwl b/test/functional/tools/parameters/cwl_file.cwl new file mode 100644 index 000000000000..ea48da6d2e82 --- /dev/null +++ b/test/functional/tools/parameters/cwl_file.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: File + +outputs: + output: File + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_float.cwl b/test/functional/tools/parameters/cwl_float.cwl new file mode 100644 index 000000000000..9e7e09469055 --- /dev/null +++ b/test/functional/tools/parameters/cwl_float.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: float + +outputs: + output: float + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_float_optional.cwl b/test/functional/tools/parameters/cwl_float_optional.cwl new file mode 100644 index 000000000000..1bc34fa1bc56 --- /dev/null +++ b/test/functional/tools/parameters/cwl_float_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: float? + +outputs: + output: float? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_int.cwl b/test/functional/tools/parameters/cwl_int.cwl new file mode 100644 index 000000000000..5ba0a8d5c76c --- /dev/null +++ b/test/functional/tools/parameters/cwl_int.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: int + +outputs: + output: int + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_int_optional.cwl b/test/functional/tools/parameters/cwl_int_optional.cwl new file mode 100644 index 000000000000..63d9c6915862 --- /dev/null +++ b/test/functional/tools/parameters/cwl_int_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: int? + +outputs: + output: int? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_string.cwl b/test/functional/tools/parameters/cwl_string.cwl new file mode 100644 index 000000000000..571a4cefc6ec --- /dev/null +++ b/test/functional/tools/parameters/cwl_string.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: string + +outputs: + output: string + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_string_optional.cwl b/test/functional/tools/parameters/cwl_string_optional.cwl new file mode 100644 index 000000000000..fecc8272d99a --- /dev/null +++ b/test/functional/tools/parameters/cwl_string_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: string? + +outputs: + output: string? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/gx_boolean.xml b/test/functional/tools/parameters/gx_boolean.xml new file mode 100644 index 000000000000..e42c9c9b6af6 --- /dev/null +++ b/test/functional/tools/parameters/gx_boolean.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_boolean_optional.xml b/test/functional/tools/parameters/gx_boolean_optional.xml new file mode 100644 index 000000000000..dc667b614a1a --- /dev/null +++ b/test/functional/tools/parameters/gx_boolean_optional.xml @@ -0,0 +1,82 @@ + + > '$output'; +cat '$inputs' >> $inputs_json; + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_color.xml b/test/functional/tools/parameters/gx_color.xml new file mode 100644 index 000000000000..fe158d0e6fb1 --- /dev/null +++ b/test/functional/tools/parameters/gx_color.xml @@ -0,0 +1,21 @@ + + + echo "$parameter" > $out_file1; + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean.xml b/test/functional/tools/parameters/gx_conditional_boolean.xml new file mode 100644 index 000000000000..7c5feffab0ec --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean.xml @@ -0,0 +1,101 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean_checked.xml b/test/functional/tools/parameters/gx_conditional_boolean_checked.xml new file mode 100644 index 000000000000..09fdbd71fe6d --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean_checked.xml @@ -0,0 +1,53 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml b/test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml new file mode 100644 index 000000000000..6bc790d81ad9 --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml @@ -0,0 +1,113 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean_optional.xml b/test/functional/tools/parameters/gx_conditional_boolean_optional.xml new file mode 100644 index 000000000000..69fa3d830499 --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean_optional.xml @@ -0,0 +1,79 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_conditional_boolean.xml b/test/functional/tools/parameters/gx_conditional_conditional_boolean.xml new file mode 100644 index 000000000000..343f3576cf6e --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_conditional_boolean.xml @@ -0,0 +1,30 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data.xml b/test/functional/tools/parameters/gx_data.xml new file mode 100644 index 000000000000..ea05c074c033 --- /dev/null +++ b/test/functional/tools/parameters/gx_data.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_collection.xml b/test/functional/tools/parameters/gx_data_collection.xml new file mode 100644 index 000000000000..5669f2921f64 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_collection.xml @@ -0,0 +1,14 @@ + + > '$output' + ]]> + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_collection_optional.xml b/test/functional/tools/parameters/gx_data_collection_optional.xml new file mode 100644 index 000000000000..9802176c4f76 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_collection_optional.xml @@ -0,0 +1,14 @@ + + > '$output' + ]]> + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_multiple.xml b/test/functional/tools/parameters/gx_data_multiple.xml new file mode 100644 index 000000000000..8529f2c7cac9 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_multiple.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_multiple_optional.xml b/test/functional/tools/parameters/gx_data_multiple_optional.xml new file mode 100644 index 000000000000..01b63bd83692 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_multiple_optional.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_optional.xml b/test/functional/tools/parameters/gx_data_optional.xml new file mode 100644 index 000000000000..3578d4e436e7 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_optional.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_float.xml b/test/functional/tools/parameters/gx_float.xml new file mode 100644 index 000000000000..5da8bc9790b9 --- /dev/null +++ b/test/functional/tools/parameters/gx_float.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_float_optional.xml b/test/functional/tools/parameters/gx_float_optional.xml new file mode 100644 index 000000000000..7d2ad2be396e --- /dev/null +++ b/test/functional/tools/parameters/gx_float_optional.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_hidden.xml b/test/functional/tools/parameters/gx_hidden.xml new file mode 100644 index 000000000000..e6da3bfb9279 --- /dev/null +++ b/test/functional/tools/parameters/gx_hidden.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_hidden_optional.xml b/test/functional/tools/parameters/gx_hidden_optional.xml new file mode 100644 index 000000000000..e3969f2a8074 --- /dev/null +++ b/test/functional/tools/parameters/gx_hidden_optional.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_int.xml b/test/functional/tools/parameters/gx_int.xml new file mode 100644 index 000000000000..e6f2e6758d26 --- /dev/null +++ b/test/functional/tools/parameters/gx_int.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_int_optional.xml b/test/functional/tools/parameters/gx_int_optional.xml new file mode 100644 index 000000000000..73b0141c064b --- /dev/null +++ b/test/functional/tools/parameters/gx_int_optional.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_repeat_boolean.xml b/test/functional/tools/parameters/gx_repeat_boolean.xml new file mode 100644 index 000000000000..c57ba06b4838 --- /dev/null +++ b/test/functional/tools/parameters/gx_repeat_boolean.xml @@ -0,0 +1,15 @@ + + > '$output' + ]]> + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_repeat_boolean_min.xml b/test/functional/tools/parameters/gx_repeat_boolean_min.xml new file mode 100644 index 000000000000..7356a6f2ce75 --- /dev/null +++ b/test/functional/tools/parameters/gx_repeat_boolean_min.xml @@ -0,0 +1,15 @@ + + > '$output' + ]]> + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_section_boolean.xml b/test/functional/tools/parameters/gx_section_boolean.xml new file mode 100644 index 000000000000..af948dd3276f --- /dev/null +++ b/test/functional/tools/parameters/gx_section_boolean.xml @@ -0,0 +1,35 @@ + + > '$output' + ]]> + +
+ +
+ + + + + + +
+ +
+ + + + + + + +
+ +
+ + + + + + + + diff --git a/test/functional/tools/parameters/gx_section_data.xml b/test/functional/tools/parameters/gx_section_data.xml new file mode 100644 index 000000000000..76e0e5734c16 --- /dev/null +++ b/test/functional/tools/parameters/gx_section_data.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + +
+ +
+ + + + + + +
+ +
+ + + + diff --git a/test/functional/tools/parameters/gx_select.xml b/test/functional/tools/parameters/gx_select.xml new file mode 100644 index 000000000000..a4f095b7f3cd --- /dev/null +++ b/test/functional/tools/parameters/gx_select.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_multiple.xml b/test/functional/tools/parameters/gx_select_multiple.xml new file mode 100644 index 000000000000..0e32bf9653cf --- /dev/null +++ b/test/functional/tools/parameters/gx_select_multiple.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_multiple_optional.xml b/test/functional/tools/parameters/gx_select_multiple_optional.xml new file mode 100644 index 000000000000..8e42fb8b14af --- /dev/null +++ b/test/functional/tools/parameters/gx_select_multiple_optional.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_optional.xml b/test/functional/tools/parameters/gx_select_optional.xml new file mode 100644 index 000000000000..5f6b63813dd3 --- /dev/null +++ b/test/functional/tools/parameters/gx_select_optional.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_text.xml b/test/functional/tools/parameters/gx_text.xml new file mode 100644 index 000000000000..16707f63e878 --- /dev/null +++ b/test/functional/tools/parameters/gx_text.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_text_optional.xml b/test/functional/tools/parameters/gx_text_optional.xml new file mode 100644 index 000000000000..41fe11cea418 --- /dev/null +++ b/test/functional/tools/parameters/gx_text_optional.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/macros.xml b/test/functional/tools/parameters/macros.xml new file mode 100644 index 000000000000..e47d243f75c4 --- /dev/null +++ b/test/functional/tools/parameters/macros.xml @@ -0,0 +1,33 @@ + + + > '$output'; +cat '$inputs' >> '$inputs_json'; + ]]> + + + + + + + + + This is a test tool used to establish and verify the behavior of some aspect of Galaxy's + parameter handling. + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/sample_tool_conf.xml b/test/functional/tools/sample_tool_conf.xml index 45eee3cc16c9..80c032c93601 100644 --- a/test/functional/tools/sample_tool_conf.xml +++ b/test/functional/tools/sample_tool_conf.xml @@ -266,6 +266,8 @@ + +
diff --git a/test/unit/tool_shed/_util.py b/test/unit/tool_shed/_util.py index df50c5270ef2..3002c6a82fab 100644 --- a/test/unit/tool_shed/_util.py +++ b/test/unit/tool_shed/_util.py @@ -18,6 +18,7 @@ from galaxy.util import safe_makedirs from tool_shed.context import ProvidesRepositoriesContext from tool_shed.managers.repositories import upload_tar_and_set_metadata +from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.managers.users import create_user from tool_shed.repository_types import util as rt_util from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry @@ -80,6 +81,7 @@ def __init__(self, temp_directory=None): self.config = TestToolShedConfig(temp_directory) self.security = IdEncodingHelper(id_secret=self.config.id_secret) self.repository_registry = tool_shed.repository_registry.Registry(self) + self.tool_state_cache = ToolStateCache(os.path.join(temp_directory, "tool_state_cache")) @property def security_agent(self): diff --git a/test/unit/tool_shed/test_tool_source.py b/test/unit/tool_shed/test_tool_source.py new file mode 100644 index 000000000000..cb4c4091376a --- /dev/null +++ b/test/unit/tool_shed/test_tool_source.py @@ -0,0 +1,33 @@ +from tool_shed.context import ProvidesRepositoriesContext +from tool_shed.managers.tools import ( + tool_input_models_cached_for, + tool_input_models_for, + tool_source_for, +) +from tool_shed.webapp.model import Repository +from ._util import upload_directories_to_repository + + +def test_get_tool(provides_repositories: ProvidesRepositoriesContext, new_repository: Repository): + upload_directories_to_repository(provides_repositories, new_repository, "column_maker") + owner = new_repository.user.username + name = new_repository.name + encoded_id = f"{owner}~{name}~Add_a_column1" + + repo_path = new_repository.repo_path(app=provides_repositories.app) + tool_source = tool_source_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) + assert tool_source.parse_id() == "Add_a_column1" + bundle = tool_input_models_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) + assert len(bundle.input_models) == 3 + + cached_bundle = tool_input_models_cached_for( + provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path + ) + assert len(cached_bundle.input_models) == 3 + + cached_bundle = tool_input_models_cached_for( + provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path + ) + assert len(cached_bundle.input_models) == 3 + + print(RequestToolState.parameter_model_for(cached_bundle).model_json_schema()) diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml new file mode 100644 index 000000000000..48e92c210ef0 --- /dev/null +++ b/test/unit/tool_util/parameter_specification.yml @@ -0,0 +1,631 @@ +# Tools to create. + +# Notes on conditional boolean values... +# - if you set truevalue/falsevalue - it doesn't look like the when can remain +# true/false - so go through and simplify that. means don't need to create test +# cases that test that. Linting also at very least warns on this. + +# - gx_conditional_boolean_empty_default +# - gx_conditional_boolean_empty_else +# - gx_conditional_select_* +# - gx_repeat_select_required +# - gx_repeat_repeat_select_required +# - gx_repeat_conditional_boolean_optional + +# Things to verify: +# - non optional, multi-selects require a selection (see TODO below...) +gx_int: + request_valid: + - parameter: 5 + - parameter: 6 + # galaxy parameters created with a value - so doesn't need to appear in request even though non-optional + - {} + request_invalid: + - parameter: null + - parameter: "null" + - parameter: "None" + - parameter: { 5 } + test_case_valid: + - parameter: 5 + - {} + test_case_invalid: + - parameter: null + - parameter: "5" + +gx_boolean: + request_valid: + - parameter: True + - parameter: False + - {} + request_invalid: + - parameter: {} + - parameter: "true" + # This is borderline... should we allow truevalue/falsevalue in the API. + # Marius and John were on fence here. + - parameter: "mytrue" + - parameter: null + +gx_int_optional: + request_valid: + - parameter: 5 + - parameter: null + - {} + request_invalid: + - parameter: "5" + - parameter: "None" + - parameter: "null" + - parameter: [5] + +gx_text: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + request_invalid: + - parameter: 5 + - parameter: null + - parameter: {} + - parameter: { "moo": "cow" } + +gx_text_optional: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + - parameter: null + request_invalid: + - parameter: 5 + - parameter: {} + - parameter: { "moo": "cow" } + +gx_select: + request_valid: + - parameter: "--ex1" + - parameter: "ex2" + request_invalid: + # Not allowing selecting booleans by truevalue/falsevalue - don't allow selecting + # selects by label. + - parameter: "Ex1" + # Do not allow lists for non-multi-selects + - parameter: ["ex2"] + - parameter: null + - parameter: {} + - parameter: 5 + - {} + +gx_select_optional: + request_valid: + - parameter: "--ex1" + - parameter: "ex2" + - parameter: null + - {} + request_invalid: + # Not allowing selecting booleans by truevalue/falsevalue - don't allow selecting + # selects by label. + - parameter: "Ex1" + # Do not allow lists for non-multi-selects + - parameter: ["ex2"] + - parameter: {} + - parameter: 5 + +# TODO: confirm null should vaguely not be allowed here +gx_select_multiple: + request_valid: + - parameter: ["--ex1"] + - parameter: ["ex2"] + request_invalid: + - parameter: ["Ex1"] + - parameter: null + - parameter: {} + - parameter: 5 + - {} + +gx_select_multiple_optional: + request_valid: + - parameter: ["--ex1"] + - parameter: ["ex2"] + - {} + - parameter: null + request_invalid: + - parameter: ["Ex1"] + - parameter: {} + - parameter: 5 + +gx_hidden: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + request_invalid: + - parameter: null + - parameter: 5 + - parameter: {} + - parameter: { "moo": "cow" } + +gx_hidden_optional: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + - parameter: null + request_invalid: + - parameter: 5 + - parameter: {} + - parameter: { "moo": "cow" } + +gx_float: + request_valid: + - parameter: 5 + - parameter: 5.0 + - parameter: 5.0001 + # galaxy parameters created with a value - so doesn't need to appear in request even though non-optional + - {} + request_invalid: + - parameter: null + - parameter: "5" + - parameter: "5.0" + - parameter: { "moo": "cow" } + +gx_float_optional: + request_valid: + - parameter: 5 + - parameter: 5.0 + - parameter: 5.0001 + - parameter: null + - {} + request_invalid: + - parameter: "5" + - parameter: "5.0" + - parameter: {} + - parameter: { "moo": "cow" } + +gx_color: + request_valid: + - parameter: '#aabbcc' + - parameter: '#000000' + request_invalid: + - parameter: null + - parameter: {} + - parameter: '#abcd' + +gx_data: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {__class__: "Batch", values: [{src: hdca, id: abcdabcd}]} + request_invalid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: {src: hda, id: 7} + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: {src: hda, id: 5} + - parameter: {src: hda, id: 0} + request_internal_invalid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: abcdabcd}]} + - parameter: {src: hda, id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_optional: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: null + - {} + request_invalid: + - parameter: {src: hda, id: 5} + - parameter: {src: hdca, id: abcdabcd} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: null + - {} + request_internal_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {src: hdca, id: abcdabcd} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_multiple: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {src: hdca, id: abcdabcd} + - parameter: [{src: hda, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] + request_invalid: + - parameter: {src: hda, id: 5} + - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + - parameter: null + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: {src: hdca, id: 5} + - parameter: [{src: hda, id: 5}] + - parameter: [{src: hdca, id: 5}] + - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + request_internal_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] + - parameter: null + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_multiple_optional: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {src: hdca, id: abcdabcd} + - parameter: [{src: hda, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] + - parameter: null + - {} + request_invalid: + - parameter: {src: hda, id: 5} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: {src: hdca, id: 5} + - parameter: [{src: hda, id: 5}] + - parameter: [{src: hdca, id: 5}] + - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + - parameter: null + - {} + request_internal_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_collection: + request_valid: + - parameter: {src: hdca, id: abcdabcd} + request_invalid: + - parameter: {src: hdca, id: 7} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hdca, id: 5} + request_internal_invalid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_collection_optional: + request_valid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - {} + request_invalid: + - parameter: {src: hdca, id: 7} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: true + - parameter: 5 + - parameter: "5" + - parameter: {} + request_internal_valid: + - parameter: {src: hdca, id: 5} + - parameter: null + - {} + request_internal_invalid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: true + - parameter: 5 + - parameter: "5" + - parameter: {} + +gx_conditional_boolean: + request_valid: + - conditional_parameter: + test_parameter: true + integer_parameter: 1 + - conditional_parameter: + test_parameter: true + integer_parameter: 2 + - conditional_parameter: + test_parameter: false + boolean_parameter: true + # Test parameter has default and so does it "case" - so this should be fine + - {} + # The boolean_parameter is optional so just setting a test_parameter is fine + - conditional_parameter: + test_parameter: true + - conditional_parameter: + test_parameter: false + # if test parameter is missing, it should be false in this case (TODO: test inverse) + # so boolean_parameter or either type or missing should be fine. + - conditional_parameter: + boolean_parameter: true + - conditional_parameter: + boolean_parameter: false + - conditional_parameter: {} + request_invalid: + - conditional_parameter: + test_parameter: false + integer_parameter: 1 + - conditional_parameter: + test_parameter: null + - conditional_parameter: + test_parameter: true + integer_parameter: "1" + - conditional_parameter: + test_parameter: true + integer_parameter: null + # if test parameter is missing, it should be false in this case + # in that case having an integer_parameter is not acceptable. + - conditional_parameter: + integer_parameter: 5 + +gx_conditional_boolean_checked: + request_valid: + # if no test parameter is defined, the default is 'checked' so the test + # parameter is true. + - conditional_parameter: + integer_parameter: 5 + - conditional_parameter: + integer_parameter: 0 + + request_invalid: + # if test parameter is missing, it should be true (it is 'checked') in this case + # in that case having a boolean_parameter is not acceptable. + - conditional_parameter: + boolean_parameter: false + +gx_conditional_conditional_boolean: + request_valid: + - outer_conditional_parameter: + outer_test_parameter: false + boolean_parameter: true + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: true + integer_parameter: 5 + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: false + boolean_parameter: true + # Test parameter has default and so does it "case" - so this should be fine + - {} + request_invalid: + - outer_conditional_parameter: + outer_test_parameter: true + boolean_parameter: true + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: false + integer_parameter: 5 + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: true + integer_parameter: true + +gx_repeat_boolean: + request_valid: + - parameter: + - { boolean_parameter: true } + - parameter: [] + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - parameter: [{}] + - parameter: [{}, {}] + request_invalid: + - parameter: + - { boolean_parameter: 4 } + - parameter: + - { foo: 4 } + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - { boolean_parameter: 4 } + - parameter: 5 + +gx_repeat_boolean_min: + request_valid: + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - parameter: [{}, {}] + request_invalid: + - parameter: [] + - parameter: [{}] + - parameter: + - { boolean_parameter: true } + - parameter: + - { boolean_parameter: 4 } + - parameter: + - { foo: 4 } + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - { boolean_parameter: 4 } + - parameter: 5 + +gx_section_boolean: + request_valid: + - parameter: { boolean_parameter: true } + # booleans are optional in requests, so this should be fine? + - {} + request_invalid: + - parameter: { boolean_parameter: 4 } + +gx_section_data: + request_valid: + - parameter: { data_parameter: { src: hda, id: abcdabcd } } + request_invalid: + - parameter: { data_parameter: 4 } + - parameter: { data_parameter: { src: hda, id: 5 } } + # data parameter is non-optional, so this should be invalid (unlike boolean parameter above) + # - {} + request_internal_valid: + - parameter: { data_parameter: { src: hda, id: 5 } } + request_internal_invalid: + - parameter: { data_parameter: { src: hda, id: abcdabcd } } +cwl_int: + request_valid: + - parameter: 5 + request_invalid: + - parameter: "5" + - {} + - parameter: null + - parameter: "None" + + +# TODO: Not a thing perhaps? +# cwl_null: +# request_valid: +# - parameter: null +# - {} +# request_invalid: +# - parameter: "5" +# - parameter: 5 +# - parameter: {} + +cwl_int_optional: + request_valid: + - parameter: 5 + - parameter: null + request_invalid: + - parameter: "5" + - {} + - parameter: "None" + +cwl_float: + request_valid: + - parameter: 5 + - parameter: 5.0 + request_invalid: + - parameter: null + - parameter: "5" + - {} + - parameter: "None" + +cwl_float_optional: + request_valid: + - parameter: 5 + - parameter: 5.0 + - parameter: null + request_invalid: + - parameter: "5" + - {} + - parameter: "None" + +cwl_string: + request_valid: + - parameter: "moo" + - parameter: "" + request_invalid: + - parameter: null + - {} + - parameter: 5 + +cwl_string_optional: + request_valid: + - parameter: "moo" + - parameter: "" + - parameter: null + request_invalid: + - {} + - parameter: 5 + +cwl_boolean: + request_valid: + - parameter: true + - parameter: false + request_invalid: + - parameter: null + - {} + - parameter: 5 + - parameter: "true" + - parameter: "True" + +cwl_boolean_optional: + request_valid: + - parameter: true + - parameter: false + - parameter: null + request_invalid: + - {} + - parameter: 5 + - parameter: "true" + - parameter: "True" + +cwl_file: + request_valid: + - parameter: {src: hda, id: abcdabcd} + request_invalid: + - parameter: {src: hda, id: 7} + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +cwl_directory: + request_valid: + - parameter: {src: hda, id: abcdabcd} + request_invalid: + - parameter: {src: hda, id: 7} + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + diff --git a/test/unit/tool_util/test_parameter_specification.py b/test/unit/tool_util/test_parameter_specification.py new file mode 100644 index 000000000000..5023710597ff --- /dev/null +++ b/test/unit/tool_util/test_parameter_specification.py @@ -0,0 +1,201 @@ +from functools import partial +from typing import ( + Any, + Callable, + Dict, + List, +) + +import yaml + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.tool_util.parameters import ( + decode, + encode, + RequestInternalToolState, + RequestToolState, + ToolParameterModel, + validate_internal_request, + validate_request, + validate_test_case, +) +from galaxy.tool_util.parameters.json import to_json_schema_string +from galaxy.tool_util.unittest_utils.parameters import ( + parameter_bundle, + parameter_bundle_for_file, + tool_parameter, +) +from galaxy.util.resources import resource_string + + +def specification_object(): + try: + yaml_str = resource_string(__package__, "parameter_specification.yml") + except AttributeError: + # hack for the main() function below where this file is interpreted as part of the + # Galaxy tree. + yaml_str = open("test/unit/tool_util/parameter_specification.yml").read() + return yaml.safe_load(yaml_str) + + +def test_specification(): + parameter_spec = specification_object() + for file in parameter_spec.keys(): + _test_file(file, parameter_spec) + + +def test_single(): + # _test_file("gx_int") + # _test_file("gx_float") + # _test_file("gx_boolean") + # _test_file("gx_int_optional") + # _test_file("gx_float_optional") + # _test_file("gx_conditional_boolean") + # _test_file("gx_conditional_conditional_boolean") + _test_file("gx_conditional_boolean_checked") + + +def _test_file(file: str, specification=None): + spec = specification or specification_object() + combos = spec[file] + tool_parameter_model = tool_parameter(file) + for valid_or_invalid, tests in combos.items(): + if valid_or_invalid == "request_valid": + _assert_requests_validate(tool_parameter_model, tests) + elif valid_or_invalid == "request_invalid": + _assert_requests_invalid(tool_parameter_model, tests) + elif valid_or_invalid == "request_internal_valid": + _assert_internal_requests_validate(tool_parameter_model, tests) + elif valid_or_invalid == "request_internal_invalid": + _assert_internal_requests_invalid(tool_parameter_model, tests) + elif valid_or_invalid == "test_case_valid": + _assert_test_cases_validate(tool_parameter_model, tests) + elif valid_or_invalid == "test_case_invalid": + _assert_test_cases_invalid(tool_parameter_model, tests) + + # Assume request validation will work here. + if "request_internal_valid" not in combos and "request_valid" in combos: + _assert_internal_requests_validate(tool_parameter_model, combos["request_valid"]) + if "request_internal_invalid" not in combos and "request_invvalid" in combos: + _assert_internal_requests_invalid(tool_parameter_model, combos["request_invalid"]) + + +def _for_each(test: Callable, parameter: ToolParameterModel, requests: List[Dict[str, Any]]) -> None: + for request in requests: + test(parameter, request) + + +def _assert_request_validates(parameter, request) -> None: + try: + validate_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate request {request}. {e}") + + +def _assert_request_invalid(parameter, request) -> None: + exc = None + try: + validate_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + exc = e + assert exc is not None, f"Parameter {parameter} didn't result in validation error on request {request} as expected." + + +def _assert_internal_request_validates(parameter, request) -> None: + try: + validate_internal_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate internal request {request}. {e}") + + +def _assert_internal_request_invalid(parameter, request) -> None: + exc = None + try: + validate_internal_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + exc = e + assert ( + exc is not None + ), f"Parameter {parameter} didn't result in validation error on internal request {request} as expected." + + +def _assert_test_case_validates(parameter, test_case) -> None: + try: + validate_test_case(parameter_bundle(parameter), test_case) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate test_case {test_case}. {e}") + + +def _assert_test_case_invalid(parameter, test_case) -> None: + exc = None + try: + validate_test_case(parameter_bundle(parameter), test_case) + except RequestParameterInvalidException as e: + exc = e + assert ( + exc is not None + ), f"Parameter {parameter} didn't result in validation error on test_case {test_case} as expected." + + +_assert_requests_validate = partial(_for_each, _assert_request_validates) +_assert_requests_invalid = partial(_for_each, _assert_request_invalid) +_assert_internal_requests_validate = partial(_for_each, _assert_internal_request_validates) +_assert_internal_requests_invalid = partial(_for_each, _assert_internal_request_invalid) +_assert_test_cases_validate = partial(_for_each, _assert_test_case_validates) +_assert_test_cases_invalid = partial(_for_each, _assert_test_case_invalid) + + +def decode_val(val: str) -> int: + assert val == "abcdabcd" + return 5 + + +def test_decode_gx_data(): + input_bundle = parameter_bundle_for_file("gx_data") + + request_tool_state = RequestToolState({"parameter": {"src": "hda", "id": "abcdabcd"}}) + request_internal_tool_state = decode(request_tool_state, input_bundle, decode_val) + assert request_internal_tool_state.input_state["parameter"]["id"] == 5 + assert request_internal_tool_state.input_state["parameter"]["src"] == "hda" + + +def test_decode_gx_int(): + input_bundle = parameter_bundle_for_file("gx_int") + + request_tool_state = RequestToolState({"parameter": 5}) + request_internal_tool_state = decode(request_tool_state, input_bundle, decode_val) + assert request_internal_tool_state.input_state["parameter"] == 5 + + +def test_json_schema_for_conditional(): + input_bundle = parameter_bundle_for_file("gx_conditional_boolean") + tool_state = RequestToolState.parameter_model_for(input_bundle) + print(to_json_schema_string(tool_state)) + + +def test_encode_gx_data(): + input_bundle = parameter_bundle_for_file("gx_data") + + def encode_val(val: int) -> str: + assert val == 5 + return "abcdabcd" + + request_internal_tool_state = RequestInternalToolState({"parameter": {"src": "hda", "id": 5}}) + request_tool_state = encode(request_internal_tool_state, input_bundle, encode_val) + assert request_tool_state.input_state["parameter"]["id"] == "abcdabcd" + assert request_tool_state.input_state["parameter"]["src"] == "hda" + + +if __name__ == "__main__": + parameter_spec = specification_object() + parameter_models_json = {} + for file in parameter_spec.keys(): + tool_parameter_model = tool_parameter(file) + parameter_models_json[file] = tool_parameter_model.dict() + yaml_str = yaml.safe_dump(parameter_models_json) + with open("client/src/components/Tool/parameter_models.yml", "w") as f: + f.write("# auto generated file for JavaScript testing, do not modify manually\n") + f.write("# -----\n") + f.write('# PYTHONPATH="lib" python test/unit/tool_util/test_parameter_specification.py\n') + f.write("# -----\n") + f.write(yaml_str) diff --git a/test/unit/tool_util/test_parameter_test_cases.py b/test/unit/tool_util/test_parameter_test_cases.py new file mode 100644 index 000000000000..38b0f2327b93 --- /dev/null +++ b/test/unit/tool_util/test_parameter_test_cases.py @@ -0,0 +1,29 @@ +from typing import List + +from galaxy.tool_util.parameters.case import test_case_state as case_state +from galaxy.tool_util.unittest_utils.parameters import ( + parameter_bundle_for_file, + parameter_tool_source, +) + + +def test_parameter_test_cases_validate(): + validate_test_cases_for("gx_int") + warnings = validate_test_cases_for("gx_float") + assert len(warnings[0]) == 0 + assert len(warnings[1]) == 1 + + +def validate_test_cases_for(tool_name: str) -> List[List[str]]: + tool_parameter_bundle = parameter_bundle_for_file(tool_name) + tool_source = parameter_tool_source(tool_name) + profile = tool_source.parse_profile() + test_cases = tool_source.parse_tests_to_dict()["tests"] + warnings_by_test = [] + for test_case in test_cases: + test_case_state_and_warnings = case_state(test_case, tool_parameter_bundle, profile) + tool_state = test_case_state_and_warnings.tool_state + warnings = test_case_state_and_warnings.warnings + assert tool_state.state_representation == "test_case" + warnings_by_test.append(warnings) + return warnings_by_test