Input parameter schema.

.gitignore

@@ -153,6 +153,7 @@ doc/build
 doc/schema.md
 doc/source/admin/config_logging_default_yaml.rst
 doc/source/dev/schema.md
+doc/source/dev/plantuml.jar
 client/docs/dist
 
 # Webpack stats

doc/source/dev/image.Makefile

@@ -0,0 +1,11 @@
+MINDMAPS := $(wildcard *.mindmap.yml)
+INPUTS := $(wildcard *.plantuml.txt)
+OUTPUTS := $(INPUTS:.txt=.svg)
+
+all: plantuml.jar $(MINDMAPS) $(OUTPUTS)
+
+$(OUTPUTS): $(INPUTS) $(MINDMAPS)
+	java -jar plantuml.jar -c plantuml_options.txt -tsvg $(INPUTS)
+
+plantuml.jar:
+	wget http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar || curl --output plantuml.jar http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar

doc/source/dev/plantuml_options.txt

@@ -0,0 +1,51 @@
+' skinparam handwritten true
+' skinparam roundcorner 20
+
+skinparam class {
+    ArrowFontColor DarkOrange
+    BackgroundColor #FFEFD5
+    ArrowColor Orange
+    BorderColor DarkOrange
+}
+
+skinparam object {
+    ArrowFontColor DarkOrange
+    BackgroundColor #FFEFD5
+    BackgroundColor #FFEFD5
+    ArrowColor Orange
+    BorderColor DarkOrange
+}
+
+skinparam ComponentBackgroundColor #FFEFD5
+skinparam ComponentBorderColor DarkOrange
+
+skinparam DatabaseBackgroundColor #FFEFD5
+skinparam DatabaseBorderColor DarkOrange
+
+skinparam StorageBackgroundColor #FFEFD5
+skinparam StorageBorderColor DarkOrange
+
+skinparam QueueBackgroundColor #FFEFD5
+skinparam QueueBorderColor DarkOrange
+
+skinparam note {
+    BackgroundColor #FFEFD5
+    BorderColor #BF5700
+}
+
+skinparam sequence {
+    ArrowColor Orange
+    ArrowFontColor DarkOrange
+    ActorBorderColor DarkOrange
+    ActorBackgroundColor #FFEFD5
+
+    ParticipantBorderColor DarkOrange
+    ParticipantBackgroundColor #FFEFD5
+
+    LifeLineBorderColor DarkOrange
+    LifeLineBackgroundColor #FFEFD5
+
+    DividerBorderColor DarkOrange
+    GroupBorderColor DarkOrange
+}
+

doc/source/dev/plantuml_style.txt

@@ -0,0 +1,9 @@
+<style>
+mindmapDiagram {
+    node {
+        BackgroundColor #FFEFD5
+        BorderColor DarkOrange
+        LineColor Orange
+    }
+}
+</style>

doc/source/dev/tool_state_api.plantuml.txt

@@ -0,0 +1,17 @@
+@startuml
+'!include plantuml_options.txt
+participant "API Request" as apireq
+boundary    "Jobs API"    as api
+participant     "Job Service" as service
+database    Database    as database
+queue       TaskQueue   as queue
+apireq -> api : HTTP JSON
+api -> service : To boundary
+service -> service : Build RequestToolState
+service -> service : Validate RequestToolState (pydantic)
+service -> service : decode() RequestToolState \ninto RequestInternalToolState
+service -> database : Serialize RequestInternalToolState
+service -> queue : Queue QueueJobs with reference to\npersisted RequestInternalToolState
+service -> api : JobCreateResponse\n (pydantic model)
+api -> apireq : JobCreateResponse\n (as json)
+@enduml

doc/source/dev/tool_state_state_classes.plantuml.txt

@@ -0,0 +1,41 @@
+@startuml
+!include plantuml_options.txt
+
+package galaxy.tool_util.parameters.state {
+
+class ToolState {
+state_representation: str
+input_state: Dict[str, Any]
++ validate(input_models: ToolParameterBundle)
++ {abstract} _to_base_model(input_models: ToolParameterBundle): Optional[Type[BaseModel]]
+}
+
+class RequestToolState {
+state_representation = "request"
++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel]
+}
+note bottom: Object references of the form \n{src: "hda", id: <encoded_id>}.\n Allow mapping/reduce constructs.
+
+class RequestInternalToolState {
+state_representation = "request_internal"
++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel]
+}
+note bottom: Object references of the form \n{src: "hda", id: <decoded_id>}.\n Allow mapping/reduce constructs.
+
+class JobInternalToolState {
+state_representation = "job_internal"
++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel]
+
+}
+note bottom: Object references of the form \n{src: "hda", id: <decoded_id>}.\n Mapping constructs expanded out.\n (Defaults are inserted?)
+
+ToolState <|-- RequestToolState
+ToolState <|-- RequestInternalToolState
+ToolState <|-- JobInternalToolState
+
+RequestToolState - RequestInternalToolState : decode >
+
+RequestInternalToolState o-- JobInternalToolState : expand >
+
+}
+@enduml

lib/galaxy/config/schemas/tool_shed_config_schema.yml

@@ -102,6 +102,13 @@ mapping:
           the repositories and tools within the Tool Shed given that you specify
           the following two config options.
 
+      tool_state_cache_dir:
+        type: str
+        default: database/tool_state_cache
+        required: false
+        desc: |
+          Cache directory for tool state.
+
       repo_name_boost:
         type: float
         default: 0.9

lib/galaxy/tool_util/cwl/parser.py

@@ -144,6 +144,10 @@ def galaxy_id(self) -> str:
             tool_id = tool_id[1:]
         return tool_id
 
+    @abstractmethod
+    def input_fields(self) -> list:
+        """Return InputInstance objects describing mapping to Galaxy inputs."""
+
     @abstractmethod
     def input_instances(self):
         """Return InputInstance objects describing mapping to Galaxy inputs."""
@@ -236,7 +240,7 @@ def label(self):
         else:
             return ""
 
-    def input_fields(self):
+    def input_fields(self) -> list:
         input_records_schema = self._eval_schema(self._tool.inputs_record_schema)
         if input_records_schema["type"] != "record":
             raise Exception("Unhandled CWL tool input structure")

lib/galaxy/tool_util/parameters/__init__.py

@@ -0,0 +1,99 @@
+from .convert import (
+    decode,
+    encode,
+)
+from .factory import (
+    from_input_source,
+    input_models_for_pages,
+    input_models_for_tool_source,
+    input_models_from_json,
+    tool_parameter_bundle_from_json,
+)
+from .json import to_json_schema_string
+from .models import (
+    BooleanParameterModel,
+    ColorParameterModel,
+    ConditionalParameterModel,
+    ConditionalWhen,
+    CwlBooleanParameterModel,
+    CwlDirectoryParameterModel,
+    CwlFileParameterModel,
+    CwlFloatParameterModel,
+    CwlIntegerParameterModel,
+    CwlNullParameterModel,
+    CwlStringParameterModel,
+    CwlUnionParameterModel,
+    DataCollectionParameterModel,
+    DataParameterModel,
+    FloatParameterModel,
+    HiddenParameterModel,
+    IntegerParameterModel,
+    LabelValue,
+    RepeatParameterModel,
+    RulesParameterModel,
+    SelectParameterModel,
+    TextParameterModel,
+    ToolParameterBundle,
+    ToolParameterBundleModel,
+    ToolParameterModel,
+    ToolParameterT,
+    validate_against_model,
+    validate_internal_request,
+    validate_request,
+    validate_test_case,
+)
+from .state import (
+    JobInternalToolState,
+    RequestInternalToolState,
+    RequestToolState,
+    TestCaseToolState,
+    ToolState,
+)
+from .visitor import visit_input_values
+
+__all__ = (
+    "from_input_source",
+    "input_models_for_pages",
+    "input_models_for_tool_source",
+    "tool_parameter_bundle_from_json",
+    "input_models_from_json",
+    "JobInternalToolState",
+    "ToolParameterBundle",
+    "ToolParameterBundleModel",
+    "ToolParameterModel",
+    "IntegerParameterModel",
+    "BooleanParameterModel",
+    "CwlFileParameterModel",
+    "CwlFloatParameterModel",
+    "CwlIntegerParameterModel",
+    "CwlStringParameterModel",
+    "CwlNullParameterModel",
+    "CwlUnionParameterModel",
+    "CwlBooleanParameterModel",
+    "CwlDirectoryParameterModel",
+    "TextParameterModel",
+    "FloatParameterModel",
+    "HiddenParameterModel",
+    "ColorParameterModel",
+    "RulesParameterModel",
+    "DataParameterModel",
+    "DataCollectionParameterModel",
+    "LabelValue",
+    "SelectParameterModel",
+    "ConditionalParameterModel",
+    "ConditionalWhen",
+    "RepeatParameterModel",
+    "validate_against_model",
+    "validate_internal_request",
+    "validate_request",
+    "validate_test_case",
+    "ToolState",
+    "TestCaseToolState",
+    "ToolParameterT",
+    "to_json_schema_string",
+    "RequestToolState",
+    "RequestInternalToolState",
+    "visit_input_values",
+    "decode",
+    "encode",
+)

lib/galaxy/tool_util/parameters/_types.py

@@ -0,0 +1,53 @@
+"""Type utilities for building pydantic models for tool parameters.
+
+Lots of mypy exceptions in here - this code is all well tested and the exceptions
+are fine otherwise because we're using the typing system to interact with pydantic
+and build runtime models not to use mypy to type check static code.
+"""
+
+from typing import (
+    Any,
+    cast,
+    Generic,
+    List,
+    Optional,
+    Type,
+    Union,
+)
+
+# https://stackoverflow.com/questions/56832881/check-if-a-field-is-typing-optional
+# Python >= 3.8
+try:
+    from typing import get_args  # type: ignore[attr-defined,unused-ignore]
+    from typing import get_origin  # type: ignore[attr-defined,unused-ignore]
+# Compatibility
+except ImportError:
+
+    def get_args(tp: Any) -> tuple:
+        return getattr(tp, "__args__", ()) if tp is not Generic else Generic  # type: ignore[return-value,assignment,unused-ignore]
+
+    def get_origin(tp: Any) -> Optional[Any]:
+        return getattr(tp, "__origin__", None)
+
+
+def optional_if_needed(type: Type, is_optional: bool) -> Type:
+    return_type: Type = type
+    if is_optional:
+        return_type = Optional[type]  # type: ignore[assignment]
+    return return_type
+
+
+def union_type(args: List[Type]) -> Type:
+    return Union[tuple(args)]  # type: ignore[return-value]
+
+
+def list_type(arg: Type) -> Type:
+    return List[arg]  # type: ignore[valid-type]
+
+
+def cast_as_type(arg) -> Type:
+    return cast(Type, arg)
+
+
+def is_optional(field) -> bool:
+    return get_origin(field) is Union and type(None) in get_args(field)