Implement request header and body templating

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -4039,6 +4039,16 @@ of "type" specified for this expression block.
     </xs:restriction>
   </xs:simpleType>
 
+  <xs:simpleType name="RequestMethodType">
+    <xs:annotation>
+      <xs:documentation xml:lang="en">Select a request method, defaults to GET if unspecified</xs:documentation>
+    </xs:annotation>
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="GET" />
+      <xs:enumeration value="POST" />
+    </xs:restriction>
+  </xs:simpleType>
+
   <xs:complexType name="ParamSelectOption">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
@@ -4413,6 +4423,11 @@ used to generate dynamic options.
         <xs:documentation xml:lang="en">Determine options from data hosted at specified URL. </xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="request_method" type="RequestMethodType">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Set the request method to use for options provided using from_url. </xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="from_parameter" type="xs:string">
       <xs:annotation>
         <xs:documentation xml:lang="en">Deprecated.</xs:documentation>
@@ -4446,10 +4461,12 @@ used to generate dynamic options.
   </xs:complexType>
   <xs:group name="OptionsElement">
     <xs:choice>
-      <xs:element name="filter" type="Filter" minOccurs="0" maxOccurs="unbounded"/>
-      <xs:element name="column" type="Column" minOccurs="0" maxOccurs="unbounded"/>
-      <xs:element name="validator" type="Validator" minOccurs="0" maxOccurs="1"/>
-      <xs:element name="postprocess_expression" type="Expression" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="filter" type="Filter" minOccurs="0" maxOccurs="unbounded" />
+      <xs:element name="column" type="Column" minOccurs="0" maxOccurs="unbounded" />
+      <xs:element name="validator" type="Validator" minOccurs="0" maxOccurs="1" />
+      <xs:element name="postprocess_expression" type="Expression" minOccurs="0" maxOccurs="1" />
+      <xs:element name="request_body" type="RequestBody" minOccurs="0" maxOccurs="1" />
+      <xs:element name="request_headers" type="RequestHeaders" minOccurs="0" maxOccurs="1" />
       <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded">
         <xs:annotation>
           <xs:documentation xml:lang="en">Documentation for file</xs:documentation>
@@ -4458,6 +4475,20 @@ used to generate dynamic options.
       <xs:element name="option" type="ParamDrillDownOption" minOccurs="0" maxOccurs="unbounded"/>
     </xs:choice>
   </xs:group>
+  <xs:complexType name="RequestBody">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="type" type="xs:string"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+  <xs:complexType name="RequestHeaders">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="type" type="xs:string"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
   <xs:complexType name="Column">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[Optionally contained within an

lib/galaxy/tools/parameters/dynamic_options.py

@@ -4,10 +4,21 @@
 """
 
 import copy
+import json
 import logging
 import os
 import re
+from dataclasses import dataclass
 from io import StringIO
+from typing import (
+    Any,
+    cast,
+    Dict,
+    get_args,
+    Optional,
+)
+
+from typing_extensions import Literal
 
 from galaxy.model import (
     DatasetCollectionElement,
@@ -17,7 +28,10 @@
     User,
 )
 from galaxy.tools.expressions import do_eval
-from galaxy.util import string_as_bool
+from galaxy.util import (
+    Element,
+    string_as_bool,
+)
 from galaxy.util.template import fill_template
 from . import validation
 from .cancelable_request import request
@@ -538,7 +552,7 @@ def filter_options(self, options, trans, other_values):
 class DynamicOptions:
     """Handles dynamically generated SelectToolParameter options"""
 
-    def __init__(self, elem, tool_param):
+    def __init__(self, elem: Element, tool_param):
         def load_from_parameter(from_parameter, transform_lines=None):
             obj = self.tool_param
             for field in from_parameter.split("."):
@@ -548,7 +562,7 @@ def load_from_parameter(from_parameter, transform_lines=None):
             return self.parse_file_fields(obj)
 
         self.tool_param = tool_param
-        self.columns = {}
+        self.columns: Dict[str, int] = {}
         self.filters = []
         self.file_fields = None
         self.largest_index = 0
@@ -568,11 +582,7 @@ def load_from_parameter(from_parameter, transform_lines=None):
         dataset_file = elem.get("from_dataset", None)
         from_parameter = elem.get("from_parameter", None)
         self.tool_data_table_name = elem.get("from_data_table", None)
-        self.from_url = elem.get("from_url")
-        self.from_url_postprocess = None
-        from_url_postprocess = elem.find("postprocess_expression")
-        if from_url_postprocess is not None:
-            self.from_url_postprocess = from_url_postprocess.text.strip()
+        self.from_url_options = parse_from_url_options(elem)
         # Options are defined from a data table loaded by the app
         self._tool_data_table = None
         self.elem = elem
@@ -781,25 +791,35 @@ def to_triple(values):
             else:
                 return [str(values[0]), str(values[1]), bool(values[2])]
 
-        if self.from_url:
+        if from_url_options := self.from_url_options:
             context = User.user_template_environment(trans.user)
-            url = fill_template(self.from_url, context)
+            url = fill_template(from_url_options.from_url, context)
+            request_body = template_or_none(from_url_options.request_body, context)
+            request_headers = template_or_none(from_url_options.request_headers, context)
             try:
                 unset_value = object()
-                cached_value = trans.get_cache_value(url, unset_value)
+                cached_value = trans.get_cache_value(
+                    (url, from_url_options.request_method, request_body, request_headers), unset_value
+                )
                 if cached_value is unset_value:
-                    data = request(url, timeout=10)
-                    trans.set_cache_value(url, data)
+                    data = request(
+                        url=url,
+                        method=from_url_options.request_method,
+                        data=json.loads(request_body) if request_body else None,
+                        headers=json.loads(request_headers) if request_headers else None,
+                        timeout=10,
+                    )
+                    trans.set_cache_value((url, from_url_options.request_method, request_body, request_headers), data)
                 else:
                     data = cached_value
             except Exception as e:
                 log.warning("Fetching from url '%s' failed: %s", url, str(e))
                 data = None
 
-            if self.from_url_postprocess:
+            if from_url_options.postprocess_expression:
                 try:
                     data = do_eval(
-                        self.from_url_postprocess,
+                        from_url_options.postprocess_expression,
                         data,
                     )
                 except Exception as eval_error:
@@ -836,6 +856,46 @@ def column_spec_to_index(self, column_spec):
         return int(column_spec)
 
 
+@dataclass
+class FromUrlOptions:
+    from_url: str
+    request_method: Literal["GET", "POST"]
+    request_body: Optional[str]
+    request_headers: Optional[str]
+    postprocess_expression: Optional[str]
+
+
+def strip_or_none(maybe_string: Optional[Element]) -> Optional[str]:
+    if maybe_string is not None:
+        if maybe_string.text:
+            return maybe_string.text.strip()
+    return None
+
+
+def parse_from_url_options(elem: Element) -> Optional[FromUrlOptions]:
+    from_url = elem.get("from_url")
+    if from_url:
+        request_method = cast(Literal["GET", "POST"], elem.get("request_method", "GET"))
+        assert request_method in get_args(FromUrlOptions.request_headers)
+        request_headers = strip_or_none(elem.find("request_headers"))
+        request_body = strip_or_none(elem.find("request_body"))
+        postprocess_expression = strip_or_none(elem.find("postprocess_expression"))
+        return FromUrlOptions(
+            from_url,
+            request_method=request_method,
+            request_headers=request_headers,
+            request_body=request_body,
+            postprocess_expression=postprocess_expression,
+        )
+    return None
+
+
+def template_or_none(template: Optional[str], context: Dict[str, Any]) -> Optional[str]:
+    if template:
+        return fill_template(template, context=context)
+    return None
+
+
 def _get_ref_data(other_values, ref_name):
     """
     get the list of data sets from ref_name

lib/galaxy/work/context.py

@@ -4,6 +4,7 @@
     Dict,
     List,
     Optional,
+    Tuple,
 )
 
 from typing_extensions import Literal
@@ -44,15 +45,15 @@ def __init__(
         self.__user_current_roles: Optional[List[Role]] = None
         self.__history = history
         self._url_builder = url_builder
-        self._short_term_cache: Dict[str, Any] = {}
+        self._short_term_cache: Dict[Tuple[str, ...], Any] = {}
         self.workflow_building_mode = workflow_building_mode
         self.galaxy_session = galaxy_session
 
-    def set_cache_value(self, key: str, value: Any):
-        self._short_term_cache[key] = value
+    def set_cache_value(self, args: Tuple[str, ...], value: Any):
+        self._short_term_cache[args] = value
 
-    def get_cache_value(self, key: str, default: Any = None) -> Any:
-        return self._short_term_cache.get(key, default)
+    def get_cache_value(self, args: Tuple[str, ...], default: Any = None) -> Any:
+        return self._short_term_cache.get(args, default)
 
     @property
     def app(self):

test/functional/tools/select_from_url.xml

@@ -2,7 +2,8 @@
     <command><![CDATA[
 echo '$url_param_value' > '$param_value' &&
 echo '$url_param_value_postprocessed' > '$param_value_postprocessed' &&
-echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postprocessed'
+echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postprocessed' &&
+echo '$url_param_value_header_and_body' > '$param_value_header_and_body'
     ]]></command>
     <inputs>
         <param name="url_param_value" type="select">
@@ -33,17 +34,34 @@ echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postproces
                 }]]></postprocess_expression>
             </options>
         </param>
+        <param name="url_param_value_header_and_body" type="select">
+            <options from_url="https://postman-echo.com/post" request_method="POST">
+                <!-- Example for accessing user secrets via extra preferences -->
+                <request_headers type="json">
+                    {"x-api-key": "${__user__.extra_preferences.fake_api_key if $__user__ else "anon"}"}
+                </request_headers>
+                <request_body type="json">
+                    {"name": "value"}
+                </request_body>
+                <!-- https://postman-echo.com/post echos values sent to it, so here's we're listing the response headers -->
+                <postprocess_expression type="ecma5.1"><![CDATA[${
+                    return Object.keys(inputs.headers).map((header) => [header, header])
+                }]]></postprocess_expression>
+            </options>
+        </param>
     </inputs>
     <outputs>
         <data format="txt" label="url param value" name="param_value"></data>
         <data format="txt" label="url param value postprocessed" name="param_value_postprocessed"></data>
         <data format="txt" label="invalid url param value postprocessed" name="invalid_param_value_postprocessed"></data>
+        <data format="txt" label="param value for header and body request" name="param_value_header_and_body"></data>
     </outputs>
     <tests>
         <test>
             <param name="url_param_value" value="dm6" />
             <param name="url_param_value_postprocessed" value="chr2L" />
             <param name="invalid_url_param_value_postprocessed" value="default" />
+            <param name="url_param_value_header_and_body" value="x-api-key" />
             <output name="param_value">
                 <assert_contents>
                     <has_text text="dm6"></has_text>
@@ -59,6 +77,11 @@ echo '$invalid_url_param_value_postprocessed' > '$invalid_param_value_postproces
                     <has_text text="default"></has_text>
                 </assert_contents>
             </output>
+            <output name="param_value_header_and_body">
+                <assert_contents>
+                    <has_text text="x-api-key"></has_text>
+                </assert_contents>
+            </output>
         </test>
     </tests>
 </tool>

test/unit/app/tools/test_dynamic_options.py

@@ -0,0 +1,57 @@
+from galaxy.app_unittest_utils.galaxy_mock import MockApp
+from galaxy.tools.parameters.dynamic_options import DynamicOptions
+from galaxy.util import XML
+from galaxy.util.bunch import Bunch
+from galaxy.work.context import WorkRequestContext
+
+
+def get_from_url_option():
+    return DynamicOptions(
+        XML(
+            """
+<options from_url="https://usegalaxy.org/api/genomes/dm6" request_method="POST">
+    <request_headers type="json">
+        {"x-api-key": "${__user__.extra_preferences.resource_api_key if $__user__ else "anon"}"}
+    </request_headers>
+    <request_body type="json">
+        {"some_key": "some_value"}
+    </request_body>
+    <postprocess_expression type="ecma5.1"><![CDATA[${
+        if (inputs) {
+            return Object.values(inputs.chrom_info).map((v) => [v.chrom, v.len])
+        } else {
+            return [["The fallback value", "default"]]
+        }
+    }]]></postprocess_expression>
+</options>
+"""
+        ),
+        Bunch(),
+    )
+
+
+def test_dynamic_option_parsing():
+    from_url_option = get_from_url_option()
+    assert from_url_option.from_url_options
+    assert from_url_option.from_url_options.from_url == "https://usegalaxy.org/api/genomes/dm6"
+
+
+def test_dynamic_option_cache():
+    app = MockApp()
+    trans = WorkRequestContext(app=app)
+    from_url_option = get_from_url_option()
+    options = from_url_option.from_url_options
+    assert options
+    args = (options.from_url, options.request_method, options.request_body, '{"x-api-key": "anon"}')
+    trans.set_cache_value(
+        args,
+        {
+            "id": "dm6",
+            "reference": True,
+            "chrom_info": [{"chrom": "chr2L", "len": 23513712}],
+            "prev_chroms": False,
+            "next_chroms": False,
+            "start_index": 0,
+        },
+    )
+    assert from_url_option.get_options(trans, {}) == [["chr2L", "23513712", False]]