WIP: file source templates

lib/galaxy/files/templates/__init__.py

@@ -0,0 +1,17 @@
+from .manager import ConfiguredFileSourceTemplates
+from .models import (
+    FileSourceConfiguration,
+    FileSourceTemplate,
+    FileSourceTemplateSummaries,
+    FileSourceTemplateType,
+    template_to_configuration,
+)
+
+__all__ = (
+    "ConfiguredFileSourceTemplates",
+    "FileSourceConfiguration",
+    "FileSourceTemplate",
+    "FileSourceTemplateSummaries",
+    "FileSourceTemplateType",
+    "template_to_configuration",
+)

lib/galaxy/files/templates/manager.py

@@ -0,0 +1,17 @@
+from typing import List
+
+from galaxy.util.config_templates import (
+    apply_syntactic_sugar,
+    RawTemplateConfig,
+)
+
+from .models import FileSourceTemplateCatalog
+
+
+class ConfiguredFileSourceTemplates:
+    catalog: FileSourceTemplateCatalog
+
+
+def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> FileSourceTemplateCatalog:
+    effective_root = apply_syntactic_sugar(raw_config)
+    return FileSourceTemplateCatalog.model_validate(effective_root)

lib/galaxy/files/templates/models.py

@@ -0,0 +1,126 @@
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Type,
+    Union,
+)
+
+from pydantic import RootModel
+
+from galaxy.util.config_templates import (
+    expand_raw_config,
+    MarkdownContent,
+    StrictModel,
+    TemplateExpansion,
+    TemplateSecret,
+    TemplateVariable,
+    TemplateVariableType,
+    TemplateVariableValueType,
+)
+
+FileSourceTemplateType = Literal["posix", "s3fs"]
+
+
+class PosixFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["posix"]
+    root: Union[str, TemplateExpansion]
+
+
+class PosixFileSourceConfiguration(StrictModel):
+    type: Literal["posix"]
+    root: str
+
+
+class S3FSFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["s3fs"]
+    endpoint_url: Optional[Union[str, TemplateExpansion]] = None
+    anon: Optional[Union[bool, TemplateExpansion]] = False
+    secret: Optional[Union[str, TemplateExpansion]] = None
+    key: Optional[Union[str, TemplateExpansion]] = None
+    bucket: Optional[Union[str, TemplateExpansion]] = None
+
+
+class S3FSFileSourceConfiguration(StrictModel):
+    type: Literal["s3fs"]
+    endpoint_url: Optional[str] = None
+    anon: Optional[bool] = False
+    secret: Optional[str] = None
+    key: Optional[str] = None
+    bucket: Optional[str] = None
+
+
+FileSourceTemplateConfiguration = Union[
+    PosixFileSourceTemplateConfiguration,
+    S3FSFileSourceTemplateConfiguration,
+]
+FileSourceConfiguration = Union[
+    PosixFileSourceConfiguration,
+    S3FSFileSourceConfiguration,
+]
+
+
+class FileSourceTemplateBase(StrictModel):
+    """Version of FileSourceTemplate we can send to the UI/API.
+
+    The configuration key in the child type may have secretes
+    and shouldn't be exposed over the API - at least to non-admins.
+    """
+
+    id: str
+    name: Optional[str]
+    description: Optional[MarkdownContent]
+    # The UI should just show the most recent version but allow
+    # admins to define newer versions with new parameterizations
+    # and keep old versions in template catalog for backward compatibility
+    # for users with existing stores of that template.
+    version: int = 0
+    # Like with multiple versions, allow admins to deprecate a
+    # template by hiding but keep it in the catalog for backward
+    # compatibility for users with existing stores of that template.
+    hidden: bool = False
+    variables: Optional[List[TemplateVariable]] = None
+    secrets: Optional[List[TemplateSecret]] = None
+
+
+class FileSourceTemplateSummary(FileSourceTemplateBase):
+    type: FileSourceTemplateType
+
+
+class FileSourceTemplate(FileSourceTemplateBase):
+    configuration: FileSourceTemplateConfiguration
+
+
+FileSourceTemplateCatalog = RootModel[List[FileSourceTemplate]]
+
+
+class FileSourceTemplateSummaries(RootModel):
+    root: List[FileSourceTemplateSummary]
+
+
+def template_to_configuration(
+    template: FileSourceTemplate,
+    variables: Dict[str, TemplateVariableValueType],
+    secrets: Dict[str, str],
+    user_details: Dict[str, Any],
+) -> FileSourceConfiguration:
+    configuration_template = template.configuration
+    raw_config = expand_raw_config(configuration_template, variables, secrets, user_details)
+    return to_configuration_object(raw_config)
+
+
+TypesToConfigurationClasses: Dict[FileSourceTemplateType, Type[FileSourceConfiguration]] = {
+    "posix": PosixFileSourceConfiguration,
+    "s3fs": S3FSFileSourceConfiguration,
+}
+
+
+def to_configuration_object(configuration_dict: Dict[str, Any]) -> FileSourceConfiguration:
+    if "type" not in configuration_dict:
+        raise KeyError("Configuration objects require a file source 'type' key, none found.")
+    object_store_type = configuration_dict["type"]
+    if object_store_type not in TypesToConfigurationClasses:
+        raise ValueError(f"Unknown file source type found in raw configuration dictionary ({object_store_type}).")
+    return TypesToConfigurationClasses[object_store_type](**configuration_dict)

lib/galaxy/objectstore/templates/manager.py

@@ -14,14 +14,16 @@
     RequestParameterMissingException,
 )
 from galaxy.objectstore.badges import serialize_badges
+from galaxy.util.config_templates import (
+    apply_syntactic_sugar,
+    RawTemplateConfig,
+)
 from .models import (
     ObjectStoreTemplate,
     ObjectStoreTemplateCatalog,
     ObjectStoreTemplateSummaries,
 )
 
-RawTemplateConfig = Dict[str, Any]
-
 
 class AppConfigProtocol(Protocol):
     object_store_templates: Optional[List[RawTemplateConfig]]
@@ -118,24 +120,5 @@ def validate(self, instance: InstanceDefinition):
 
 
 def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> ObjectStoreTemplateCatalog:
-    effective_root = _apply_syntactic_sugar(raw_config)
+    effective_root = apply_syntactic_sugar(raw_config)
     return ObjectStoreTemplateCatalog.model_validate(effective_root)
-
-
-def _apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]:
-    templates = []
-    for template in raw_templates:
-        _force_key_to_list(template, "variables")
-        _force_key_to_list(template, "secrets")
-        templates.append(template)
-    return templates
-
-
-def _force_key_to_list(template: RawTemplateConfig, key: str) -> None:
-    value = template.get(key, None)
-    if isinstance(value, dict):
-        value_as_list = []
-        for key_name, key_value in value.items():
-            key_value["name"] = key_name
-            value_as_list.append(key_value)
-        template[key] = value_as_list

lib/galaxy/objectstore/templates/models.py

@@ -7,28 +7,26 @@
     Union,
 )
 
-from boltons.iterutils import remap
-from jinja2.nativetypes import NativeEnvironment
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    RootModel,
-)
+from pydantic import RootModel
 from typing_extensions import Literal
 
 from galaxy.objectstore.badges import (
     BadgeDict,
     StoredBadgeDict,
 )
+from galaxy.util.config_templates import (
+    expand_raw_config,
+    MarkdownContent,
+    StrictModel,
+    TemplateExpansion,
+    TemplateSecret,
+    TemplateVariable,
+    TemplateVariableType,
+    TemplateVariableValueType,
+)
 
-
-class StrictModel(BaseModel):
-    model_config = ConfigDict(extra="forbid")
-
-
-ObjectStoreTemplateVariableType = Literal["string", "boolean", "integer"]
-ObjectStoreTemplateVariableValueType = Union[str, bool, int]
-TemplateExpansion = str
+ObjectStoreTemplateVariableType = TemplateVariableType
+ObjectStoreTemplateVariableValueType = TemplateVariableValueType
 ObjectStoreTemplateType = Literal["s3", "azure_blob", "disk", "generic_s3"]
 
 
@@ -155,18 +153,10 @@ class GenericS3ObjectStoreConfiguration(StrictModel):
     AzureObjectStoreConfiguration,
     GenericS3ObjectStoreConfiguration,
 ]
-MarkdownContent = str
 
 
-class ObjectStoreTemplateVariable(StrictModel):
-    name: str
-    help: Optional[MarkdownContent]
-    type: ObjectStoreTemplateVariableType
-
-
-class ObjectStoreTemplateSecret(StrictModel):
-    name: str
-    help: Optional[MarkdownContent]
+ObjectStoreTemplateVariable = TemplateVariable
+ObjectStoreTemplateSecret = TemplateSecret
 
 
 class ObjectStoreTemplateBase(StrictModel):
@@ -215,20 +205,7 @@ def template_to_configuration(
     user_details: Dict[str, Any],
 ) -> ObjectStoreConfiguration:
     configuration_template = template.configuration
-    template_variables = {
-        "variables": variables,
-        "secrets": secrets,
-        "user": user_details,
-    }
-
-    def expand_template(_, key, value):
-        if isinstance(value, str) and "{{" in value and "}}" in value:
-            # NativeEnvironment preserves Python types
-            template = NativeEnvironment().from_string(value)
-            return key, template.render(**template_variables)
-        return key, value
-
-    raw_config = remap(configuration_template.model_dump(), visit=expand_template)
+    raw_config = expand_raw_config(configuration_template, variables, secrets, user_details)
     return to_configuration_object(raw_config)
 
 

lib/galaxy/util/config_templates.py

@@ -0,0 +1,89 @@
+"""Utilities for defining user configuration bits from admin templates.
+
+This is capturing code shared by file source templates and object store templates.
+"""
+
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Union,
+)
+
+from boltons.iterutils import remap
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+)
+
+try:
+    from jinja2.nativetypes import NativeEnvironment
+except ImportError:
+    NativeEnvironment = None  # type:ignore[assignment, misc, unused-ignore]
+
+TemplateVariableType = Literal["string", "boolean", "integer"]
+TemplateVariableValueType = Union[str, bool, int]
+TemplateExpansion = str
+MarkdownContent = str
+RawTemplateConfig = Dict[str, Any]
+
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class TemplateVariable(StrictModel):
+    name: str
+    help: Optional[MarkdownContent]
+    type: TemplateVariableType
+
+
+class TemplateSecret(StrictModel):
+    name: str
+    help: Optional[MarkdownContent]
+
+
+def expand_raw_config(
+    template_configuration: StrictModel,
+    variables: Dict[str, TemplateVariableValueType],
+    secrets: Dict[str, str],
+    user_details: Dict[str, Any],
+) -> Dict[str, Any]:
+    template_variables = {
+        "variables": variables,
+        "secrets": secrets,
+        "user": user_details,
+    }
+
+    def expand_template(_, key, value):
+        if isinstance(value, str) and "{{" in value and "}}" in value:
+            # NativeEnvironment preserves Python types
+            template = NativeEnvironment().from_string(value)
+            return key, template.render(**template_variables)
+        return key, value
+
+    raw_config = remap(template_configuration.model_dump(), visit=expand_template)
+    return raw_config
+
+
+# cwl-like - convert simple dictionary to list of dictionaries for quickly
+# configuring variables and secrets
+def apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]:
+    templates = []
+    for template in raw_templates:
+        _force_key_to_list(template, "variables")
+        _force_key_to_list(template, "secrets")
+        templates.append(template)
+    return templates
+
+
+def _force_key_to_list(template: RawTemplateConfig, key: str) -> None:
+    value = template.get(key, None)
+    if isinstance(value, dict):
+        value_as_list = []
+        for key_name, key_value in value.items():
+            key_value["name"] = key_name
+            value_as_list.append(key_value)
+        template[key] = value_as_list

packages/util/setup.cfg

@@ -53,6 +53,8 @@ jstree =
 template =
     Cheetah3
     future>=1.0.0
+config_template =
+    Jinja2
 
 [options.packages.find]
 exclude =