WIP: file source templates

lib/galaxy/app.py

@@ -36,6 +36,7 @@
 from galaxy.config_watchers import ConfigWatchers
 from galaxy.datatypes.registry import Registry
 from galaxy.files import ConfiguredFileSources
+from galaxy.files.templates import ConfiguredFileSourceTemplates
 from galaxy.job_metrics import JobMetrics
 from galaxy.jobs.manager import JobManager
 from galaxy.managers.api_keys import ApiKeyManager
@@ -585,6 +586,9 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl
         )
 
         # ConfiguredFileSources
+        vault_configured = is_vault_configured(self.vault)
+        templates = ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=vault_configured)
+        self.file_source_templates = self._register_singleton(ConfiguredFileSourceTemplates, templates)
         self.file_sources = self._register_singleton(
             ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)
         )

lib/galaxy/files/__init__.py

@@ -7,6 +7,7 @@
     List,
     NamedTuple,
     Optional,
+    Protocol,
     Set,
 )
 
@@ -37,20 +38,79 @@ class NoMatchingFileSource(Exception):
     pass
 
 
+class UserDefinedFileSources(Protocol):
+    """Entry-point for Galaxy to inject user-defined object stores.
+
+    Supplied object of this class is used to write out concrete
+    description of file sources when serializing all file sources
+    available to a user.
+    """
+
+    def validate_uri_root(self, uri: str, user_context: "ProvidesUserFileSourcesUserContext") -> None:
+        pass
+
+    def find_best_match(self, url: str) -> Optional[FileSourceScore]:
+        pass
+
+    def user_file_sources_to_dicts(
+        self,
+        for_serialization: bool,
+        user_context: "FileSourceDictifiable",
+        browsable_only: Optional[bool] = False,
+        include_kind: Optional[Set[PluginKind]] = None,
+        exclude_kind: Optional[Set[PluginKind]] = None,
+    ) -> List[FilesSourceProperties]:
+        """Write out user file sources as list of config dictionaries."""
+        # config_dicts: List[FilesSourceProperties] = []
+        # for file_source in self.user_file_sources():
+        #     as_dict = file_source.to_dict(for_serialization=for_serialization, user_context=user_context)
+        #     config_dicts.append(as_dict)
+        # return config_dicts
+
+
+class NullUserDefinedFileSources(UserDefinedFileSources):
+
+    def validate_uri_root(self, uri: str, user_context: "ProvidesUserFileSourcesUserContext") -> None:
+        return None
+
+    def find_best_match(self, url: str) -> Optional[FileSourceScore]:
+        return None
+
+    def user_file_sources_to_dicts(
+        self,
+        for_serialization: bool,
+        user_context: "FileSourceDictifiable",
+        browsable_only: Optional[bool] = False,
+        include_kind: Optional[Set[PluginKind]] = None,
+        exclude_kind: Optional[Set[PluginKind]] = None,
+    ) -> List[FilesSourceProperties]:
+        return []
+
+
+def _ensure_user_defined_file_sources(user_defined_file_sources: Optional[UserDefinedFileSources] = None) -> UserDefinedFileSources:
+    if user_defined_file_sources is not None:
+        return user_defined_file_sources
+    else:
+        return NullUserDefinedFileSources()
+
+
 class ConfiguredFileSources:
     """Load plugins and resolve Galaxy URIs to FileSource objects."""
 
     _file_sources: List[BaseFilesSource]
+    _user_defined_file_sources: UserDefinedFileSources
 
     def __init__(
         self,
         file_sources_config: "ConfiguredFileSourcesConfig",
         conf_file=None,
         conf_dict=None,
         load_stock_plugins=False,
+        user_defined_file_sources: Optional[UserDefinedFileSources] = None,
     ):
         self._file_sources_config = file_sources_config
         self._plugin_classes = self._file_source_plugins_dict()
+        self._user_defined_file_sources = _ensure_user_defined_file_sources(user_defined_file_sources)
         file_sources: List[BaseFilesSource] = []
         if conf_file is not None:
             file_sources = self._load_plugins_from_file(conf_file)
@@ -158,6 +218,7 @@ def validate_uri_root(self, uri: str, user_context: "ProvidesUserFileSourcesUser
                 raise exceptions.ObjectNotFound(
                     "Your FTP directory does not exist, attempting to upload files to it may cause it to be created."
                 )
+        self._user_defined_file_sources.validate_uri_root(uri, user_context)
 
     def looks_like_uri(self, path_or_uri):
         # is this string a URI this object understands how to realize
@@ -187,6 +248,16 @@ def plugins_to_dict(
                 continue
             el = file_source.to_dict(for_serialization=for_serialization, user_context=user_context)
             rval.append(el)
+        if user_context:
+            rval.extend(
+                self._user_defined_file_sources.user_file_sources_to_dicts(
+                    for_serialization,
+                    user_context,
+                    browsable_only=browsable_only,
+                    include_kind=include_kind,
+                    exclude_kind=exclude_kind,
+                )
+            )
         return rval
 
     def to_dict(

lib/galaxy/files/templates/__init__.py

@@ -0,0 +1,17 @@
+from .manager import ConfiguredFileSourceTemplates
+from .models import (
+    FileSourceConfiguration,
+    FileSourceTemplate,
+    FileSourceTemplateSummaries,
+    FileSourceTemplateType,
+    template_to_configuration,
+)
+
+__all__ = (
+    "ConfiguredFileSourceTemplates",
+    "FileSourceConfiguration",
+    "FileSourceTemplate",
+    "FileSourceTemplateSummaries",
+    "FileSourceTemplateType",
+    "template_to_configuration",
+)

lib/galaxy/files/templates/manager.py

@@ -0,0 +1,86 @@
+import os
+from typing import (
+    List,
+    Optional,
+    Protocol,
+)
+
+from yaml import safe_load
+
+from galaxy.util.config_templates import (
+    apply_syntactic_sugar,
+    find_template,
+    find_template_by,
+    InstanceDefinition,
+    RawTemplateConfig,
+    TemplateReference,
+    validate_secrets_and_variables,
+    verify_vault_configured_if_uses_secrets,
+)
+from .models import (
+    FileSourceTemplate,
+    FileSourceTemplateCatalog,
+    FileSourceTemplateSummaries,
+)
+
+SECRETS_NEED_VAULT_MESSAGE = "The file source templates configuration can not be used - a Galaxy vault must be configured for templates that use secrets - please set the vault_config_file configuration option to point at a valid vault configuration."
+
+
+class AppConfigProtocol(Protocol):
+    file_source_templates: Optional[List[RawTemplateConfig]]
+    file_source_templates_config_file: Optional[str]
+
+
+class ConfiguredFileSourceTemplates:
+    catalog: FileSourceTemplateCatalog
+
+    def __init__(self, catalog: FileSourceTemplateCatalog):
+        self.catalog = catalog
+
+    @staticmethod
+    def from_app_config(config: AppConfigProtocol, vault_configured=None) -> "ConfiguredFileSourceTemplates":
+        raw_config = config.file_source_templates
+        if raw_config is None:
+            config_file = config.file_source_templates_config_file
+            if config_file and os.path.exists(config_file):
+                with open(config_file) as f:
+                    raw_config = safe_load(f)
+        if raw_config is None:
+            raw_config = []
+        catalog = raw_config_to_catalog(raw_config)
+        verify_vault_configured_if_uses_secrets(
+            catalog,
+            vault_configured,
+            SECRETS_NEED_VAULT_MESSAGE,
+        )
+        templates = ConfiguredFileSourceTemplates(catalog)
+        return templates
+
+    @property
+    def summaries(self) -> FileSourceTemplateSummaries:
+        templates = self.catalog.root
+        summaries = []
+        for template in templates:
+            template_dict = template.model_dump()
+            configuration = template_dict.pop("configuration")
+            object_store_type = configuration["type"]
+            template_dict["type"] = object_store_type
+            summaries.append(template_dict)
+        return FileSourceTemplateSummaries.model_validate(summaries)
+
+    def find_template(self, instance_reference: TemplateReference) -> FileSourceTemplate:
+        """Find the corresponding template and throw ObjectNotFound if not available."""
+        return find_template(self.catalog.root, instance_reference, "object store")
+
+    def find_template_by(self, template_id: str, template_version: int) -> FileSourceTemplate:
+        return find_template_by(self.catalog.root, template_id, template_version, "object store")
+
+    def validate(self, instance: InstanceDefinition):
+        template = self.find_template(instance)
+        validate_secrets_and_variables(instance, template)
+        # TODO: validate no extra variables
+
+
+def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> FileSourceTemplateCatalog:
+    effective_root = apply_syntactic_sugar(raw_config)
+    return FileSourceTemplateCatalog.model_validate(effective_root)

lib/galaxy/files/templates/models.py

@@ -0,0 +1,126 @@
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Type,
+    Union,
+)
+
+from pydantic import RootModel
+
+from galaxy.util.config_templates import (
+    expand_raw_config,
+    MarkdownContent,
+    StrictModel,
+    Template,
+    TemplateExpansion,
+    TemplateSecret,
+    TemplateVariable,
+    TemplateVariableValueType,
+)
+
+FileSourceTemplateType = Literal["posix", "s3fs"]
+
+
+class PosixFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["posix"]
+    root: Union[str, TemplateExpansion]
+
+
+class PosixFileSourceConfiguration(StrictModel):
+    type: Literal["posix"]
+    root: str
+
+
+class S3FSFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["s3fs"]
+    endpoint_url: Optional[Union[str, TemplateExpansion]] = None
+    anon: Optional[Union[bool, TemplateExpansion]] = False
+    secret: Optional[Union[str, TemplateExpansion]] = None
+    key: Optional[Union[str, TemplateExpansion]] = None
+    bucket: Optional[Union[str, TemplateExpansion]] = None
+
+
+class S3FSFileSourceConfiguration(StrictModel):
+    type: Literal["s3fs"]
+    endpoint_url: Optional[str] = None
+    anon: Optional[bool] = False
+    secret: Optional[str] = None
+    key: Optional[str] = None
+    bucket: Optional[str] = None
+
+
+FileSourceTemplateConfiguration = Union[
+    PosixFileSourceTemplateConfiguration,
+    S3FSFileSourceTemplateConfiguration,
+]
+FileSourceConfiguration = Union[
+    PosixFileSourceConfiguration,
+    S3FSFileSourceConfiguration,
+]
+
+
+class FileSourceTemplateBase(StrictModel):
+    """Version of FileSourceTemplate we can send to the UI/API.
+
+    The configuration key in the child type may have secretes
+    and shouldn't be exposed over the API - at least to non-admins.
+    """
+
+    id: str
+    name: Optional[str]
+    description: Optional[MarkdownContent]
+    # The UI should just show the most recent version but allow
+    # admins to define newer versions with new parameterizations
+    # and keep old versions in template catalog for backward compatibility
+    # for users with existing stores of that template.
+    version: int = 0
+    # Like with multiple versions, allow admins to deprecate a
+    # template by hiding but keep it in the catalog for backward
+    # compatibility for users with existing stores of that template.
+    hidden: bool = False
+    variables: Optional[List[TemplateVariable]] = None
+    secrets: Optional[List[TemplateSecret]] = None
+
+
+class FileSourceTemplateSummary(FileSourceTemplateBase):
+    type: FileSourceTemplateType
+
+
+class FileSourceTemplate(FileSourceTemplateBase):
+    configuration: FileSourceTemplateConfiguration
+
+
+FileSourceTemplateCatalog = RootModel[List[FileSourceTemplate]]
+
+
+class FileSourceTemplateSummaries(RootModel):
+    root: List[FileSourceTemplateSummary]
+
+
+def template_to_configuration(
+    template: FileSourceTemplate,
+    variables: Dict[str, TemplateVariableValueType],
+    secrets: Dict[str, str],
+    user_details: Dict[str, Any],
+) -> FileSourceConfiguration:
+    configuration_template = template.configuration
+    raw_config = expand_raw_config(configuration_template, variables, secrets, user_details)
+    return to_configuration_object(raw_config)
+
+
+TypesToConfigurationClasses: Dict[FileSourceTemplateType, Type[FileSourceConfiguration]] = {
+    "posix": PosixFileSourceConfiguration,
+    "s3fs": S3FSFileSourceConfiguration,
+}
+
+
+def to_configuration_object(configuration_dict: Dict[str, Any]) -> FileSourceConfiguration:
+    if "type" not in configuration_dict:
+        raise KeyError("Configuration objects require a file source 'type' key, none found.")
+    object_store_type = configuration_dict["type"]
+    if object_store_type not in TypesToConfigurationClasses:
+        raise ValueError(f"Unknown file source type found in raw configuration dictionary ({object_store_type}).")
+    return TypesToConfigurationClasses[object_store_type](**configuration_dict)