diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 682b85fc751b..b94b5c387086 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -36,6 +36,7 @@ from galaxy.config_watchers import ConfigWatchers from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources +from galaxy.files.templates import ConfiguredFileSourceTemplates from galaxy.job_metrics import JobMetrics from galaxy.jobs.manager import JobManager from galaxy.managers.api_keys import ApiKeyManager @@ -585,6 +586,9 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl ) # ConfiguredFileSources + vault_configured = is_vault_configured(self.vault) + templates = ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=vault_configured) + self.file_source_templates = self._register_singleton(ConfiguredFileSourceTemplates, templates) self.file_sources = self._register_singleton( ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config) ) diff --git a/lib/galaxy/files/__init__.py b/lib/galaxy/files/__init__.py index 3ddf2a35b1da..fcc4886ecae8 100644 --- a/lib/galaxy/files/__init__.py +++ b/lib/galaxy/files/__init__.py @@ -7,6 +7,7 @@ List, NamedTuple, Optional, + Protocol, Set, ) @@ -37,10 +38,67 @@ class NoMatchingFileSource(Exception): pass +class UserDefinedFileSources(Protocol): + """Entry-point for Galaxy to inject user-defined object stores. + + Supplied object of this class is used to write out concrete + description of file sources when serializing all file sources + available to a user. + """ + + def validate_uri_root(self, uri: str, user_context: "ProvidesUserFileSourcesUserContext") -> None: + pass + + def find_best_match(self, url: str) -> Optional[FileSourceScore]: + pass + + def user_file_sources_to_dicts( + self, + for_serialization: bool, + user_context: "FileSourceDictifiable", + browsable_only: Optional[bool] = False, + include_kind: Optional[Set[PluginKind]] = None, + exclude_kind: Optional[Set[PluginKind]] = None, + ) -> List[FilesSourceProperties]: + """Write out user file sources as list of config dictionaries.""" + # config_dicts: List[FilesSourceProperties] = [] + # for file_source in self.user_file_sources(): + # as_dict = file_source.to_dict(for_serialization=for_serialization, user_context=user_context) + # config_dicts.append(as_dict) + # return config_dicts + + +class NullUserDefinedFileSources(UserDefinedFileSources): + + def validate_uri_root(self, uri: str, user_context: "ProvidesUserFileSourcesUserContext") -> None: + return None + + def find_best_match(self, url: str) -> Optional[FileSourceScore]: + return None + + def user_file_sources_to_dicts( + self, + for_serialization: bool, + user_context: "FileSourceDictifiable", + browsable_only: Optional[bool] = False, + include_kind: Optional[Set[PluginKind]] = None, + exclude_kind: Optional[Set[PluginKind]] = None, + ) -> List[FilesSourceProperties]: + return [] + + +def _ensure_user_defined_file_sources(user_defined_file_sources: Optional[UserDefinedFileSources] = None) -> UserDefinedFileSources: + if user_defined_file_sources is not None: + return user_defined_file_sources + else: + return NullUserDefinedFileSources() + + class ConfiguredFileSources: """Load plugins and resolve Galaxy URIs to FileSource objects.""" _file_sources: List[BaseFilesSource] + _user_defined_file_sources: UserDefinedFileSources def __init__( self, @@ -48,9 +106,11 @@ def __init__( conf_file=None, conf_dict=None, load_stock_plugins=False, + user_defined_file_sources: Optional[UserDefinedFileSources] = None, ): self._file_sources_config = file_sources_config self._plugin_classes = self._file_source_plugins_dict() + self._user_defined_file_sources = _ensure_user_defined_file_sources(user_defined_file_sources) file_sources: List[BaseFilesSource] = [] if conf_file is not None: file_sources = self._load_plugins_from_file(conf_file) @@ -158,6 +218,7 @@ def validate_uri_root(self, uri: str, user_context: "ProvidesUserFileSourcesUser raise exceptions.ObjectNotFound( "Your FTP directory does not exist, attempting to upload files to it may cause it to be created." ) + self._user_defined_file_sources.validate_uri_root(uri, user_context) def looks_like_uri(self, path_or_uri): # is this string a URI this object understands how to realize @@ -187,6 +248,16 @@ def plugins_to_dict( continue el = file_source.to_dict(for_serialization=for_serialization, user_context=user_context) rval.append(el) + if user_context: + rval.extend( + self._user_defined_file_sources.user_file_sources_to_dicts( + for_serialization, + user_context, + browsable_only=browsable_only, + include_kind=include_kind, + exclude_kind=exclude_kind, + ) + ) return rval def to_dict( diff --git a/lib/galaxy/files/templates/__init__.py b/lib/galaxy/files/templates/__init__.py new file mode 100644 index 000000000000..aa8faf85fd30 --- /dev/null +++ b/lib/galaxy/files/templates/__init__.py @@ -0,0 +1,17 @@ +from .manager import ConfiguredFileSourceTemplates +from .models import ( + FileSourceConfiguration, + FileSourceTemplate, + FileSourceTemplateSummaries, + FileSourceTemplateType, + template_to_configuration, +) + +__all__ = ( + "ConfiguredFileSourceTemplates", + "FileSourceConfiguration", + "FileSourceTemplate", + "FileSourceTemplateSummaries", + "FileSourceTemplateType", + "template_to_configuration", +) diff --git a/lib/galaxy/files/templates/manager.py b/lib/galaxy/files/templates/manager.py new file mode 100644 index 000000000000..911fb5a3020e --- /dev/null +++ b/lib/galaxy/files/templates/manager.py @@ -0,0 +1,86 @@ +import os +from typing import ( + List, + Optional, + Protocol, +) + +from yaml import safe_load + +from galaxy.util.config_templates import ( + apply_syntactic_sugar, + find_template, + find_template_by, + InstanceDefinition, + RawTemplateConfig, + TemplateReference, + validate_secrets_and_variables, + verify_vault_configured_if_uses_secrets, +) +from .models import ( + FileSourceTemplate, + FileSourceTemplateCatalog, + FileSourceTemplateSummaries, +) + +SECRETS_NEED_VAULT_MESSAGE = "The file source templates configuration can not be used - a Galaxy vault must be configured for templates that use secrets - please set the vault_config_file configuration option to point at a valid vault configuration." + + +class AppConfigProtocol(Protocol): + file_source_templates: Optional[List[RawTemplateConfig]] + file_source_templates_config_file: Optional[str] + + +class ConfiguredFileSourceTemplates: + catalog: FileSourceTemplateCatalog + + def __init__(self, catalog: FileSourceTemplateCatalog): + self.catalog = catalog + + @staticmethod + def from_app_config(config: AppConfigProtocol, vault_configured=None) -> "ConfiguredFileSourceTemplates": + raw_config = config.file_source_templates + if raw_config is None: + config_file = config.file_source_templates_config_file + if config_file and os.path.exists(config_file): + with open(config_file) as f: + raw_config = safe_load(f) + if raw_config is None: + raw_config = [] + catalog = raw_config_to_catalog(raw_config) + verify_vault_configured_if_uses_secrets( + catalog, + vault_configured, + SECRETS_NEED_VAULT_MESSAGE, + ) + templates = ConfiguredFileSourceTemplates(catalog) + return templates + + @property + def summaries(self) -> FileSourceTemplateSummaries: + templates = self.catalog.root + summaries = [] + for template in templates: + template_dict = template.model_dump() + configuration = template_dict.pop("configuration") + object_store_type = configuration["type"] + template_dict["type"] = object_store_type + summaries.append(template_dict) + return FileSourceTemplateSummaries.model_validate(summaries) + + def find_template(self, instance_reference: TemplateReference) -> FileSourceTemplate: + """Find the corresponding template and throw ObjectNotFound if not available.""" + return find_template(self.catalog.root, instance_reference, "object store") + + def find_template_by(self, template_id: str, template_version: int) -> FileSourceTemplate: + return find_template_by(self.catalog.root, template_id, template_version, "object store") + + def validate(self, instance: InstanceDefinition): + template = self.find_template(instance) + validate_secrets_and_variables(instance, template) + # TODO: validate no extra variables + + +def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> FileSourceTemplateCatalog: + effective_root = apply_syntactic_sugar(raw_config) + return FileSourceTemplateCatalog.model_validate(effective_root) diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py new file mode 100644 index 000000000000..0a9ccf7cdfcb --- /dev/null +++ b/lib/galaxy/files/templates/models.py @@ -0,0 +1,126 @@ +from typing import ( + Any, + Dict, + List, + Literal, + Optional, + Type, + Union, +) + +from pydantic import RootModel + +from galaxy.util.config_templates import ( + expand_raw_config, + MarkdownContent, + StrictModel, + Template, + TemplateExpansion, + TemplateSecret, + TemplateVariable, + TemplateVariableValueType, +) + +FileSourceTemplateType = Literal["posix", "s3fs"] + + +class PosixFileSourceTemplateConfiguration(StrictModel): + type: Literal["posix"] + root: Union[str, TemplateExpansion] + + +class PosixFileSourceConfiguration(StrictModel): + type: Literal["posix"] + root: str + + +class S3FSFileSourceTemplateConfiguration(StrictModel): + type: Literal["s3fs"] + endpoint_url: Optional[Union[str, TemplateExpansion]] = None + anon: Optional[Union[bool, TemplateExpansion]] = False + secret: Optional[Union[str, TemplateExpansion]] = None + key: Optional[Union[str, TemplateExpansion]] = None + bucket: Optional[Union[str, TemplateExpansion]] = None + + +class S3FSFileSourceConfiguration(StrictModel): + type: Literal["s3fs"] + endpoint_url: Optional[str] = None + anon: Optional[bool] = False + secret: Optional[str] = None + key: Optional[str] = None + bucket: Optional[str] = None + + +FileSourceTemplateConfiguration = Union[ + PosixFileSourceTemplateConfiguration, + S3FSFileSourceTemplateConfiguration, +] +FileSourceConfiguration = Union[ + PosixFileSourceConfiguration, + S3FSFileSourceConfiguration, +] + + +class FileSourceTemplateBase(StrictModel): + """Version of FileSourceTemplate we can send to the UI/API. + + The configuration key in the child type may have secretes + and shouldn't be exposed over the API - at least to non-admins. + """ + + id: str + name: Optional[str] + description: Optional[MarkdownContent] + # The UI should just show the most recent version but allow + # admins to define newer versions with new parameterizations + # and keep old versions in template catalog for backward compatibility + # for users with existing stores of that template. + version: int = 0 + # Like with multiple versions, allow admins to deprecate a + # template by hiding but keep it in the catalog for backward + # compatibility for users with existing stores of that template. + hidden: bool = False + variables: Optional[List[TemplateVariable]] = None + secrets: Optional[List[TemplateSecret]] = None + + +class FileSourceTemplateSummary(FileSourceTemplateBase): + type: FileSourceTemplateType + + +class FileSourceTemplate(FileSourceTemplateBase): + configuration: FileSourceTemplateConfiguration + + +FileSourceTemplateCatalog = RootModel[List[FileSourceTemplate]] + + +class FileSourceTemplateSummaries(RootModel): + root: List[FileSourceTemplateSummary] + + +def template_to_configuration( + template: FileSourceTemplate, + variables: Dict[str, TemplateVariableValueType], + secrets: Dict[str, str], + user_details: Dict[str, Any], +) -> FileSourceConfiguration: + configuration_template = template.configuration + raw_config = expand_raw_config(configuration_template, variables, secrets, user_details) + return to_configuration_object(raw_config) + + +TypesToConfigurationClasses: Dict[FileSourceTemplateType, Type[FileSourceConfiguration]] = { + "posix": PosixFileSourceConfiguration, + "s3fs": S3FSFileSourceConfiguration, +} + + +def to_configuration_object(configuration_dict: Dict[str, Any]) -> FileSourceConfiguration: + if "type" not in configuration_dict: + raise KeyError("Configuration objects require a file source 'type' key, none found.") + object_store_type = configuration_dict["type"] + if object_store_type not in TypesToConfigurationClasses: + raise ValueError(f"Unknown file source type found in raw configuration dictionary ({object_store_type}).") + return TypesToConfigurationClasses[object_store_type](**configuration_dict) diff --git a/lib/galaxy/managers/file_source_instances.py b/lib/galaxy/managers/file_source_instances.py new file mode 100644 index 000000000000..78ad62b2fce2 --- /dev/null +++ b/lib/galaxy/managers/file_source_instances.py @@ -0,0 +1,110 @@ +from typing import ( + Dict, + List, + Optional, + Union, +) + +from pydantic import BaseModel + +from galaxy.files.templates import ( + ConfiguredFileSourceTemplates, + FileSourceTemplateSummaries, + FileSourceTemplateType, +) +from galaxy.managers.context import ProvidesUserContext +from galaxy.model import UserFileSource +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.security.vault import Vault +from galaxy.util.config_templates import TemplateVariableValueType +from .object_store_instances import ( + CreateInstancePayload, + ModifyInstancePayload, + UpdateInstancePayload, + UpdateInstanceSecretPayload, + UpgradeInstancePayload, +) + + +class UserFileSourceModel(BaseModel): + id: int + name: str + description: Optional[str] + type: FileSourceTemplateType + template_id: str + template_version: int + variables: Optional[Dict[str, TemplateVariableValueType]] + secrets: List[str] + + +class FileSourceInstancesManager: + _catalog: ConfiguredFileSourceTemplates + _sa_session: galaxy_scoped_session + + def __init__( + self, + catalog: ConfiguredFileSourceTemplates, + sa_session: galaxy_scoped_session, + vault: Vault, + ): + self._catalog = catalog + self._sa_session = sa_session + self._app_vault = vault + + @property + def summaries(self) -> FileSourceTemplateSummaries: + return self._catalog.summaries + + def modify_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: ModifyInstancePayload + ) -> UserFileSourceModel: + if isinstance(payload, UpgradeInstancePayload): + return self._upgrade_instance(trans, id, payload) + elif isinstance(payload, UpdateInstanceSecretPayload): + return self._update_instance_secret(trans, id, payload) + else: + assert isinstance(payload, UpdateInstancePayload) + return self._update_instance(trans, id, payload) + + def _upgrade_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpgradeInstancePayload + ) -> UserFileSourceModel: + pass + + def _update_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpdateInstancePayload + ) -> UserFileSourceModel: + pass + + def _update_instance_secret( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpdateInstanceSecretPayload + ) -> UserFileSourceModel: + pass + + def create_instance(self, trans: ProvidesUserContext, payload: CreateInstancePayload) -> UserFileSourceModel: + pass + + def _to_model(self, trans, persisted_file_source: UserFileSource) -> UserFileSourceModel: + file_source_type = persisted_file_source.template.configuration.type + # These shouldn't be null but sometimes can be? + secrets = persisted_file_source.template_secrets or [] + return UserFileSourceModel( + id=persisted_file_source.id, + type=file_source_type, + template_id=persisted_file_source.template_id, + template_version=persisted_file_source.template_version, + variables=persisted_file_source.template_variables, + secrets=secrets, + name=persisted_file_source.name, + description=persisted_file_source.description, + ) + + +__all__ = ( + "CreateInstancePayload", + "FileSourceInstancesManager", + "ModifyInstancePayload", + "UpdateInstancePayload", + "UpdateInstanceSecretPayload", + "UpgradeInstancePayload", +) diff --git a/lib/galaxy/managers/object_store_instances.py b/lib/galaxy/managers/object_store_instances.py index 619bb59bae8d..40d991a2974d 100644 --- a/lib/galaxy/managers/object_store_instances.py +++ b/lib/galaxy/managers/object_store_instances.py @@ -44,11 +44,11 @@ ObjectStoreTemplateSummaries, ObjectStoreTemplateType, ) -from galaxy.objectstore.templates.models import ObjectStoreTemplateVariableValueType from galaxy.security.vault import ( UserVaultWrapper, Vault, ) +from galaxy.util.config_templates import TemplateVariableValueType log = logging.getLogger(__name__) @@ -58,14 +58,14 @@ class CreateInstancePayload(BaseModel): description: Optional[str] = None template_id: str template_version: int - variables: Dict[str, ObjectStoreTemplateVariableValueType] + variables: Dict[str, TemplateVariableValueType] secrets: Dict[str, str] class UpdateInstancePayload(BaseModel): name: Optional[str] = None description: Optional[str] = None - variables: Optional[Dict[str, ObjectStoreTemplateVariableValueType]] = None + variables: Optional[Dict[str, TemplateVariableValueType]] = None class UpdateInstanceSecretPayload(BaseModel): @@ -75,7 +75,7 @@ class UpdateInstanceSecretPayload(BaseModel): class UpgradeInstancePayload(BaseModel): template_version: int - variables: Dict[str, ObjectStoreTemplateVariableValueType] + variables: Dict[str, TemplateVariableValueType] secrets: Dict[str, str] @@ -84,7 +84,7 @@ class UserConcreteObjectStoreModel(ConcreteObjectStoreModel): type: ObjectStoreTemplateType template_id: str template_version: int - variables: Optional[Dict[str, ObjectStoreTemplateVariableValueType]] + variables: Optional[Dict[str, TemplateVariableValueType]] secrets: List[str] @@ -159,7 +159,7 @@ def _upgrade_instance( continue secret_value = new_secrets[secret_name] - key = user_vault_key(persisted_object_store, secret_name, self._app_config) + key = persisted_object_store.vault_key(secret_name, self._app_config) user_vault.write_secret(key, secret_value) if secret_name not in recorded_secrets: recorded_secrets.append(secret_name) @@ -167,7 +167,7 @@ def _upgrade_instance( secrets_to_delete: List[str] = [] for recorded_secret in recorded_secrets: if recorded_secret not in upgraded_template_secrets: - key = user_vault_key(persisted_object_store, recorded_secret, self._app_config) + key = persisted_object_store.vault_key(recorded_secret, self._app_config) log.info(f"deleting {key} from user vault") user_vault.delete_secret(key) secrets_to_delete.append(recorded_secret) @@ -202,7 +202,7 @@ def _update_instance_secret( ) -> UserConcreteObjectStoreModel: persisted_object_store = self._get(trans, id) user_vault = trans.user_vault - key = user_vault_key(persisted_object_store, payload.secret_name, self._app_config) + key = persisted_object_store.vault_key(payload.secret_name, self._app_config) user_vault.write_secret(key, payload.secret_value) return self._to_model(trans, persisted_object_store) @@ -237,7 +237,7 @@ def create_instance( recorded_secrets = [] try: for secret, value in payload.secrets.items(): - key = user_vault_key(persisted_object_store, secret, self._app_config) + key = persisted_object_store.vault_key(secret, self._app_config) user_vault.write_secret(key, value) recorded_secrets.append(secret) except Exception: @@ -303,17 +303,6 @@ def _to_model(self, trans, persisted_object_store: UserObjectStore) -> UserConcr ) -def user_vault_key(user_object_store: UserObjectStore, secret: str, app_config: UserObjectStoresAppConfig) -> str: - if app_config.user_object_store_index_by == "id": - uos_id = user_object_store.id - else: - uos_id = str(user_object_store.uuid) - assert uos_id - user_vault_id_prefix = f"object_store_config/{uos_id}" - key = f"{user_vault_id_prefix}/{secret}" - return key - - def recover_secrets(user_object_store: UserObjectStore, vault: Vault, app_config: UserObjectStoresAppConfig) -> Dict[str, str]: user: User = user_object_store.user user_vault = UserVaultWrapper(vault, user) @@ -324,7 +313,7 @@ def recover_secrets(user_object_store: UserObjectStore, vault: Vault, app_config # persisted. persisted_secret_names = user_object_store.template_secrets or [] for secret in persisted_secret_names: - vault_key = user_vault_key(user_object_store, secret, app_config) + vault_key = user_object_store.vault_key(secret, app_config) secret_value = user_vault.read_secret(vault_key) # assert secret_value if secret_value is not None: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fcfb27a28026..4d55993a7c32 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -129,6 +129,11 @@ ) import galaxy.exceptions +from galaxy.files.templates import ( + FileSourceConfiguration, + FileSourceTemplate, + template_to_configuration as file_source_template_to_configuration, +) import galaxy.model.metadata import galaxy.model.tags import galaxy.security.passwords @@ -155,7 +160,7 @@ from galaxy.objectstore.templates import ( ObjectStoreConfiguration, ObjectStoreTemplate, - template_to_configuration, + template_to_configuration as object_store_template_to_configuration, ) from galaxy.schema.invocation import ( InvocationCancellationUserRequest, @@ -10884,8 +10889,26 @@ def __init__(self, name=None, value=None): self.value = value -class UserObjectStore(Base, RepresentById): +class UsesTemplatesAppConfig(Protocol): + user_object_store_index_by: Literal["uuid", "id"] + + +class HasConfigSecrets(RepresentById): + secret_config_type: str + + def vault_key(self, secret: str, app_config: UsesTemplatesAppConfig) -> str: + if app_config.user_object_store_index_by == "id": + id_str = str(self.id) + else: + id_str = str(self.uuid) + user_vault_id_prefix = f"{self.secret_config_type}/{id_str}" + key = f"{user_vault_id_prefix}/{secret}" + return key + + +class UserObjectStore(Base, HasConfigSecrets, RepresentById): __tablename__ = "user_object_store" + secret_config_type = "object_store_config" id: Mapped[int] = mapped_column(Integer, primary_key=True) user_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("galaxy_user.id"), index=True) @@ -10930,7 +10953,62 @@ def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConf "id": user.id, } variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {} - return template_to_configuration( + return object_store_template_to_configuration( + self.template, + variables=variables, + secrets=secrets, + user_details=user_details, + ) + + +class UserFileSource(Base, HasConfigSecrets, RepresentById): + __tablename__ = "user_file_source" + secret_config_type = "file_source_config" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[Optional[int]] = mapped_column(ForeignKey("galaxy_user.id"), index=True) + uuid: Mapped[Union[UUID, str]] = mapped_column(UUIDType(), index=True) + create_time: Mapped[datetime] = mapped_column(default=now) + update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, index=True) + # user specified name of the instance they've created + name: Mapped[str] = mapped_column(String(255), index=True) + # user specified description of the instance they've created + description: Mapped[Optional[str]] = mapped_column(Text) + # the template store id + template_id: Mapped[str] = mapped_column(String(255), index=True) + # the template store version (0, 1, ...) + template_version: Mapped[int] = mapped_column(index=True) + # Full template from file_sources_templates.yml catalog. + # For tools we just store references, so here we could easily just use + # the id/version and not record the definition... as the templates change + # over time this choice has some big consequences despite being easy to swap + # implementations. + template_definition: Mapped[Optional[OBJECT_STORE_TEMPLATE_DEFINITION_TYPE]] = mapped_column(JSONType) + # Big JSON blob of the variable name -> value mapping defined for the store's + # variables by the user. + template_variables: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE]] = ( + mapped_column(JSONType) + ) + # Track a list of secrets that were defined for this object store at creation + template_secrets: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_SECRET_NAMES_TYPE]] = ( + mapped_column(JSONType) + ) + + user = relationship("User", back_populates="file_sources") + + @property + def template(self) -> FileSourceTemplate: + return FileSourceTemplate(**self.template_definition or {}) + + def file_source_configuration(self, secrets: Dict[str, Any]) -> FileSourceConfiguration: + user = self.user + user_details = { + "username": user.username, + "email": user.email, + "id": user.id, + } + variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {} + return file_source_template_to_configuration( self.template, variables=variables, secrets=secrets, diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py new file mode 100644 index 000000000000..8e0b57760820 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py @@ -0,0 +1,57 @@ +"""add user defined file sources + +Revision ID: c14a3c93d66b +Revises: c14a3c93d66a +Create Date: 2023-04-01 17:25:37.553039 + +""" + +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + Integer, + String, + Text, +) + +from galaxy.model.custom_types import ( + JSONType, + UUIDType, +) +from galaxy.model.migrations.util import ( + create_table, + drop_table, +) + +# revision identifiers, used by Alembic. +revision = "c14a3c93d66b" +down_revision = "c14a3c93d66a" +branch_labels = None +depends_on = None + + +# database object names used in this revision +table_name = "user_file_source" + + +def upgrade(): + create_table( + table_name, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("galaxy_user.id"), nullable=False, index=True), + Column("uuid", UUIDType, nullable=False, index=True), + Column("name", String(255), index=True), + Column("description", Text, index=True), + Column("create_time", DateTime), + Column("update_time", DateTime), + Column("template_id", String(255), index=True), + Column("template_version", Integer, index=True), + Column("template_definition", JSONType), + Column("template_variables", JSONType), + Column("template_secrets", JSONType), + ) + + +def downgrade(): + drop_table(table_name) diff --git a/lib/galaxy/objectstore/templates/manager.py b/lib/galaxy/objectstore/templates/manager.py index 901931b956c2..418bdf478ddd 100644 --- a/lib/galaxy/objectstore/templates/manager.py +++ b/lib/galaxy/objectstore/templates/manager.py @@ -9,35 +9,29 @@ from typing_extensions import Protocol from yaml import safe_load -from galaxy.exceptions import ( - ObjectNotFound, - RequestParameterMissingException, -) from galaxy.objectstore.badges import serialize_badges +from galaxy.util.config_templates import ( + apply_syntactic_sugar, + find_template, + find_template_by, + InstanceDefinition, + RawTemplateConfig, + TemplateReference, + validate_secrets_and_variables, + verify_vault_configured_if_uses_secrets, +) from .models import ( ObjectStoreTemplate, ObjectStoreTemplateCatalog, ObjectStoreTemplateSummaries, ) -RawTemplateConfig = Dict[str, Any] - class AppConfigProtocol(Protocol): object_store_templates: Optional[List[RawTemplateConfig]] object_store_templates_config_file: Optional[str] -class TemplateReference(Protocol): - template_id: str - template_version: int - - -class InstanceDefinition(TemplateReference, Protocol): - variables: Dict[str, Any] - secrets: Dict[str, str] - - SECRETS_NEED_VAULT_MESSAGE = "The object store templates configuration can not be used - a Galaxy vault must be configured for templates that use secrets - please set the vault_config_file configuration option to point at a valid vault configuration." @@ -57,19 +51,15 @@ def from_app_config(config: AppConfigProtocol, vault_configured=None) -> "Config raw_config = safe_load(f) if raw_config is None: raw_config = [] - templates = ConfiguredObjectStoreTemplates(raw_config_to_catalog(raw_config)) - if vault_configured is False and templates.configuration_uses_secrets: - raise Exception(SECRETS_NEED_VAULT_MESSAGE) + catalog = raw_config_to_catalog(raw_config) + verify_vault_configured_if_uses_secrets( + catalog, + vault_configured, + SECRETS_NEED_VAULT_MESSAGE, + ) + templates = ConfiguredObjectStoreTemplates(catalog) return templates - @property - def configuration_uses_secrets(self) -> bool: - templates = self.catalog.root - for template in templates: - if template.secrets and len(template.secrets) > 0: - return True - return False - @property def summaries(self) -> ObjectStoreTemplateSummaries: templates = self.catalog.root @@ -87,55 +77,17 @@ def summaries(self) -> ObjectStoreTemplateSummaries: def find_template(self, instance_reference: TemplateReference) -> ObjectStoreTemplate: """Find the corresponding template and throw ObjectNotFound if not available.""" - template_id = instance_reference.template_id - template_version = instance_reference.template_version - return self.find_template_by(template_id, template_version) + return find_template(self.catalog.root, instance_reference, "object store") def find_template_by(self, template_id: str, template_version: int) -> ObjectStoreTemplate: - templates = self.catalog.root - - for template in templates: - if template.id == template_id and template.version == template_version: - return template - - raise ObjectNotFound( - f"Could not find a object store template with id {template_id} and version {template_version}" - ) + return find_template_by(self.catalog.root, template_id, template_version, "object store") def validate(self, instance: InstanceDefinition): template = self.find_template(instance) - secrets = instance.secrets - for template_secret in template.secrets or []: - name = template_secret.name - if name not in secrets: - raise RequestParameterMissingException(f"Must define secret '{name}'") - variables = instance.variables - for template_variable in template.variables or []: - name = template_variable.name - if name not in variables: - raise RequestParameterMissingException(f"Must define variable '{name}'") + validate_secrets_and_variables(instance, template) # TODO: validate no extra variables def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> ObjectStoreTemplateCatalog: - effective_root = _apply_syntactic_sugar(raw_config) + effective_root = apply_syntactic_sugar(raw_config) return ObjectStoreTemplateCatalog.model_validate(effective_root) - - -def _apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]: - templates = [] - for template in raw_templates: - _force_key_to_list(template, "variables") - _force_key_to_list(template, "secrets") - templates.append(template) - return templates - - -def _force_key_to_list(template: RawTemplateConfig, key: str) -> None: - value = template.get(key, None) - if isinstance(value, dict): - value_as_list = [] - for key_name, key_value in value.items(): - key_value["name"] = key_name - value_as_list.append(key_value) - template[key] = value_as_list diff --git a/lib/galaxy/objectstore/templates/models.py b/lib/galaxy/objectstore/templates/models.py index 9f225786194a..391a11cbd715 100644 --- a/lib/galaxy/objectstore/templates/models.py +++ b/lib/galaxy/objectstore/templates/models.py @@ -7,28 +7,27 @@ Union, ) -from boltons.iterutils import remap -from jinja2.nativetypes import NativeEnvironment -from pydantic import ( - BaseModel, - ConfigDict, - RootModel, -) +from pydantic import RootModel from typing_extensions import Literal from galaxy.objectstore.badges import ( BadgeDict, StoredBadgeDict, ) +from galaxy.util.config_templates import ( + expand_raw_config, + MarkdownContent, + StrictModel, + Template, + TemplateExpansion, + TemplateSecret, + TemplateVariable, + TemplateVariableType, + TemplateVariableValueType, +) - -class StrictModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - -ObjectStoreTemplateVariableType = Literal["string", "boolean", "integer"] -ObjectStoreTemplateVariableValueType = Union[str, bool, int] -TemplateExpansion = str +ObjectStoreTemplateVariableType = TemplateVariableType +ObjectStoreTemplateVariableValueType = TemplateVariableValueType ObjectStoreTemplateType = Literal["s3", "azure_blob", "disk", "generic_s3"] @@ -155,18 +154,10 @@ class GenericS3ObjectStoreConfiguration(StrictModel): AzureObjectStoreConfiguration, GenericS3ObjectStoreConfiguration, ] -MarkdownContent = str -class ObjectStoreTemplateVariable(StrictModel): - name: str - help: Optional[MarkdownContent] - type: ObjectStoreTemplateVariableType - - -class ObjectStoreTemplateSecret(StrictModel): - name: str - help: Optional[MarkdownContent] +ObjectStoreTemplateVariable = TemplateVariable +ObjectStoreTemplateSecret = TemplateSecret class ObjectStoreTemplateBase(StrictModel): @@ -188,8 +179,8 @@ class ObjectStoreTemplateBase(StrictModel): # template by hiding but keep it in the catalog for backward # compatibility for users with existing stores of that template. hidden: bool = False - variables: Optional[List[ObjectStoreTemplateVariable]] = None - secrets: Optional[List[ObjectStoreTemplateSecret]] = None + variables: Optional[List[TemplateVariable]] = None + secrets: Optional[List[TemplateSecret]] = None class ObjectStoreTemplateSummary(ObjectStoreTemplateBase): @@ -215,20 +206,7 @@ def template_to_configuration( user_details: Dict[str, Any], ) -> ObjectStoreConfiguration: configuration_template = template.configuration - template_variables = { - "variables": variables, - "secrets": secrets, - "user": user_details, - } - - def expand_template(_, key, value): - if isinstance(value, str) and "{{" in value and "}}" in value: - # NativeEnvironment preserves Python types - template = NativeEnvironment().from_string(value) - return key, template.render(**template_variables) - return key, value - - raw_config = remap(configuration_template.model_dump(), visit=expand_template) + raw_config = expand_raw_config(configuration_template, variables, secrets, user_details) return to_configuration_object(raw_config) diff --git a/lib/galaxy/util/config_templates.py b/lib/galaxy/util/config_templates.py new file mode 100644 index 000000000000..bc1ddc1261a7 --- /dev/null +++ b/lib/galaxy/util/config_templates.py @@ -0,0 +1,157 @@ +"""Utilities for defining user configuration bits from admin templates. + +This is capturing code shared by file source templates and object store templates. +""" + +from typing import ( + Any, + Dict, + Generic, + List, + Literal, + Optional, + Protocol, + TypeVar, + Union, +) + +from boltons.iterutils import remap +from pydantic import ( + BaseModel, + ConfigDict, +) + +try: + from jinja2.nativetypes import NativeEnvironment +except ImportError: + NativeEnvironment = None # type:ignore[assignment, misc, unused-ignore] + +from galaxy.exceptions import ( + ObjectNotFound, + RequestParameterMissingException, +) + +TemplateVariableType = Literal["string", "boolean", "integer"] +TemplateVariableValueType = Union[str, bool, int] +TemplateExpansion = str +MarkdownContent = str +RawTemplateConfig = Dict[str, Any] + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class TemplateVariable(StrictModel): + name: str + help: Optional[MarkdownContent] + type: TemplateVariableType + + +class TemplateSecret(StrictModel): + name: str + help: Optional[MarkdownContent] + + +def expand_raw_config( + template_configuration: StrictModel, + variables: Dict[str, TemplateVariableValueType], + secrets: Dict[str, str], + user_details: Dict[str, Any], +) -> Dict[str, Any]: + template_variables = { + "variables": variables, + "secrets": secrets, + "user": user_details, + } + + def expand_template(_, key, value): + if isinstance(value, str) and "{{" in value and "}}" in value: + # NativeEnvironment preserves Python types + template = NativeEnvironment().from_string(value) + return key, template.render(**template_variables) + return key, value + + raw_config = remap(template_configuration.model_dump(), visit=expand_template) + return raw_config + + +def verify_vault_configured_if_uses_secrets(catalog, vault_configured: bool, exception_message: str) -> None: + if _catalog_uses_secrets(catalog) and not vault_configured: + raise Exception(exception_message) + + +def _catalog_uses_secrets(catalog) -> bool: + templates = catalog.root + for template in templates: + if template.secrets and len(template.secrets) > 0: + return True + return False + + +# cwl-like - convert simple dictionary to list of dictionaries for quickly +# configuring variables and secrets +def apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]: + templates = [] + for template in raw_templates: + _force_key_to_list(template, "variables") + _force_key_to_list(template, "secrets") + templates.append(template) + return templates + + +def _force_key_to_list(template: RawTemplateConfig, key: str) -> None: + value = template.get(key, None) + if isinstance(value, dict): + value_as_list = [] + for key_name, key_value in value.items(): + key_value["name"] = key_name + value_as_list.append(key_value) + template[key] = value_as_list + + +class TemplateReference(Protocol): + template_id: str + template_version: int + + +class InstanceDefinition(TemplateReference, Protocol): + variables: Dict[str, Any] + secrets: Dict[str, str] + + +class Template(Protocol): + id: str + version: int + variables: List[TemplateVariable] + secrets: List[TemplateSecret] + + +T = TypeVar("T", bound=Template) + + +def find_template(templates: List[T], instance_reference: TemplateReference, what: str) -> T: + template_id = instance_reference.template_id + template_version = instance_reference.template_version + return find_template_by(templates, template_id, template_version, what) + + +def find_template_by(templates: List[T], template_id: str, template_version: int, what: str) -> T: + for template in templates: + if template.id == template_id and template.version == template_version: + return template + + raise ObjectNotFound(f"Could not find a {what} template with id {template_id} and version {template_version}") + + +def validate_secrets_and_variables(instance: InstanceDefinition, template: Template) -> None: + secrets = instance.secrets + for template_secret in template.secrets or []: + name = template_secret.name + if name not in secrets: + raise RequestParameterMissingException(f"Must define secret '{name}'") + variables = instance.variables + for template_variable in template.variables or []: + name = template_variable.name + if name not in variables: + raise RequestParameterMissingException(f"Must define variable '{name}'") diff --git a/lib/galaxy/webapps/galaxy/api/file_sources.py b/lib/galaxy/webapps/galaxy/api/file_sources.py new file mode 100644 index 000000000000..f7ecec2978f3 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/file_sources.py @@ -0,0 +1,98 @@ +import logging + +from typing import List + +from fastapi import ( + Body, + Path, + Query, +) + +from . import ( + depends, + DependsOnTrans, + Router, +) + +from galaxy.files.templates import FileSourceTemplateSummaries +from galaxy.managers.file_source_instances import ( + CreateInstancePayload, + ModifyInstancePayload, + FileSourceInstancesManager, + UserFileSourceModel, +) +from galaxy.managers.context import ProvidesUserContext + +log = logging.getLogger(__name__) + +router = Router(tags=["file_sources"]) + + +UserFileSourceIdPathParam: str = Path( + ..., title="User File Source ID", description="The index for a persisted UserFileSourceStore object." +) + + +@router.cbv +class FastAPIFileSources: + file_source_instances_manager: FileSourceInstancesManager = depends(FileSourceInstancesManager) + + @router.get( + "/api/file_source_templates", + summary="Get a list of file source templates available to build user defined file sources from", + response_description="A list of the configured file source templates.", + operation_id="file_sources__templates_index", + ) + def index_templates( + self, + trans: ProvidesUserContext = DependsOnTrans, + ) -> FileSourceTemplateSummaries: + return self.file_source_instances_manager.summaries + + @router.post( + "/api/file_source_instances", + summary="Create a user-bound object store.", + operation_id="file_sources__create_instance", + ) + def create( + self, + trans: ProvidesUserContext = DependsOnTrans, + payload: CreateInstancePayload = Body(...), + ) -> UserFileSourceModel: + return self.file_source_instances_manager.create_instance(trans, payload) + + @router.get( + "/api/file_source_instances", + summary="Get a list of persisted file source instances defined by the requesting user.", + operation_id="file_sources__instances_index", + ) + def instance_index( + self, + trans: ProvidesUserContext = DependsOnTrans, + ) -> List[UserFileSourceModel]: + return self.file_source_instances_manager.index(trans) + + @router.get( + "/api/file_source_instances/{user_file_source_id}", + summary="Get a list of persisted file source instances defined by the requesting user.", + operation_id="file_sources__instances_get", + ) + def instances_show( + self, + trans: ProvidesUserContext = DependsOnTrans, + user_file_source_id: str = UserFileSourceIdPathParam, + ) -> UserFileSourceModel: + return self.file_source_instances_manager.show(trans, user_file_source_id) + + @router.put( + "/api/file_source_instances/{user_file_source_id}", + summary="Update or upgrade user file source instance.", + operation_id="file_sources__instances_update", + ) + def update_instance( + self, + trans: ProvidesUserContext = DependsOnTrans, + user_file_source_id: str = UserFileSourceIdPathParam, + payload: ModifyInstancePayload = Body(...), + ) -> UserFileSourceModel: + return self.file_source_instances_manager.modify_instance(trans, user_file_source_id, payload) diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg index 70add5d8311c..a3a9d4619367 100644 --- a/packages/util/setup.cfg +++ b/packages/util/setup.cfg @@ -53,6 +53,8 @@ jstree = template = Cheetah3 future>=1.0.0 +config_template = + Jinja2 [options.packages.find] exclude = diff --git a/test/unit/files/test_template_manager.py b/test/unit/files/test_template_manager.py new file mode 100644 index 000000000000..5c89405b0c8b --- /dev/null +++ b/test/unit/files/test_template_manager.py @@ -0,0 +1,58 @@ +from galaxy.files.templates import ConfiguredFileSourceTemplates +from .test_template_models import ( + LIBRARY_AWS, + LIBRARY_HOME_DIRECTORY, +) + + +class MockConfig: + def __init__(self, config_path): + self.file_source_templates = None + self.file_source_templates_config_file = config_path + + +def test_manager(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_HOME_DIRECTORY, "utf-8") + config = MockConfig(config_path) + templates = ConfiguredFileSourceTemplates.from_app_config(config) + summaries = templates.summaries + assert summaries + assert len(summaries.root) == 1 + + +def test_manager_throws_exception_if_vault_is_required_but_configured(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_AWS, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=False) + except Exception as e: + exc = e + assert exc, "catalog creation should result in an exception" + assert "vault must be configured" in str(exc) + + +def test_manager_with_secrets_is_fine_if_vault_is_required_and_configured(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_AWS, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=True) + except Exception as e: + exc = e + assert exc is None + + +def test_manager_does_not_throw_exception_if_vault_is_not_required(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_HOME_DIRECTORY, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=False) + except Exception as e: + exc = e + assert exc is None diff --git a/test/unit/files/test_template_models.py b/test/unit/files/test_template_models.py new file mode 100644 index 000000000000..eccfaf073e42 --- /dev/null +++ b/test/unit/files/test_template_models.py @@ -0,0 +1,91 @@ +from yaml import safe_load + +from galaxy.files.templates.manager import raw_config_to_catalog +from galaxy.files.templates.models import ( + FileSourceTemplate, + FileSourceTemplateCatalog, + PosixFileSourceConfiguration, + S3FSFileSourceConfiguration, + template_to_configuration, +) + +LIBRARY_AWS = """ +- id: aws_bucket + name: Amazon Bucket + description: An Amazon S3 Bucket + variables: + bucket_name: + type: string + help: Name of bucket to use when connecting to AWS resources. + secrets: + access_key: + help: AWS access key to use when connecting to AWS resources. + secret_key: + help: AWS secret key to use when connecting to AWS resources. + configuration: + type: s3fs + key: '{{ secrets.access_key}}' + secret: '{{ secrets.secret_key}}' + bucket: '{{ variables.bucket_name}}' +""" + + +def test_aws_s3_config(): + template_library = _parse_template_library(LIBRARY_AWS) + s3_template = _assert_has_one_template(template_library) + assert s3_template.description == "An Amazon S3 Bucket" + configuration_obj = template_to_configuration( + s3_template, + {"bucket_name": "sec3"}, + {"access_key": "sec1", "secret_key": "sec2"}, + user_details={}, + ) + + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, S3FSFileSourceConfiguration) + configuration = configuration_obj.model_dump() + + assert configuration["type"] == "s3fs" + assert configuration["key"] == "sec1" + assert configuration["secret"] == "sec2" + assert configuration["bucket"] == "sec3" + + +LIBRARY_HOME_DIRECTORY = """ +- id: home_directory + name: Home Directory + description: Your Home Directory on this System + configuration: + type: posix + root: "/home/{{ user.username}}/" +""" + + +def test_a_posix_template(): + template_library = _parse_template_library(LIBRARY_HOME_DIRECTORY) + posix_template = _assert_has_one_template(template_library) + assert posix_template.description == "Your Home Directory on this System" + configuration_obj = template_to_configuration( + posix_template, + {}, + {}, + user_details={"username": "foobar"}, + ) + + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, PosixFileSourceConfiguration) + configuration = configuration_obj.model_dump() + + assert configuration["type"] == "posix" + assert configuration["root"] == "/home/foobar/" + + +def _assert_has_one_template(catalog: FileSourceTemplateCatalog) -> FileSourceTemplate: + assert len(catalog.root) == 1 + template = catalog.root[0] + return template + + +def _parse_template_library(contents: str) -> FileSourceTemplateCatalog: + raw_contents = safe_load(contents) + return raw_config_to_catalog(raw_contents)