diff --git a/client/src/components/ObjectStore/Templates/CreateForm.vue b/client/src/components/ObjectStore/Templates/CreateForm.vue new file mode 100644 index 000000000000..957ddc274f5a --- /dev/null +++ b/client/src/components/ObjectStore/Templates/CreateForm.vue @@ -0,0 +1,54 @@ + + diff --git a/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue new file mode 100644 index 000000000000..dbbdb36211af --- /dev/null +++ b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue @@ -0,0 +1,65 @@ + + diff --git a/client/src/components/ObjectStore/Templates/SelectTemplate.vue b/client/src/components/ObjectStore/Templates/SelectTemplate.vue new file mode 100644 index 000000000000..b33966794e75 --- /dev/null +++ b/client/src/components/ObjectStore/Templates/SelectTemplate.vue @@ -0,0 +1,52 @@ + + + diff --git a/client/src/components/ObjectStore/services.ts b/client/src/components/ObjectStore/services.ts index 23c0f4e3b3fc..92f2ed8bfab7 100644 --- a/client/src/components/ObjectStore/services.ts +++ b/client/src/components/ObjectStore/services.ts @@ -1,8 +1,18 @@ +import type { components } from "@/schema"; import { fetcher } from "@/schema/fetcher"; const getObjectStores = fetcher.path("/api/object_stores").method("get").create(); +const getObjectStoreTemplates = fetcher.path("/api/object_store_templates").method("get").create(); + +export type ObjectStoreTemplateSummaries = components["schemas"]["ObjectStoreTemplateSummaries"]; +export type ObjectStoreTemplateSummary = components["schemas"]["ObjectStoreTemplateSummary"]; export async function getSelectableObjectStores() { const { data } = await getObjectStores({ selectable: true }); return data; } + +export async function getTemplates(): Promise { + const { data } = await getObjectStoreTemplates({}); + return data; +} diff --git a/client/src/entry/analysis/router.js b/client/src/entry/analysis/router.js index c353d4fb67c3..23e06baa42ef 100644 --- a/client/src/entry/analysis/router.js +++ b/client/src/entry/analysis/router.js @@ -61,6 +61,7 @@ import VisualizationPublished from "components/Visualizations/VisualizationPubli import WorkflowImport from "components/Workflow/WorkflowImport"; import WorkflowList from "components/Workflow/WorkflowList"; import WorkflowPublished from "components/Workflow/WorkflowPublished"; +import CreateUserObjectStore from "components/ObjectStore/Templates/CreateUserObjectStore"; import { APIKey } from "components/User/APIKey"; import { CloudAuth } from "components/User/CloudAuth"; import { ExternalIdentities } from "components/User/ExternalIdentities"; @@ -278,6 +279,10 @@ export function getRouter(Galaxy) { termsUrl: Galaxy.config.terms_url, }, }, + { + path: "object_store/create", + component: CreateUserObjectStore, + }, { path: "pages/create", component: FormGeneric, diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index edea14e22c3c..76aaba8b8c4c 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -43,6 +43,7 @@ from galaxy.managers.jobs import JobSearch from galaxy.managers.libraries import LibraryManager from galaxy.managers.library_datasets import LibraryDatasetsManager +from galaxy.managers.object_store_instances import UserObjectStoreResolverImpl from galaxy.managers.roles import RoleManager from galaxy.managers.session import GalaxySessionManager from galaxy.managers.tasks import ( @@ -80,7 +81,10 @@ from galaxy.objectstore import ( BaseObjectStore, build_object_store_from_config, + UserObjectStoreResolver, + UserObjectStoresAppConfig, ) +from galaxy.objectstore.templates import ConfiguredObjectStoreTemplates from galaxy.queue_worker import ( GalaxyQueueWorker, send_local_control_task, @@ -236,8 +240,6 @@ def __init__(self, fsmon=False, **kwargs) -> None: # Read config file and check for errors self.config = self._register_singleton(config.GalaxyAppConfiguration, config.GalaxyAppConfiguration(**kwargs)) self.config.check() - self._configure_object_store(fsmon=True) - self._register_singleton(BaseObjectStore, self.object_store) config_file = kwargs.get("global_conf", {}).get("__file__", None) if config_file: log.debug('Using "galaxy.ini" config file: %s', config_file) @@ -249,6 +251,10 @@ def __init__(self, fsmon=False, **kwargs) -> None: self._register_singleton(GalaxyModelMapping, self.model) self._register_singleton(galaxy_scoped_session, self.model.context) self._register_singleton(install_model_scoped_session, self.install_model.context) + self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) # type: ignore[type-abstract] + self._configure_object_store(fsmon=True) + self._register_singleton(BaseObjectStore, self.object_store) + galaxy.model.Dataset.object_store = self.object_store def configure_fluent_log(self): if self.config.fluent_log: @@ -389,6 +395,21 @@ def _configure_datatypes_registry(self, use_display_applications=True, use_conve ) def _configure_object_store(self, **kwds): + app_config = UserObjectStoresAppConfig( + jobs_directory=self.config.jobs_directory, + new_file_path=self.config.new_file_path, + umask=self.config.umask, + ) + self._register_singleton(UserObjectStoresAppConfig, app_config) + user_object_store_resolver = self._register_abstract_singleton( + UserObjectStoreResolver, UserObjectStoreResolverImpl # type: ignore[type-abstract] + ) # Ignored because of https://github.com/python/mypy/issues/4717 + + self.object_store_templates = self._register_singleton( + ConfiguredObjectStoreTemplates, ConfiguredObjectStoreTemplates.from_app_config(self.config) + ) + # kwds["object_store_templates"] = self.object_store_templates + kwds["user_object_store_resolver"] = user_object_store_resolver self.object_store = build_object_store_from_config(self.config, **kwds) def _configure_security(self): @@ -430,7 +451,7 @@ def _configure_models(self, check_migrate_databases=False, config_file=None): self.model = mapping.configure_model_mapping( self.config.file_path, - self.object_store, + None, # setting object store later now... self.config.use_pbkdf2, engine, combined_install_database, @@ -549,7 +570,6 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config) ) - self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) # type: ignore[type-abstract] # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.model.security.HostAgent( diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index ccc0955444b7..2f0303607064 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -551,6 +551,21 @@ mapping: desc: | FileSource plugins described embedded into Galaxy's config. + object_store_templates_config_file: + type: str + default: object_store_templates.yml + path_resolves_to: config_dir + required: false + desc: | + Configured Object Store templates configuration file. + + object_store_templates: + type: seq + sequence: + - type: any + desc: | + Configured Object Store templates embedded into Galaxy's config. + enable_mulled_containers: type: bool default: true diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 7a4cdb41db5d..8d67bb52c149 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1681,6 +1681,7 @@ def split_object_stores(output_name): require_shareable = job.requires_shareable_storage(self.app.security_agent) if not split_object_stores: + log.info(f"\n\n\n\nNot splitting stores and have object_store_id {object_store_id}\n\n") object_store_populator = ObjectStorePopulator(self.app, user) if object_store_id: diff --git a/lib/galaxy/managers/object_store_instances.py b/lib/galaxy/managers/object_store_instances.py new file mode 100644 index 000000000000..72f24809994a --- /dev/null +++ b/lib/galaxy/managers/object_store_instances.py @@ -0,0 +1,170 @@ +from pydantic import BaseModel +from typing import ( + Any, + Dict, + Optional, +) + +from galaxy.exceptions import ItemOwnershipException +from galaxy.managers.context import ProvidesUserContext +from galaxy.model import ( + User, + UserObjectStore, +) +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.objectstore import ( + ConcreteObjectStore, + ConcreteObjectStoreModel, + QuotaModel, + UserObjectStoreResolver, + UserObjectStoresAppConfig, + concrete_object_store, +) +from galaxy.objectstore.templates import ( + ConfiguredObjectStoreTemplates, + ObjectStoreTemplateSummaries, +) +from galaxy.security.vault import ( + UserVaultWrapper, + Vault, +) + + +class CreateInstancePayload(BaseModel): + name: str + description: Optional[str] + template_id: str + template_version: int + variables: Dict[str, Any] + secrets: Dict[str, str] + + +class UserObjectStoreReference(BaseModel): + id: int + + +class UpdateInstancePayload(UserObjectStoreReference): + name: str + description: Optional[str] + variables: Dict[str, Any] + + +class UpgradeInstancePayload(UserObjectStoreReference): + template_id: str + template_version: int + variables: Dict[str, Any] + secrets: Dict[str, str] + + +class ObjectStoreInstancesManager: + _catalog: ConfiguredObjectStoreTemplates + _sa_session: galaxy_scoped_session + + def __init__( + self, + catalog: ConfiguredObjectStoreTemplates, + sa_session: galaxy_scoped_session, + vault: Vault, + ): + self._catalog = catalog + self._sa_session = sa_session + self._app_vault = vault + + @property + def summaries(self) -> ObjectStoreTemplateSummaries: + return self._catalog.summaries + + def upgrade_instance(self, trans: ProvidesUserContext, payload: UpgradeInstancePayload) -> ConcreteObjectStoreModel: + persisted_object_store = self._get(trans, payload) + # TODO: + return self._to_model(trans, persisted_object_store) + + def update_instance(self, trans: ProvidesUserContext, payload: UpdateInstancePayload) -> ConcreteObjectStoreModel: + persisted_object_store = self._get(trans, payload) + # TODO: + return self._to_model(trans, persisted_object_store) + + def create_instance(self, trans: ProvidesUserContext, payload: CreateInstancePayload) -> ConcreteObjectStoreModel: + catalog = self._catalog + catalog.validate(payload) + template = catalog.find_template(payload) + assert template + user_vault = trans.user_vault + persisted_object_store = UserObjectStore() + persisted_object_store.object_store_template_definition = template.dict() + persisted_object_store.object_store_template_id = template.id + persisted_object_store.object_store_template_version = template.version + persisted_object_store.object_store_template_variables = payload.variables + persisted_object_store.name = payload.name + persisted_object_store.description = payload.description + self._sa_session.add(persisted_object_store) + self._sa_session.flush([persisted_object_store]) + recorded_secrets = [] + try: + for secret, value in payload.secrets.items(): + key = user_vault_key(persisted_object_store, secret) + user_vault.write_secret(key, value) + recorded_secrets.append(secret) + except Exception: + self._sa_session.delete(persisted_object_store) + raise + persisted_object_store.object_store_template_secrets = recorded_secrets + persisted_object_store.user = trans.user + self._sa_session.add(persisted_object_store) + self._sa_session.flush([persisted_object_store]) + return self._to_model(trans, persisted_object_store) + + def _get(self, trans: ProvidesUserContext, object_reference: UserObjectStoreReference) -> UserObjectStore: + user_object_store = self._sa_session.query(UserObjectStore).get(object_reference.id) + if user_object_store.user != trans.user: + raise ItemOwnershipException() + return user_object_store + + def _to_model(self, trans, persisted_object_store: UserObjectStore) -> ConcreteObjectStoreModel: + quota = QuotaModel(source=None, enabled=False) + return ConcreteObjectStoreModel( + name=persisted_object_store.name, + description=persisted_object_store.description, + object_store_id=f"user_objects://{persisted_object_store.id}", + private=True, + quota=quota, + badges=[], + ) + + +def user_vault_key(user_object_store: UserObjectStore, secret: str) -> str: + uos_id = user_object_store.id + assert uos_id + user_vault_id_prefix = f"object_store_config/{uos_id}" + key = f"{user_vault_id_prefix}/{secret}" + return key + + +def recover_secrets(user_object_store: UserObjectStore, vault: Vault) -> Dict[str, str]: + user: User = user_object_store.user + user_vault = UserVaultWrapper(vault, user) + secrets: Dict[str, str] = {} + # now we could recover the list of secrets to fetch from... + # ones recorded as written in the persisted object, the ones + # expected in the catalog, or the ones expected in the definition + # persisted. + for secret in user_object_store.object_store_template_secrets: + vault_key = user_vault_key(user_object_store, secret) + secret_value = user_vault.read_secret(vault_key) + assert secret_value + secrets[secret] = secret_value + return secrets + + +class UserObjectStoreResolverImpl(UserObjectStoreResolver): + def __init__(self, sa_session: galaxy_scoped_session, vault: Vault, app_config: UserObjectStoresAppConfig): + self._sa_session = sa_session + self._vault = vault + self._app_config = app_config + + def resolve_object_store_uri(self, uri: str) -> "ConcreteObjectStore": + user_object_store_id = uri.split("://", 1)[1] + user_object_store: UserObjectStore = self._sa_session.query(UserObjectStore).get(user_object_store_id) + secrets = recover_secrets(user_object_store, self._vault) + object_store_configuration = user_object_store.object_store_configuration(secrets=secrets) + return concrete_object_store(object_store_configuration, self._app_config) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 2c58dd7e1052..00007559851d 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -701,9 +701,9 @@ def add_deserializers(self): } self.deserializers.update(history_deserializers) - def deserialize_preferred_object_store_id(self, item: Any, key: Any, val: Any, **context): + def deserialize_preferred_object_store_id(self, item: Any, key: Any, val: Any, trans=None, **context): preferred_object_store_id = val - validation_error = validate_preferred_object_store_id(self.app.object_store, preferred_object_store_id) + validation_error = validate_preferred_object_store_id(trans, self.app.object_store, preferred_object_store_id) if validation_error: raise base.ModelDeserializingError(validation_error) return self.default_deserializer(item, key, preferred_object_store_id, **context) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 26e7773e137a..030edcae8919 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -128,6 +128,11 @@ from galaxy.model.orm.util import add_object_to_object_session from galaxy.model.view import HistoryDatasetCollectionJobStateSummary from galaxy.objectstore import ObjectStore +from galaxy.objectstore.templates import ( + ObjectStoreTemplate, + ObjectStoreConfiguration, + template_to_configuration, +) from galaxy.security import get_permitted_actions from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.validate_user_input import validate_password_str @@ -696,6 +701,7 @@ class User(Base, Dictifiable, RepresentById): galaxy_sessions = relationship( "GalaxySession", back_populates="user", order_by=lambda: desc(GalaxySession.update_time) # type: ignore[has-type] ) + object_stores = relationship("UserObjectStore", back_populates="user") quotas = relationship("UserQuotaAssociation", back_populates="user") quota_source_usages = relationship("UserQuotaSourceUsage", back_populates="user") social_auth = relationship("UserAuthnzToken", back_populates="user") @@ -10063,6 +10069,44 @@ def __init__(self, name=None, value=None): self.value = value +class UserObjectStore(Base, RepresentById): + __tablename__ = "user_object_store" + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now, index=True) + name = Column(String(255), index=True) + description = Column(Text) + object_store_template_id = Column(String(255), index=True) + object_store_template_version = Column(Integer, index=True) + # Maybe just use the id/version and don't record the definition + object_store_template_definition = Column(JSONType) + # Maybe convert these maps to association tables encoding maps... + object_store_template_variables = Column(JSONType) + object_store_template_secrets = Column(JSONType) + + user = relationship("User", back_populates="object_stores") + + @property + def template(self) -> ObjectStoreTemplate: + return ObjectStoreTemplate(**self.object_store_template_definition) + + def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConfiguration: + user = self.user + user_details = { + "username": user.username, + "email": user.email, + "id": user.id, + } + return template_to_configuration( + self.template, + variables=self.object_store_template_variables, + secrets=secrets, + user_details=user_details, + ) + + class UserAction(Base, RepresentById): __tablename__ = "user_action" diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py new file mode 100644 index 000000000000..c244fa4fd759 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py @@ -0,0 +1,52 @@ +"""add user defined object stores + +Revision ID: c14a3c93d66a +Revises: 460d0ecd1dd8 +Create Date: 2023-04-01 17:25:37.553039 + +""" +from alembic import op +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + Integer, + String, + Text, +) + +from galaxy.model.custom_types import JSONType + +# revision identifiers, used by Alembic. +revision = "c14a3c93d66a" +down_revision = "460d0ecd1dd8" +branch_labels = None +depends_on = None + + +# database object names used in this revision +table_name = "user_object_store" + + +def upgrade(): + op.create_table( + table_name, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("galaxy_user.id"), nullable=False, index=True), + Column("name", String(255), index=True), + Column("description", Text, index=True), + Column("create_time", DateTime), + Column("update_time", DateTime), + Column("template_id", String(255), index=True), + Column("template_version", Integer, index=True), + # Maybe just use the id/version and don't record the definition + Column("template_definition", JSONType), + # Maybe convert these maps to association tables encoding maps... + Column("template_variables", JSONType), + # following needed? + Column("template_secrets", JSONType), + ) + + +def downgrade(): + op.drop_table(table_name) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 337123cc9dad..6771741a2fdf 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -27,6 +27,7 @@ from pydantic import BaseModel from typing_extensions import ( Literal, + Protocol, TypedDict, ) @@ -47,6 +48,7 @@ safe_relpath, ) from galaxy.util.sleeper import Sleeper +from .templates import ObjectStoreConfiguration NO_SESSION_ERROR_MESSAGE = ( "Attempted to 'create' object store entity in configuration with no database session present." @@ -96,6 +98,11 @@ class BadgeDict(TypedDict): source: BadgeSourceT +class UserObjectStoreResolver(Protocol): + def resolve_object_store_uri(self, uri: str) -> "ConcreteObjectStore": + pass + + class ObjectStore(metaclass=abc.ABCMeta): """ObjectStore interface. @@ -317,6 +324,11 @@ def object_store_allows_id_selection(self) -> bool: """Return True if this object store respects object_store_id and allow selection of this.""" return False + def validate_selected_object_store_id(self, user, object_store_id: Optional[str]) -> Optional[str]: + if object_store_id and not self.object_store_allows_id_selection(): + return "The current configuration doesn't allow selecting preferred object stores." + return None + def object_store_ids_allowing_selection(self) -> List[str]: """Return a non-emtpy list of allowed selectable object store IDs during creation.""" return [] @@ -1064,7 +1076,9 @@ class DistributedObjectStore(NestedObjectStore): store_type = "distributed" - def __init__(self, config, config_dict, fsmon=False): + def __init__( + self, config, config_dict, fsmon=False, user_object_store_resolver: Optional[UserObjectStoreResolver] = None + ): """ :type config: object :param config: An object, most likely populated from @@ -1110,8 +1124,9 @@ def __init__(self, config, config_dict, fsmon=False): self.weighted_backend_ids.append(backened_id) self.original_weighted_backend_ids = self.weighted_backend_ids + self.user_object_store_resolver = user_object_store_resolver self.user_selection_allowed = user_selection_allowed - self.allow_user_selection = bool(user_selection_allowed) + self.allow_user_selection = bool(user_selection_allowed) or (user_object_store_resolver is not None) self.sleeper = None if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]): self.sleeper = Sleeper() @@ -1156,7 +1171,9 @@ def parse_xml(clazz, config_xml, legacy=False): return config_dict @classmethod - def from_xml(clazz, config, config_xml, fsmon=False): + def from_xml( + clazz, config, config_xml, fsmon=False, user_object_store_resolver: Optional[UserObjectStoreResolver] = None + ): legacy = False if config_xml is None: distributed_config = config.distributed_object_store_config_file @@ -1173,7 +1190,7 @@ def from_xml(clazz, config, config_xml, fsmon=False): log.debug("Loading backends for distributed object store from %s", config_xml.get("id")) config_dict = clazz.parse_xml(config_xml, legacy=legacy) - return clazz(config, config_dict, fsmon=fsmon) + return clazz(config, config_dict, fsmon=fsmon, user_object_store_resolver=user_object_store_resolver) def to_dict(self) -> Dict[str, Any]: as_dict = super().to_dict() @@ -1210,7 +1227,7 @@ def _create(self, obj, **kwargs): """The only method in which obj.object_store_id may be None.""" object_store_id = obj.object_store_id if object_store_id is None or not self._exists(obj, **kwargs): - if object_store_id is None or object_store_id not in self.backends: + if object_store_id is None or (object_store_id not in self.backends and "://" not in object_store_id): try: object_store_id = random.choice(self.weighted_backend_ids) obj.object_store_id = object_store_id @@ -1226,14 +1243,14 @@ def _create(self, obj, **kwargs): "Using preferred backend '%s' for creation of %s %s" % (object_store_id, obj.__class__.__name__, obj.id) ) - return self.backends[object_store_id].create(obj, **kwargs) + return self._resolve_backend(object_store_id).create(obj, **kwargs) else: - return self.backends[object_store_id] + return self._resolve_backend(object_store_id) def _call_method(self, method, obj, default, default_is_exception, **kwargs): object_store_id = self.__get_store_id_for(obj, **kwargs) if object_store_id is not None: - return self.backends[object_store_id].__getattribute__(method)(obj, **kwargs) + return self._resolve_backend(object_store_id).__getattribute__(method)(obj, **kwargs) if default_is_exception: raise default( "objectstore, _call_method failed: %s on %s, kwargs: %s" @@ -1242,6 +1259,14 @@ def _call_method(self, method, obj, default, default_is_exception, **kwargs): else: return default + def _resolve_backend(self, object_store_id: str): + try: + return self.backends[object_store_id] + except KeyError: + if object_store_id.startswith("user_objects://") and self.user_object_store_resolver: + return self.user_object_store_resolver.resolve_object_store_uri(object_store_id) + raise + def get_quota_source_map(self): if self._quota_source_map is None: quota_source_map = QuotaSourceMap() @@ -1259,7 +1284,7 @@ def _merge_quota_source_map(clz, quota_source_map, object_store): def __get_store_id_for(self, obj, **kwargs): if obj.object_store_id is not None: - if obj.object_store_id in self.backends: + if obj.object_store_id in self.backends or obj.object_store_id.startswith("user_objects://"): return obj.object_store_id else: log.warning( @@ -1291,8 +1316,26 @@ def object_store_allows_id_selection(self) -> bool: """Return True if this object store respects object_store_id and allow selection of this.""" return self.allow_user_selection + def validate_selected_object_store_id(self, user, object_store_id: Optional[str]) -> Optional[str]: + parent_check = super().validate_selected_object_store_id(user, object_store_id) + if parent_check or object_store_id is None: + return parent_check + # user selection allowed and object_store_id is not None + if object_store_id.startswith("user_objects://"): + if not user: + return "Supplied object store id is not accessible" + rest_of_uri = object_store_id.split("://", 1)[1] + user_object_store_id = int(rest_of_uri) + for user_object_store in user.object_stores: + if user_object_store.id == user_object_store_id: + return None + return "Supplied object store id was not found" + if object_store_id not in self.object_store_ids_allowing_selection(): + return "Supplied object store id is not an allowed object store selection" + return None + def object_store_ids_allowing_selection(self) -> List[str]: - """Return a non-emtpy list of allowed selectable object store IDs during creation.""" + """Return a non-empty list of allowed selectable object store IDs during creation.""" return self.user_selection_allowed def get_concrete_store_by_object_store_id(self, object_store_id: str) -> Optional["ConcreteObjectStore"]: @@ -1402,7 +1445,9 @@ class ConcreteObjectStoreModel(BaseModel): badges: List[BadgeDict] -def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[BaseObjectStore], Dict[str, Any]]: +def type_to_object_store_class( + store: str, fsmon: bool = False, user_object_store_resolver: Optional[UserObjectStoreResolver] = None +) -> Tuple[Type[BaseObjectStore], Dict[str, Any]]: objectstore_class: Type[BaseObjectStore] objectstore_constructor_kwds = {} if store == "disk": @@ -1422,6 +1467,7 @@ def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[Ba elif store == "distributed": objectstore_class = DistributedObjectStore objectstore_constructor_kwds["fsmon"] = fsmon + objectstore_constructor_kwds["user_object_store_resolver"] = user_object_store_resolver elif store == "hierarchical": objectstore_class = HierarchicalObjectStore objectstore_constructor_kwds["fsmon"] = fsmon @@ -1447,7 +1493,13 @@ def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[Ba return objectstore_class, objectstore_constructor_kwds -def build_object_store_from_config(config, fsmon=False, config_xml=None, config_dict=None): +def build_object_store_from_config( + config, + fsmon=False, + config_xml=None, + config_dict=None, + user_object_store_resolver: Optional[UserObjectStoreResolver] = None, +): """ Invoke the appropriate object store. @@ -1490,13 +1542,55 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None, config_ from_object = "dict" store = config_dict.get("type") - objectstore_class, objectstore_constructor_kwds = type_to_object_store_class(store, fsmon=fsmon) + objectstore_class, objectstore_constructor_kwds = type_to_object_store_class( + store, fsmon=fsmon, user_object_store_resolver=user_object_store_resolver + ) if from_object == "xml": return objectstore_class.from_xml(config=config, config_xml=config_xml, **objectstore_constructor_kwds) else: return objectstore_class(config=config, config_dict=config_dict, **objectstore_constructor_kwds) +# View into the application configuration that is shared between the global object store +# and user defined object stores as produced by concrete_object_store. +class UserObjectStoresAppConfig(BaseModel): + jobs_directory: str + new_file_path: str + umask: int + + +# TODO: this will need app details... +# TODO: unit test from configuration dict... +def concrete_object_store( + object_store_configuration: ObjectStoreConfiguration, app_config: UserObjectStoresAppConfig +) -> ConcreteObjectStore: + + # Adapt structured UserObjectStoresAppConfig into a more full configuration object as expected by + # the object stores + class GalaxyConfigAdapter: + + # Hard code these, these will not support legacy features + object_store_check_old_style = False + object_store_store_by = "uuid" + + # Set this to false for now... not sure but we may want to revisit this + enable_quotas = False + + # These need to come in from Galaxy's config + jobs_directory = app_config.jobs_directory + new_file_path = app_config.new_file_path + umask = app_config.umask + + objectstore_class, objectstore_constructor_kwds = type_to_object_store_class( + store=object_store_configuration.type, + fsmon=False, + ) + assert issubclass(objectstore_class, ConcreteObjectStore) + return objectstore_class( + config=GalaxyConfigAdapter(), config_dict=object_store_configuration.dict(), **objectstore_constructor_kwds + ) + + def local_extra_dirs(func): """Non-local plugin decorator using local directories for the extra_dirs (job_work and temp).""" diff --git a/lib/galaxy/objectstore/templates/__init__.py b/lib/galaxy/objectstore/templates/__init__.py new file mode 100644 index 000000000000..d2aa11d7f5c7 --- /dev/null +++ b/lib/galaxy/objectstore/templates/__init__.py @@ -0,0 +1,15 @@ +from .manager import ConfiguredObjectStoreTemplates +from .models import ( + ObjectStoreConfiguration, + ObjectStoreTemplate, + ObjectStoreTemplateSummaries, + template_to_configuration, +) + +__all__ = ( + "ConfiguredObjectStoreTemplates", + "ObjectStoreConfiguration", + "ObjectStoreTemplate", + "ObjectStoreTemplateSummaries", + "template_to_configuration", +) diff --git a/lib/galaxy/objectstore/templates/manager.py b/lib/galaxy/objectstore/templates/manager.py new file mode 100644 index 000000000000..a4b00938096d --- /dev/null +++ b/lib/galaxy/objectstore/templates/manager.py @@ -0,0 +1,117 @@ +import os +from typing import ( + Any, + Dict, + List, + Optional, +) + +from typing_extensions import Protocol +from yaml import safe_load + +from galaxy.exceptions import ( + ObjectNotFound, + RequestParameterMissingException, +) +from .models import ( + ObjectStoreTemplate, + ObjectStoreTemplateCatalog, + ObjectStoreTemplateSummaries, +) + +RawTemplateConfig = Dict[str, Any] + + +class AppConfigProtocol(Protocol): + object_store_templates: Optional[List[RawTemplateConfig]] + object_store_templates_config_file: Optional[str] + + +class TemplateReference(Protocol): + template_id: str + template_version: int + + +class InstanceDefinition(TemplateReference, Protocol): + variables: Dict[str, Any] + secrets: Dict[str, str] + + +class ConfiguredObjectStoreTemplates: + catalog: ObjectStoreTemplateCatalog + + def __init__(self, catalog: ObjectStoreTemplateCatalog): + self.catalog = catalog + + @staticmethod + def from_app_config(config: AppConfigProtocol) -> "ConfiguredObjectStoreTemplates": + raw_config = config.object_store_templates + if raw_config is None: + config_file = config.object_store_templates_config_file + if config_file and os.path.exists(config_file): + with open(config_file) as f: + raw_config = safe_load(f) + if raw_config is None: + raw_config = [] + return ConfiguredObjectStoreTemplates(raw_config_to_catalog(raw_config)) + + @property + def summaries(self) -> ObjectStoreTemplateSummaries: + templates = self.catalog.__root__ + summaries = [] + for template in templates: + template_dict = template.dict() + template_dict.pop("configuration") + summaries.append(template_dict) + return ObjectStoreTemplateSummaries.parse_obj(summaries) + + def find_template(self, instance_reference: TemplateReference) -> ObjectStoreTemplate: + """Find the corresponding template and throw ObjectNotFound if not available.""" + templates = self.catalog.__root__ + template_id = instance_reference.template_id + template_version = instance_reference.template_version + for template in templates: + if template.id == template_id and template.version == template_version: + return template + + raise ObjectNotFound( + f"Could not find a object store template with id {template_id} and version {template_version}" + ) + + def validate(self, instance: InstanceDefinition): + template = self.find_template(instance) + secrets = instance.secrets + for template_secret in template.secrets or []: + name = template_secret.name + if name not in secrets: + raise RequestParameterMissingException(f"Must define secret '{name}'") + variables = instance.variables + for template_variable in template.variables or []: + name = template_variable.name + if name not in variables: + raise RequestParameterMissingException(f"Must define variable '{name}'") + # TODO: validate no extra variables + + +def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> ObjectStoreTemplateCatalog: + effective_root = _apply_syntactic_sugar(raw_config) + return ObjectStoreTemplateCatalog.parse_obj(effective_root) + + +def _apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]: + templates = [] + for template in raw_templates: + _force_key_to_list(template, "variables") + _force_key_to_list(template, "secrets") + templates.append(template) + return templates + + +def _force_key_to_list(template: RawTemplateConfig, key: str) -> None: + value = template.get(key, None) + if isinstance(value, dict): + value_as_list = [] + for key_name, key_value in value.items(): + key_value["name"] = key_name + value_as_list.append(key_value) + template[key] = value_as_list diff --git a/lib/galaxy/objectstore/templates/models.py b/lib/galaxy/objectstore/templates/models.py new file mode 100644 index 000000000000..aea222218e50 --- /dev/null +++ b/lib/galaxy/objectstore/templates/models.py @@ -0,0 +1,200 @@ +from typing import ( + Any, + Dict, + List, + Optional, + Type, + Union, +) + +from boltons.iterutils import remap +from jinja2.nativetypes import NativeEnvironment +from pydantic import ( + BaseModel, + Extra, +) +from typing_extensions import Literal + + +class StrictModel(BaseModel): + class Config: + extra = Extra.forbid + + +ObjectStoreTemplateVariableType = Literal["string", "boolean", "integer"] +TemplateExpansion = str + + +class S3AuthTemplate(StrictModel): + access_key: Union[str, TemplateExpansion] + secret_key: Union[str, TemplateExpansion] + + +class S3Auth(StrictModel): + access_key: str + secret_key: str + + +class S3BucketTemplate(StrictModel): + name: Union[str, TemplateExpansion] + use_reduced_redundancy: Optional[Union[bool, TemplateExpansion]] = None + + +class S3Bucket(StrictModel): + name: str + use_reduced_redundancy: Optional[bool] = None + + +class S3ObjectStoreTemplateConfiguration(StrictModel): + type: Literal["s3"] + auth: S3AuthTemplate + bucket: S3BucketTemplate + + +class S3ObjectStoreConfiguration(StrictModel): + type: Literal["s3"] + auth: S3Auth + bucket: S3Bucket + + +class AzureAuthTemplate(StrictModel): + account_name: Union[str, TemplateExpansion] + account_key: Union[str, TemplateExpansion] + + +class AzureAuth(StrictModel): + account_name: str + account_key: str + + +class AzureContainerTemplate(StrictModel): + name: Union[str, TemplateExpansion] + + +class AzureContainer(StrictModel): + name: str + + +class AzureObjectStoreTemplateConfiguration(StrictModel): + type: Literal["azure"] + auth: AzureAuthTemplate + container: AzureContainerTemplate + + +class AzureObjectStoreConfiguration(StrictModel): + type: Literal["azure"] + auth: AzureAuth + container: AzureContainer + + +class DiskObjectStoreTemplateConfiguration(StrictModel): + type: Literal["disk"] + files_dir: Union[str, TemplateExpansion] + + +class DiskObjectStoreConfiguration(StrictModel): + type: Literal["disk"] + files_dir: str + + +ObjectStoreTemplateConfiguration = Union[ + S3ObjectStoreTemplateConfiguration, + DiskObjectStoreTemplateConfiguration, + AzureObjectStoreTemplateConfiguration, +] +ObjectStoreConfiguration = Union[ + S3ObjectStoreConfiguration, + DiskObjectStoreConfiguration, + AzureObjectStoreConfiguration, +] +MarkdownContent = str + + +class ObjectStoreTemplateVariable(StrictModel): + name: str + help: Optional[MarkdownContent] + type: ObjectStoreTemplateVariableType + + +class ObjectStoreTemplateSecret(StrictModel): + name: str + help: Optional[MarkdownContent] + + +class ObjectStoreTemplateSummary(StrictModel): + """Version of ObjectStoreTemplate we can send to the UI/API. + + The configuration key in the child type may have secretes + and shouldn't be exposed over the API - at least to non-admins. + """ + + id: str + name: Optional[str] + description: Optional[MarkdownContent] + # The UI should just show the most recent version but allow + # admins to define newer versions with new parameterizations + # and keep old versions in template catalog for backward compatibility + # for users with existing stores of that template. + version: int = 0 + # Like with multiple versions, allow admins to deprecate a + # template by hiding but keep it in the catalog for backward + # compatibility for users with existing stores of that template. + hidden: bool = False + variables: Optional[List[ObjectStoreTemplateVariable]] + secrets: Optional[List[ObjectStoreTemplateSecret]] + + +class ObjectStoreTemplate(ObjectStoreTemplateSummary): + configuration: ObjectStoreTemplateConfiguration + + +class ObjectStoreTemplateCatalog(StrictModel): + """Represents a collection of ObjectStoreTemplates.""" + + __root__: List[ObjectStoreTemplate] + + +class ObjectStoreTemplateSummaries(StrictModel): + """Represents a collection of ObjectStoreTemplate summaries.""" + + __root__: List[ObjectStoreTemplateSummary] + + +def template_to_configuration( + template: ObjectStoreTemplate, + variables: Dict[str, Any], + secrets: Dict[str, str], + user_details: Dict[str, Any], +) -> ObjectStoreConfiguration: + configuration_template = template.configuration + template_variables = { + "variables": variables, + "secrets": secrets, + "user": user_details, + } + + def expand_template(_, key, value): + if isinstance(value, str) and "{{" in value and "}}" in value: + # NativeEnvironment preserves Python types + template = NativeEnvironment().from_string(value) + return key, template.render(**template_variables) + return key, value + + raw_config = remap(configuration_template.dict(), visit=expand_template) + return to_configuration_object(raw_config) + + +TypesToConfigurationClasses: Dict[str, Type[ObjectStoreConfiguration]] = { + "s3": S3ObjectStoreConfiguration, + "azure": AzureObjectStoreConfiguration, + "disk": DiskObjectStoreConfiguration, +} + + +def to_configuration_object(configuration_dict: Dict[str, Any]) -> ObjectStoreConfiguration: + if "type" not in configuration_dict: + raise KeyError("Configuration objects require an object store 'type' key, none found.") + object_store_type = configuration_dict["type"] + if object_store_type not in TypesToConfigurationClasses: + raise ValueError(f"Unknown object store type found in raw configuration dictionary ({object_store_type}).") + return TypesToConfigurationClasses[object_store_type](**configuration_dict) diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 86d890cdf3cb..45576667173f 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -160,10 +160,7 @@ def validate_password(trans, password, confirm): return validate_password_str(password) -def validate_preferred_object_store_id(object_store: ObjectStore, preferred_object_store_id: Optional[str]) -> str: - if not object_store.object_store_allows_id_selection() and preferred_object_store_id is not None: - return "The current configuration doesn't allow selecting preferred object stores." - if object_store.object_store_allows_id_selection() and preferred_object_store_id: - if preferred_object_store_id not in object_store.object_store_ids_allowing_selection(): - return "Supplied object store id is not an allowed object store selection" - return "" +def validate_preferred_object_store_id( + trans, object_store: ObjectStore, preferred_object_store_id: Optional[str] +) -> str: + return object_store.validate_selected_object_store_id(trans.user, preferred_object_store_id) or "" diff --git a/lib/galaxy/webapps/galaxy/api/object_store.py b/lib/galaxy/webapps/galaxy/api/object_store.py index f8d7319b885f..b1ce504f82cf 100644 --- a/lib/galaxy/webapps/galaxy/api/object_store.py +++ b/lib/galaxy/webapps/galaxy/api/object_store.py @@ -5,6 +5,7 @@ from typing import List from fastapi import ( + Body, Path, Query, ) @@ -14,10 +15,15 @@ RequestParameterInvalidException, ) from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.object_store_instances import ( + CreateInstancePayload, + ObjectStoreInstancesManager, +) from galaxy.objectstore import ( BaseObjectStore, ConcreteObjectStoreModel, ) +from galaxy.objectstore.templates import ObjectStoreTemplateSummaries from . import ( depends, DependsOnTrans, @@ -26,7 +32,7 @@ log = logging.getLogger(__name__) -router = Router(tags=["object sstore"]) +router = Router(tags=["object_stores"]) ConcreteObjectStoreIdPathParam: str = Path( ..., title="Concrete Object Store ID", description="The concrete object store ID." @@ -42,6 +48,7 @@ @router.cbv class FastAPIObjectStore: object_store: BaseObjectStore = depends(BaseObjectStore) + object_store_instance_manager: ObjectStoreInstancesManager = depends(ObjectStoreInstancesManager) @router.get( "/api/object_stores", @@ -58,7 +65,24 @@ def index( "The object store index query currently needs to be called with selectable=true" ) selectable_ids = self.object_store.object_store_ids_allowing_selection() - return [self._model_for(selectable_id) for selectable_id in selectable_ids] + instances = [self._model_for(selectable_id) for selectable_id in selectable_ids] + if trans.user: + user_object_stores = trans.user.object_stores + for user_object_store in user_object_stores: + instances.append(self.object_store_instance_manager._to_model(trans, user_object_store)) + return instances + + @router.post( + "/api/object_stores", + summary="Create a user-bound object store.", + operation_id="object_stores__create", + ) + def create( + self, + trans: ProvidesUserContext = DependsOnTrans, + payload: CreateInstancePayload = Body(...), + ) -> ConcreteObjectStoreModel: + return self.object_store_instance_manager.create_instance(trans, payload) @router.get( "/api/object_stores/{object_store_id}", @@ -76,3 +100,14 @@ def _model_for(self, object_store_id: str) -> ConcreteObjectStoreModel: if concrete_object_store is None: raise ObjectNotFound() return concrete_object_store.to_model(object_store_id) + + @router.get( + "/api/object_store_templates", + summary="Get a list of object store templates available to build user defined object stores from", + response_description="A list of the configured object store templates.", + operation_id="object_stores__templates_index", + ) + def index_templates( + self, + ) -> ObjectStoreTemplateSummaries: + return self.object_store_instance_manager.summaries diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index a9bcaba11314..7ac883b77fc0 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -218,6 +218,7 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): webapp.add_client_route("/tours/{tour_id}") webapp.add_client_route("/user") webapp.add_client_route("/user/{form_id}") + webapp.add_client_route("/object_store/create") webapp.add_client_route("/welcome/new") webapp.add_client_route("/visualizations") webapp.add_client_route("/visualizations/edit") diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index ec5e7fefd295..c66e572ae23a 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -1094,9 +1094,13 @@ def get_usage_for(self, label: Optional[str]) -> Dict[str, Any]: def update_user(self, properties: Dict[str, Any]) -> Dict[str, Any]: update_response = self.update_user_raw(properties) - update_response.raise_for_status() + api_asserts.assert_status_code_is_ok(update_response) return update_response.json() + def set_user_preferred_object_store_id(self, store_id: Optional[str]) -> None: + user_properties = self.update_user({"preferred_object_store_id": store_id}) + assert user_properties["preferred_object_store_id"] == store_id + def update_user_raw(self, properties: Dict[str, Any]) -> Response: update_response = self.galaxy_interactor.put("users/current", properties, json=True) return update_response @@ -1375,6 +1379,19 @@ def wait_for_dataset( timeout=timeout, ) + def create_object_store_raw(self, payload: Dict[str, Any]) -> Response: + response = self._post( + "/api/object_stores", + payload, + json=True, + ) + return response + + def create_object_store(self, payload: Dict[str, Any]) -> Dict[str, Any]: + response = self.create_object_store_raw(payload) + response.raise_for_status() + return response.json() + def selectable_object_stores(self) -> List[Dict[str, Any]]: selectable_object_stores_response = self._get("object_stores?selectable=true") selectable_object_stores_response.raise_for_status() diff --git a/lib/galaxy_test/driver/integration_util.py b/lib/galaxy_test/driver/integration_util.py index efa8d1ee1622..276b37800b20 100644 --- a/lib/galaxy_test/driver/integration_util.py +++ b/lib/galaxy_test/driver/integration_util.py @@ -228,14 +228,20 @@ class ConfiguresObjectStores: object_stores_parent: ClassVar[str] _test_driver: GalaxyTestDriver + @classmethod + def write_object_store_config_file(cls, filename: str, contents: str) -> str: + temp_directory = cls.object_stores_parent + config_path = os.path.join(temp_directory, filename) + with open(config_path, "w") as f: + f.write(contents) + return config_path + @classmethod def _configure_object_store(cls, template, config): temp_directory = cls._test_driver.mkdtemp() cls.object_stores_parent = temp_directory - config_path = os.path.join(temp_directory, "object_store_conf.xml") xml = template.safe_substitute({"temp_directory": temp_directory}) - with open(config_path, "w") as f: - f.write(xml) + config_path = cls.write_object_store_config_file("object_store_conf.xml", xml) config["object_store_config_file"] = config_path for path in re.findall(r'files_dir path="([^"]*)"', xml): assert path.startswith(temp_directory) diff --git a/test/integration/objectstore/test_per_user.py b/test/integration/objectstore/test_per_user.py new file mode 100644 index 000000000000..d0b88c26eba2 --- /dev/null +++ b/test/integration/objectstore/test_per_user.py @@ -0,0 +1,111 @@ +from typing import ( + Any, + Dict, + Tuple, +) + +from galaxy_test.base import api_asserts +from ._base import BaseObjectStoreIntegrationTestCase +from .test_selection_with_resource_parameters import ( + DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, +) + +LIBRARY_2 = """ +- id: general_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/{{ user.username }}' +- id: secure_disk + name: Secure Disk + description: Secure Disk Bound to You + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}' +""" + + +class TestPerUserObjectStoreIntegration(BaseObjectStoreIntegrationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config) + template = LIBRARY_2.replace("/data", cls.object_stores_parent) + template_config_path = cls.write_object_store_config_file("templates.yml", template) + config["object_store_templates_config_file"] = template_config_path + + def test_create_and_use_simple(self): + body = { + "name": "My Cool Disk", + "template_id": "general_disk", + "template_version": 0, + "secrets": {}, + "variables": {}, + } + object_store_json = self.dataset_populator.create_object_store(body) + assert "name" in object_store_json + assert object_store_json["name"] == "My Cool Disk" + object_store_id = object_store_json["object_store_id"] + assert object_store_id.startswith("user_objects://") + + object_stores = self.dataset_populator.selectable_object_stores() + assert len(object_stores) == 1 + user_object_store = object_stores[0] + assert user_object_store["name"] == "My Cool Disk" + + with self.dataset_populator.test_history() as history_id: + storage_info, hda1 = self._create_hda_get_storage_info(history_id) + assert storage_info["object_store_id"] == "default" + self.dataset_populator.set_user_preferred_object_store_id(object_store_id) + + def _run_tool(tool_id, inputs, preferred_object_store_id=None): + response = self.dataset_populator.run_tool( + tool_id, + inputs, + history_id, + preferred_object_store_id=preferred_object_store_id, + ) + self.dataset_populator.wait_for_history(history_id) + return response + + hda1_input = {"src": "hda", "id": hda1["id"]} + response = _run_tool("multi_data_param", {"f1": hda1_input, "f2": hda1_input}) + storage_info, output = self._storage_info_for_job_output(response) + assert storage_info["object_store_id"] == object_store_id + contents = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert contents.startswith("1 2 3") + + def test_create_unknown_id(self): + body = { + "template_id": "general_disk_2", + "template_version": 0, + "secrets": {}, + "variables": {}, + } + response = self.dataset_populator.create_object_store_raw(body) + api_asserts.assert_status_code_is(response, 404) + + def test_create_invalid_version(self): + body = { + "template_id": "general_disk", + "template_version": "0.0.0", + "secrets": {}, + "variables": {}, + } + response = self.dataset_populator.create_object_store_raw(body) + api_asserts.assert_status_code_is(response, 400) + + def _create_hda_get_storage_info(self, history_id: str): + hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") + self.dataset_populator.wait_for_history(history_id) + return self.dataset_populator.dataset_storage_info(hda1["id"]), hda1 + + def _storage_info_for_job_output(self, job_dict) -> Tuple[Dict[str, Any], Dict[str, Any]]: + outputs = job_dict["outputs"] # could be a list or dictionary depending on source + try: + output = outputs[0] + except KeyError: + output = list(outputs.values())[0] + storage_info = self.dataset_populator.dataset_storage_info(output["id"]) + return storage_info, output diff --git a/test/integration/objectstore/test_selection_with_user_preferred_object_store.py b/test/integration/objectstore/test_selection_with_user_preferred_object_store.py index ecb612fe9c5b..231513f496a7 100644 --- a/test/integration/objectstore/test_selection_with_user_preferred_object_store.py +++ b/test/integration/objectstore/test_selection_with_user_preferred_object_store.py @@ -468,9 +468,8 @@ def _storage_info_for_job_output(self, job_dict) -> Dict[str, Any]: def _storage_info(self, hda): return self.dataset_populator.dataset_storage_info(hda["id"]) - def _set_user_preferred_object_store_id(self, store_id: Optional[str]): - user_properties = self.dataset_populator.update_user({"preferred_object_store_id": store_id}) - assert user_properties["preferred_object_store_id"] == store_id + def _set_user_preferred_object_store_id(self, store_id: Optional[str]) -> None: + self.dataset_populator.set_user_preferred_object_store_id(store_id) def _reset_user_preferred_object_store_id(self): self._set_user_preferred_object_store_id(None) diff --git a/test/unit/objectstore/test_from_configuration_object.py b/test/unit/objectstore/test_from_configuration_object.py new file mode 100644 index 000000000000..4bdf13467ec6 --- /dev/null +++ b/test/unit/objectstore/test_from_configuration_object.py @@ -0,0 +1,40 @@ +from galaxy.objectstore.templates.models import ( + DiskObjectStoreConfiguration, +) +from galaxy.objectstore import ( + concrete_object_store, + UserObjectStoresAppConfig, +) +from .test_objectstore import MockDataset + + +def test_disk(tmpdir): + files_dir = tmpdir / "moo" + files_dir.mkdir() + configuration = DiskObjectStoreConfiguration( + type="disk", + files_dir=str(files_dir), + ) + app_config = UserObjectStoresAppConfig( + jobs_directory=str(tmpdir / "jobs"), + new_file_path=str(tmpdir / "new_files"), + umask=0o077, + ) + object_store = concrete_object_store(configuration, app_config) + + absent_dataset = MockDataset(1) + assert not object_store.exists(absent_dataset) + + # Write empty dataset 2 in second backend, ensure it is empty and + # exists. + empty_dataset = MockDataset(2) + object_store.create(empty_dataset) + object_store.exists(empty_dataset) + assert object_store.size(empty_dataset) == 0 + + example_dataset = MockDataset(3) + temp_file = tmpdir / "example.txt" + temp_file.write_text("moo cow", "utf-8") + object_store.create(example_dataset) + object_store.update_from_file(example_dataset, file_name=str(temp_file)) + assert object_store.size(example_dataset) == 7 diff --git a/test/unit/objectstore/test_template_manager.py b/test/unit/objectstore/test_template_manager.py new file mode 100644 index 000000000000..86f245f2ac92 --- /dev/null +++ b/test/unit/objectstore/test_template_manager.py @@ -0,0 +1,18 @@ +from galaxy.objectstore.templates import ConfiguredObjectStoreTemplates +from .test_template_models import LIBRARY_2 + + +class MockConfig: + def __init__(self, config_path): + self.object_store_templates = None + self.object_store_templates_config_file = config_path + + +def test_manager(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_2, "utf-8") + config = MockConfig(config_path) + templates = ConfiguredObjectStoreTemplates.from_app_config(config) + summaries = templates.summaries + assert summaries + assert len(summaries.__root__) == 2 diff --git a/test/unit/objectstore/test_template_models.py b/test/unit/objectstore/test_template_models.py new file mode 100644 index 000000000000..37818dfc8944 --- /dev/null +++ b/test/unit/objectstore/test_template_models.py @@ -0,0 +1,137 @@ +from yaml import safe_load + +from galaxy.objectstore.templates.manager import raw_config_to_catalog +from galaxy.objectstore.templates.models import ( + AzureObjectStoreConfiguration, + DiskObjectStoreConfiguration, + ObjectStoreTemplateCatalog, + S3ObjectStoreConfiguration, + template_to_configuration, +) + +LIBRARY_1 = """ +- id: amazon_bucket + name: Amazon Bucket + description: An Amazon S3 Bucket + variables: + use_reduced_redundancy: + type: boolean + help: Reduce redundancy and save money. + secrets: + access_key: + help: AWS access key to use when connecting to AWS resources. + secret_key: + help: AWS secret key to use when connecting to AWS resources. + bucket_name: + help: Name of bucket to use when connecting to AWS resources. + configuration: + type: s3 + auth: + access_key: '{{ secrets.access_key}}' + secret_key: '{{ secrets.secret_key}}' + bucket: + name: '{{ secrets.bucket_name}}' + use_reduced_redundancy: '{{ variables.use_reduced_redundancy}}' +""" + + +def test_parsing_simple_s3(): + template_library = _parse_template_library(LIBRARY_1) + assert len(template_library.__root__) == 1 + s3_template = template_library.__root__[0] + assert s3_template.description == "An Amazon S3 Bucket" + configuration_obj = template_to_configuration( + s3_template, + {"use_reduced_redundancy": False}, + {"access_key": "sec1", "secret_key": "sec2", "bucket_name": "sec3"}, + user_details={}, + ) + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, S3ObjectStoreConfiguration) + configuration = configuration_obj.dict() + + assert configuration["type"] == "s3" + assert configuration["auth"]["access_key"] == "sec1" + assert configuration["auth"]["secret_key"] == "sec2" + assert configuration["bucket"]["name"] == "sec3" + assert configuration["bucket"]["use_reduced_redundancy"] is False + + +LIBRARY_2 = """ +- id: general_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/{{ user.username }}' +- id: secure_disk + name: Secure Disk + description: Secure Disk Bound to You + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}' +""" + + +def test_parsing_multiple_posix(): + template_library = _parse_template_library(LIBRARY_2) + assert len(template_library.__root__) == 2 + general_template = template_library.__root__[0] + secure_template = template_library.__root__[1] + + assert general_template.version == 0 + assert secure_template.version == 0 + assert secure_template.hidden is False + + general_configuration = template_to_configuration(general_template, {}, {}, user_details={"username": "jane"}) + assert isinstance(general_configuration, DiskObjectStoreConfiguration) + assert general_configuration.files_dir == "/data/general/jane" + + secure_configuration = template_to_configuration(secure_template, {}, {}, user_details={"username": "jane"}) + assert isinstance(secure_configuration, DiskObjectStoreConfiguration) + assert secure_configuration.files_dir == "/data/secure/jane" + + +LIBRARY_AZURE_CONTAINER = """ +- id: amazon_bucket + name: Azure Container + description: An Azure Container + variables: + account_name: + type: string + help: Azure account name to use when connecting to Azure resources. + secrets: + account_key: + help: Azure account key to use when connecting to Azure resources. + container_name: + help: Name of container to use when connecting to Azure cloud resources. + configuration: + type: azure + auth: + account_name: '{{ variables.account_name}}' + account_key: '{{ secrets.account_key}}' + container: + name: '{{ secrets.container_name}}' +""" + + +def test_parsing_azure(): + template_library = _parse_template_library(LIBRARY_AZURE_CONTAINER) + assert len(template_library.__root__) == 1 + azure_template = template_library.__root__[0] + assert azure_template.description == "An Azure Container" + configuration_obj = template_to_configuration( + azure_template, + {"account_name": "galaxyproject"}, + {"account_key": "sec1", "container_name": "sec2"}, + user_details={}, + ) + assert isinstance(configuration_obj, AzureObjectStoreConfiguration) + assert configuration_obj.auth.account_name == "galaxyproject" + assert configuration_obj.auth.account_key == "sec1" + assert configuration_obj.container.name == "sec2" + + +def _parse_template_library(contents: str) -> ObjectStoreTemplateCatalog: + raw_contents = safe_load(contents) + return raw_config_to_catalog(raw_contents)