diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 41c5bd3b2eec..67b97e8f6d7e 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -12500,7 +12500,7 @@ export interface components { /** Device */ device?: string | null; /** Id */ - id: number; + id: number | string; /** Name */ name?: string | null; /** Object Store Id */ @@ -12519,6 +12519,8 @@ export interface components { * @enum {string} */ type: "s3" | "azure_blob" | "disk" | "generic_s3"; + /** Uuid */ + uuid: string; /** Variables */ variables: { [key: string]: (string | boolean | number) | undefined; @@ -20529,7 +20531,7 @@ export interface operations { header?: { "run-as"?: string | null; }; - /** @description The model ID for a persisted UserObjectStore object. */ + /** @description The identifier used to index a persisted UserObjectStore object. */ path: { user_object_store_id: string; }; @@ -20556,7 +20558,7 @@ export interface operations { header?: { "run-as"?: string | null; }; - /** @description The model ID for a persisted UserObjectStore object. */ + /** @description The identifier used to index a persisted UserObjectStore object. */ path: { user_object_store_id: string; }; diff --git a/client/src/components/ObjectStore/Instances/CreateForm.vue b/client/src/components/ObjectStore/Instances/CreateForm.vue index 7ff59ae6aebe..e82f0f1c484a 100644 --- a/client/src/components/ObjectStore/Instances/CreateForm.vue +++ b/client/src/components/ObjectStore/Instances/CreateForm.vue @@ -19,7 +19,7 @@ interface CreateFormProps { } const error = ref(null); const props = defineProps(); -const title = "Create a new object store for your data"; +const title = "Create a new storage location for your data"; const submitTitle = "Submit"; const inputs = computed(() => { diff --git a/client/src/components/ObjectStore/Instances/CreateInstance.vue b/client/src/components/ObjectStore/Instances/CreateInstance.vue index 91d2b996bb8e..f4128c140b3b 100644 --- a/client/src/components/ObjectStore/Instances/CreateInstance.vue +++ b/client/src/components/ObjectStore/Instances/CreateInstance.vue @@ -28,7 +28,7 @@ async function onCreated(objectStore: UserConcreteObjectStore) { diff --git a/client/src/components/ObjectStore/Instances/EditInstance.vue b/client/src/components/ObjectStore/Instances/EditInstance.vue index fc8bcbc5fecd..0776f18e6e65 100644 --- a/client/src/components/ObjectStore/Instances/EditInstance.vue +++ b/client/src/components/ObjectStore/Instances/EditInstance.vue @@ -44,7 +44,7 @@ const inputs = computed(() => { return form; }); -const title = computed(() => `Edit Object Store ${instance.value?.name} Settings`); +const title = computed(() => `Edit Storage Location ${instance.value?.name} Settings`); const hasSecrets = computed(() => instance.value?.secrets && instance.value?.secrets.length > 0); const submitTitle = "Update Settings"; diff --git a/client/src/components/ObjectStore/Instances/EditSecrets.vue b/client/src/components/ObjectStore/Instances/EditSecrets.vue index da3bb94cc1b6..42038241d42d 100644 --- a/client/src/components/ObjectStore/Instances/EditSecrets.vue +++ b/client/src/components/ObjectStore/Instances/EditSecrets.vue @@ -14,7 +14,7 @@ interface Props { template: ObjectStoreTemplateSummary; } const props = defineProps(); -const title = computed(() => `Update Object Store ${props.objectStore?.name} Secrets`); +const title = computed(() => `Update Storage Location ${props.objectStore?.name} Secrets`); async function onUpdate(secretName: string, secretValue: string) { const payload = { diff --git a/client/src/components/User/UserPreferences.vue b/client/src/components/User/UserPreferences.vue index 94be0c793706..6807bda7f47a 100644 --- a/client/src/components/User/UserPreferences.vue +++ b/client/src/components/User/UserPreferences.vue @@ -92,13 +92,13 @@ :preferred-object-store-id="currentUser.preferred_object_store_id" :user-id="userId"> - ObjectStoreTemplateSummaries: @@ -123,10 +129,10 @@ def _upgrade_instance( ) -> UserConcreteObjectStoreModel: persisted_object_store = self._get(trans, id) catalog = self._catalog - template = catalog.find_template_by(persisted_object_store.object_store_template_id, payload.template_version) - persisted_object_store.object_store_template_version = template.version - persisted_object_store.object_store_template_definition = template.model_dump() - old_variables = persisted_object_store.object_store_template_variables or {} + template = catalog.find_template_by(persisted_object_store.template_id, payload.template_version) + persisted_object_store.template_version = template.version + persisted_object_store.template_definition = template.model_dump() + old_variables = persisted_object_store.template_variables or {} updated_variables = payload.variables actual_variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = {} for variable in template.variables or []: @@ -136,11 +142,11 @@ def _upgrade_instance( if updated_value: actual_variables[variable_name] = updated_value - persisted_object_store.object_store_template_variables = actual_variables - old_secrets = persisted_object_store.object_store_template_secrets or [] + persisted_object_store.template_variables = actual_variables + old_secrets = persisted_object_store.template_secrets or [] new_secrets = payload.secrets - recorded_secrets = persisted_object_store.object_store_template_secrets or [] + recorded_secrets = persisted_object_store.template_secrets or [] user_vault = trans.user_vault upgraded_template_secrets = [] @@ -154,7 +160,7 @@ def _upgrade_instance( continue secret_value = new_secrets[secret_name] - key = user_vault_key(persisted_object_store, secret_name) + key = user_vault_key(persisted_object_store, secret_name, self._app_config) user_vault.write_secret(key, secret_value) if secret_name not in recorded_secrets: recorded_secrets.append(secret_name) @@ -162,7 +168,7 @@ def _upgrade_instance( secrets_to_delete: List[str] = [] for recorded_secret in recorded_secrets: if recorded_secret not in upgraded_template_secrets: - key = user_vault_key(persisted_object_store, recorded_secret) + key = user_vault_key(persisted_object_store, recorded_secret, self._app_config) log.info(f"deleting {key} from user vault") user_vault.delete_secret(key) secrets_to_delete.append(recorded_secret) @@ -170,7 +176,7 @@ def _upgrade_instance( for secret_to_delete in secrets_to_delete: recorded_secrets.remove(secret_to_delete) - persisted_object_store.object_store_template_secrets = recorded_secrets + persisted_object_store.template_secrets = recorded_secrets self._save(persisted_object_store) rval = self._to_model(trans, persisted_object_store) return rval @@ -188,7 +194,7 @@ def _update_instance( persisted_object_store.description = payload.description if payload.variables is not None: # maybe just record the valid variables according to template like in upgrade - persisted_object_store.object_store_template_variables = payload.variables + persisted_object_store.template_variables = payload.variables self._save(persisted_object_store) return self._to_model(trans, persisted_object_store) @@ -197,7 +203,7 @@ def _update_instance_secret( ) -> UserConcreteObjectStoreModel: persisted_object_store = self._get(trans, id) user_vault = trans.user_vault - key = user_vault_key(persisted_object_store, payload.secret_name) + key = user_vault_key(persisted_object_store, payload.secret_name, self._app_config) user_vault.write_secret(key, payload.secret_value) return self._to_model(trans, persisted_object_store) @@ -212,10 +218,11 @@ def create_instance( persisted_object_store = UserObjectStore() persisted_object_store.user_id = trans.user.id assert persisted_object_store.user_id - persisted_object_store.object_store_template_definition = template.model_dump() - persisted_object_store.object_store_template_id = template.id - persisted_object_store.object_store_template_version = template.version - persisted_object_store.object_store_template_variables = payload.variables + persisted_object_store.uuid = uuid4().hex + persisted_object_store.template_definition = template.model_dump() + persisted_object_store.template_id = template.id + persisted_object_store.template_version = template.version + persisted_object_store.template_variables = payload.variables persisted_object_store.name = payload.name persisted_object_store.description = payload.description self._save(persisted_object_store) @@ -231,13 +238,13 @@ def create_instance( recorded_secrets = [] try: for secret, value in payload.secrets.items(): - key = user_vault_key(persisted_object_store, secret) + key = user_vault_key(persisted_object_store, secret, self._app_config) user_vault.write_secret(key, value) recorded_secrets.append(secret) except Exception: self._sa_session.delete(persisted_object_store) raise - persisted_object_store.object_store_template_secrets = recorded_secrets + persisted_object_store.template_secrets = recorded_secrets self._save(persisted_object_store) return self._to_model(trans, persisted_object_store) @@ -255,13 +262,25 @@ def _save(self, persisted_object_store: UserObjectStore) -> None: self._sa_session.commit() def _get(self, trans: ProvidesUserContext, id: Union[str, int]) -> UserObjectStore: - user_object_store = self._sa_session.query(UserObjectStore).get(int(id)) + filter = self._index_filter(id) + user_object_store = self._sa_session.query(UserObjectStore).filter(filter).one_or_none() if user_object_store is None: raise RequestParameterInvalidException(f"Failed to fetch object store for id {id}") if user_object_store.user != trans.user: raise ItemOwnershipException() return user_object_store + def _index_filter(self, id: Union[str, int]): + index_by = self._app_config.user_object_store_index_by + index_filter: Any + if index_by == "id": + id_as_int = int(id) + index_filter = UserObjectStore.__table__.c.id == id_as_int + else: + id_as_str = str(id) + index_filter = UserObjectStore.__table__.c.uuid == id_as_str + return index_filter + def _to_model(self, trans, persisted_object_store: UserObjectStore) -> UserConcreteObjectStoreModel: quota = QuotaModel(source=None, enabled=False) object_store_type = persisted_object_store.template.configuration.type @@ -274,32 +293,47 @@ def _to_model(self, trans, persisted_object_store: UserObjectStore) -> UserConcr object_store_type in ["azure_blob", "s3"], ) # These shouldn't be null but sometimes can be? - secrets = persisted_object_store.object_store_template_secrets or [] + secrets = persisted_object_store.template_secrets or [] + uos_id: str + if self._app_config.user_object_store_index_by == "id": + uos_id = str(persisted_object_store.id) + response_id = persisted_object_store.id + else: + uos_id = str(persisted_object_store.uuid) + response_id = uos_id + object_store_id = f"user_objects://{uos_id}" + return UserConcreteObjectStoreModel( - id=persisted_object_store.id, + id=response_id, + uuid=str(persisted_object_store.uuid), type=object_store_type, - template_id=persisted_object_store.object_store_template_id, - template_version=persisted_object_store.object_store_template_version, - variables=persisted_object_store.object_store_template_variables, + template_id=persisted_object_store.template_id, + template_version=persisted_object_store.template_version, + variables=persisted_object_store.template_variables, secrets=secrets, name=persisted_object_store.name, description=persisted_object_store.description, - object_store_id=f"user_objects://{persisted_object_store.id}", + object_store_id=object_store_id, private=True, quota=quota, badges=badges, ) -def user_vault_key(user_object_store: UserObjectStore, secret: str) -> str: - uos_id = user_object_store.id +def user_vault_key(user_object_store: UserObjectStore, secret: str, app_config: UserObjectStoresAppConfig) -> str: + if app_config.user_object_store_index_by == "id": + uos_id = str(user_object_store.id) + else: + uos_id = str(user_object_store.uuid) assert uos_id user_vault_id_prefix = f"object_store_config/{uos_id}" key = f"{user_vault_id_prefix}/{secret}" return key -def recover_secrets(user_object_store: UserObjectStore, vault: Vault) -> Dict[str, str]: +def recover_secrets( + user_object_store: UserObjectStore, vault: Vault, app_config: UserObjectStoresAppConfig +) -> Dict[str, str]: user: User = user_object_store.user user_vault = UserVaultWrapper(vault, user) secrets: Dict[str, str] = {} @@ -307,9 +341,9 @@ def recover_secrets(user_object_store: UserObjectStore, vault: Vault) -> Dict[st # ones recorded as written in the persisted object, the ones # expected in the catalog, or the ones expected in the definition # persisted. - persisted_secret_names = user_object_store.object_store_template_secrets or [] + persisted_secret_names = user_object_store.template_secrets or [] for secret in persisted_secret_names: - vault_key = user_vault_key(user_object_store, secret) + vault_key = user_vault_key(user_object_store, secret, app_config) secret_value = user_vault.read_secret(vault_key) # assert secret_value if secret_value is not None: @@ -325,8 +359,13 @@ def __init__(self, sa_session: galaxy_scoped_session, vault: Vault, app_config: def resolve_object_store_uri_config(self, uri: str) -> ObjectStoreConfiguration: user_object_store_id = uri.split("://", 1)[1] - id_filter = UserObjectStore.__table__.c.id == user_object_store_id - user_object_store: UserObjectStore = self._sa_session.query(UserObjectStore).filter(id_filter).one() - secrets = recover_secrets(user_object_store, self._vault) + index_by = self._app_config.user_object_store_index_by + index_filter: Any + if index_by == "id": + index_filter = UserObjectStore.__table__.c.id == user_object_store_id + else: + index_filter = UserObjectStore.__table__.c.uuid == user_object_store_id + user_object_store: UserObjectStore = self._sa_session.query(UserObjectStore).filter(index_filter).one() + secrets = recover_secrets(user_object_store, self._vault, self._app_config) object_store_configuration = user_object_store.object_store_configuration(secrets=secrets) return object_store_configuration diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3c3167141796..9a4f39a743af 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -10889,6 +10889,7 @@ class UserObjectStore(Base, RepresentById): id: Mapped[int] = mapped_column(Integer, primary_key=True) user_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("galaxy_user.id"), index=True) + uuid: Mapped[Union[UUID, str]] = mapped_column(UUIDType(), index=True) create_time: Mapped[datetime] = mapped_column(DateTime, default=now) update_time: Mapped[datetime] = mapped_column(DateTime, default=now, onupdate=now, index=True) # user specified name of the instance they've created @@ -10896,30 +10897,26 @@ class UserObjectStore(Base, RepresentById): # user specified description of the instance they've created description: Mapped[Optional[str]] = mapped_column(Text) # the template store id - object_store_template_id: Mapped[str] = mapped_column(String(255), index=True) + template_id: Mapped[str] = mapped_column(String(255), index=True) # the template store version (0, 1, ...) - object_store_template_version: Mapped[int] = mapped_column(Integer, index=True) + template_version: Mapped[int] = mapped_column(Integer, index=True) # Full template from object_store_templates.yml catalog. # For tools we just store references, so here we could easily just use # the id/version and not record the definition... as the templates change # over time this choice has some big consequences despite being easy to swap # implementations. - object_store_template_definition: Mapped[Optional[OBJECT_STORE_TEMPLATE_DEFINITION_TYPE]] = mapped_column(JSONType) + template_definition: Mapped[Optional[OBJECT_STORE_TEMPLATE_DEFINITION_TYPE]] = mapped_column(JSONType) # Big JSON blob of the variable name -> value mapping defined for the store's # variables by the user. - object_store_template_variables: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE]] = ( - mapped_column(JSONType) - ) + template_variables: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE]] = mapped_column(JSONType) # Track a list of secrets that were defined for this object store at creation - object_store_template_secrets: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_SECRET_NAMES_TYPE]] = ( - mapped_column(JSONType) - ) + template_secrets: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_SECRET_NAMES_TYPE]] = mapped_column(JSONType) user = relationship("User", back_populates="object_stores") @property def template(self) -> ObjectStoreTemplate: - return ObjectStoreTemplate(**self.object_store_template_definition or {}) + return ObjectStoreTemplate(**self.template_definition or {}) def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConfiguration: user = self.user @@ -10928,7 +10925,7 @@ def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConf "email": user.email, "id": user.id, } - variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.object_store_template_variables or {} + variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {} return template_to_configuration( self.template, variables=variables, diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py index 47ef643059f7..09d296af5a74 100644 --- a/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66a_add_user_defined_object_stores.py @@ -15,7 +15,10 @@ Text, ) -from galaxy.model.custom_types import JSONType +from galaxy.model.custom_types import ( + JSONType, + UUIDType, +) from galaxy.model.migrations.util import ( create_table, drop_table, @@ -37,15 +40,16 @@ def upgrade(): table_name, Column("id", Integer, primary_key=True), Column("user_id", Integer, ForeignKey("galaxy_user.id"), nullable=False, index=True), + Column("uuid", UUIDType, nullable=False, index=True), Column("name", String(255), index=True), Column("description", Text, index=True), Column("create_time", DateTime), Column("update_time", DateTime), - Column("object_store_template_id", String(255), index=True), - Column("object_store_template_version", Integer, index=True), - Column("object_store_template_definition", JSONType), - Column("object_store_template_variables", JSONType), - Column("object_store_template_secrets", JSONType), + Column("template_id", String(255), index=True), + Column("template_version", Integer, index=True), + Column("template_definition", JSONType), + Column("template_variables", JSONType), + Column("template_secrets", JSONType), ) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 205859833014..88fea354c436 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -26,7 +26,10 @@ import yaml from pydantic import BaseModel -from typing_extensions import Protocol +from typing_extensions import ( + Literal, + Protocol, +) from galaxy.exceptions import ( ConfigDoesNotAllowException, @@ -1360,10 +1363,17 @@ def validate_selected_object_store_id(self, user, object_store_id: Optional[str] if not user: return "Supplied object store id is not accessible" rest_of_uri = object_store_id.split("://", 1)[1] - user_object_store_id = int(rest_of_uri) - for user_object_store in user.object_stores: - if user_object_store.id == user_object_store_id: - return None + index_by = self.config.user_object_store_index_by + if index_by == "id": + user_object_store_id = int(rest_of_uri) + for user_object_store in user.object_stores: + if user_object_store.id == user_object_store_id: + return None + else: + user_object_store_uuid = rest_of_uri + for user_object_store in user.object_stores: + if str(user_object_store.uuid) == user_object_store_uuid: + return None return "Supplied object store id was not found" if object_store_id not in self.object_store_ids_allowing_selection(): return "Supplied object store id is not an allowed object store selection" @@ -1624,6 +1634,7 @@ def build_object_store_from_config( class UserObjectStoresAppConfig(BaseModel): object_store_cache_path: str object_store_cache_size: int + user_object_store_index_by: Literal["uuid", "id"] jobs_directory: str new_file_path: str umask: int diff --git a/lib/galaxy/webapps/galaxy/api/object_store.py b/lib/galaxy/webapps/galaxy/api/object_store.py index 42851c940e1f..905318604462 100644 --- a/lib/galaxy/webapps/galaxy/api/object_store.py +++ b/lib/galaxy/webapps/galaxy/api/object_store.py @@ -42,7 +42,9 @@ ) UserObjectStoreIdPathParam: str = Path( - ..., title="User Object Store ID", description="The model ID for a persisted UserObjectStore object." + ..., + title="User Object Store Identifier", + description="The identifier used to index a persisted UserObjectStore object.", ) SelectableQueryParam: bool = Query( diff --git a/test/unit/objectstore/test_from_configuration_object.py b/test/unit/objectstore/test_from_configuration_object.py index e1e9dac6edee..3829bd64fd7b 100644 --- a/test/unit/objectstore/test_from_configuration_object.py +++ b/test/unit/objectstore/test_from_configuration_object.py @@ -41,5 +41,6 @@ def app_config(tmpdir) -> UserObjectStoresAppConfig: umask=0o077, object_store_cache_path=str(tmpdir / "cache"), object_store_cache_size=1, + user_object_store_index_by="uuid", ) return app_config