From 76c0f985fbbf2e4feba818420d75c59b21f4b60f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 27 Feb 2023 09:25:44 -0500 Subject: [PATCH 1/8] Pydantic the new object store API. --- client/src/schema/schema.ts | 66 ++++++++++++++--- lib/galaxy/objectstore/__init__.py | 72 ++++++++++++++++--- lib/galaxy/webapps/galaxy/api/object_store.py | 36 +++++----- 3 files changed, 138 insertions(+), 36 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index 1723cc3a1271..dbfdf897cdf6 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -890,11 +890,11 @@ export interface paths { post: operations["create_api_metrics_post"]; }; "/api/object_store": { - /** Index */ + /** Get a list of (currently only concrete) object stores configured with this Galaxy instance. */ get: operations["index_api_object_store_get"]; }; "/api/object_store/{object_store_id}": { - /** Return boolean to indicate if Galaxy's default object store allows selection. */ + /** Get information about a concrete object store configured with Galaxy. */ get: operations["show_info_api_object_store__object_store_id__get"]; }; "/api/pages": { @@ -1432,6 +1432,34 @@ export interface components { /** Queue of task being done derived from Celery AsyncResult */ queue?: string; }; + /** BadgeDict */ + BadgeDict: { + /** Message */ + message: string; + /** + * Source + * @enum {string} + */ + source: "admin" | "galaxy"; + /** + * Type + * @enum {string} + */ + type: + | "faster" + | "slower" + | "short_term" + | "cloud" + | "backed_up" + | "not_backed_up" + | "more_secure" + | "less_secure" + | "more_stable" + | "less_stable" + | "quota" + | "no_quota" + | "restricted"; + }; /** * BasicRoleModel * @description Base model definition with common configuration used by all derived models. @@ -1718,6 +1746,20 @@ export interface components { */ hash_function?: components["schemas"]["HashFunctionNameEnum"]; }; + /** ConcreteObjectStoreModel */ + ConcreteObjectStoreModel: { + /** Badges */ + badges: components["schemas"]["BadgeDict"][]; + /** Description */ + description?: string; + /** Name */ + name?: string; + /** Object Store Id */ + object_store_id?: string; + /** Private */ + private: boolean; + quota: components["schemas"]["QuotaModel"]; + }; /** ContentsObject */ ContentsObject: { /** @@ -6243,6 +6285,13 @@ export interface components { */ users?: components["schemas"]["UserQuota"][]; }; + /** QuotaModel */ + QuotaModel: { + /** Enabled */ + enabled: boolean; + /** Source */ + source?: string; + }; /** * QuotaOperation * @description An enumeration. @@ -12568,9 +12617,9 @@ export interface operations { }; }; index_api_object_store_get: { - /** Index */ + /** Get a list of (currently only concrete) object stores configured with this Galaxy instance. */ parameters?: { - /** @description Restrict index query to user selectable object stores. */ + /** @description Restrict index query to user selectable object stores, the current implementation requires this to be true. */ query?: { selectable?: boolean; }; @@ -12580,9 +12629,10 @@ export interface operations { }; }; responses: { + /** @description A list of the configured object stores. */ 200: { content: { - "application/json": Record[]; + "application/json": components["schemas"]["ConcreteObjectStoreModel"][]; }; }; /** @description Validation Error */ @@ -12594,7 +12644,7 @@ export interface operations { }; }; show_info_api_object_store__object_store_id__get: { - /** Return boolean to indicate if Galaxy's default object store allows selection. */ + /** Get information about a concrete object store configured with Galaxy. */ parameters: { /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { @@ -12606,10 +12656,10 @@ export interface operations { }; }; responses: { - /** @description A list with details about the remote files available to the user. */ + /** @description Successful Response */ 200: { content: { - "application/json": Record; + "application/json": components["schemas"]["ConcreteObjectStoreModel"]; }; }; /** @description Validation Error */ diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 45793c187f92..d0b0e35b4c05 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -24,6 +24,11 @@ ) import yaml +from pydantic import BaseModel +from typing_extensions import ( + Literal, + TypedDict, +) from galaxy.exceptions import ( ObjectInvalid, @@ -52,6 +57,22 @@ log = logging.getLogger(__name__) +BadgeSourceT = Literal["admin", "galaxy"] +BadgeT = Literal[ + "faster", + "slower", + "short_term", + "cloud", + "backed_up", + "not_backed_up", + "more_secure", + "less_secure", + "more_stable", + "less_stable", + "quota", + "no_quota", + "restricted", +] BADGE_SPECIFICATION = [ {"type": "faster", "conflicts": ["slower"]}, @@ -69,6 +90,12 @@ BADGE_SPECIFCATION_BY_TYPE = {s["type"]: s for s in BADGE_SPECIFICATION} +class BadgeDict(TypedDict): + type: BadgeT + message: Optional[str] + source: BadgeSourceT + + class ObjectStore(metaclass=abc.ABCMeta): """ObjectStore interface. @@ -270,7 +297,7 @@ def get_concrete_store_description_markdown(self, obj): """ @abc.abstractmethod - def get_concrete_store_badges(self, obj): + def get_concrete_store_badges(self, obj) -> List[BadgeDict]: """Return a list of dictified badges summarizing the object store configuration.""" @abc.abstractmethod @@ -443,7 +470,7 @@ def get_concrete_store_name(self, obj): def get_concrete_store_description_markdown(self, obj): return self._invoke("get_concrete_store_description_markdown", obj) - def get_concrete_store_badges(self, obj) -> List[Dict[str, Any]]: + def get_concrete_store_badges(self, obj) -> List[BadgeDict]: return self._invoke("get_concrete_store_badges", obj) def get_store_usage_percent(self): @@ -549,14 +576,27 @@ def to_dict(self): rval["badges"] = self._get_concrete_store_badges(None) return rval - def _get_concrete_store_badges(self, obj) -> List[Dict[str, Any]]: - badge_dicts: List[Dict[str, Any]] = [] + def to_model(self, object_store_id: str) -> "ConcreteObjectStoreModel": + return ConcreteObjectStoreModel( + object_store_id=object_store_id, + private=self.private, + name=self.name, + description=self.description, + quota=QuotaModel(source=self.quota_source, enabled=self.quota_enabled), + badges=self._get_concrete_store_badges(None), + ) + + def _get_concrete_store_badges(self, obj) -> List[BadgeDict]: + badge_dicts: List[BadgeDict] = [] for badge in self.badges: - badge_dict = badge.copy() - badge_dict["source"] = "admin" + badge_dict: BadgeDict = { + "source": "admin", + "type": badge["type"], + "message": badge["message"], + } badge_dicts.append(badge_dict) - quota_badge_dict: Dict[str, Any] + quota_badge_dict: BadgeDict if self.galaxy_enable_quotas and self.quota_enabled: quota_badge_dict = { "type": "quota", @@ -571,7 +611,7 @@ def _get_concrete_store_badges(self, obj) -> List[Dict[str, Any]]: } badge_dicts.append(quota_badge_dict) if self.private: - restricted_badge_dict = { + restricted_badge_dict: BadgeDict = { "type": "restricted", "message": None, "source": "galaxy", @@ -981,7 +1021,7 @@ def _get_concrete_store_name(self, obj): def _get_concrete_store_description_markdown(self, obj): return self._call_method("_get_concrete_store_description_markdown", obj, None, False) - def _get_concrete_store_badges(self, obj): + def _get_concrete_store_badges(self, obj) -> List[BadgeDict]: return self._call_method("_get_concrete_store_badges", obj, [], False) def _is_private(self, obj): @@ -1348,6 +1388,20 @@ def get_quota_source_map(self): return quota_source_map +class QuotaModel(BaseModel): + source: Optional[str] + enabled: bool + + +class ConcreteObjectStoreModel(BaseModel): + object_store_id: Optional[str] + private: bool + name: Optional[str] + description: Optional[str] + quota: QuotaModel + badges: List[BadgeDict] + + def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[BaseObjectStore], Dict[str, Any]]: objectstore_class: Type[BaseObjectStore] objectstore_constructor_kwds = {} diff --git a/lib/galaxy/webapps/galaxy/api/object_store.py b/lib/galaxy/webapps/galaxy/api/object_store.py index f53e267339a2..3e3a81442d04 100644 --- a/lib/galaxy/webapps/galaxy/api/object_store.py +++ b/lib/galaxy/webapps/galaxy/api/object_store.py @@ -2,11 +2,7 @@ API operations on Galaxy's object store. """ import logging -from typing import ( - Any, - Dict, - List, -) +from typing import List from fastapi import ( Path, @@ -18,7 +14,10 @@ RequestParameterInvalidException, ) from galaxy.managers.context import ProvidesUserContext -from galaxy.objectstore import BaseObjectStore +from galaxy.objectstore import ( + BaseObjectStore, + ConcreteObjectStoreModel, +) from . import ( depends, DependsOnTrans, @@ -34,7 +33,9 @@ ) SelectableQueryParam: bool = Query( - default=False, title="Selectable", description="Restrict index query to user selectable object stores." + default=False, + title="Selectable", + description="Restrict index query to user selectable object stores, the current implementation requires this to be true.", ) @@ -44,37 +45,34 @@ class FastAPIObjectStore: @router.get( "/api/object_store", - summary="", - response_description="", + summary="Get a list of (currently only concrete) object stores configured with this Galaxy instance.", + response_description="A list of the configured object stores.", ) def index( self, trans: ProvidesUserContext = DependsOnTrans, selectable: bool = SelectableQueryParam, - ) -> List[Dict[str, Any]]: + ) -> List[ConcreteObjectStoreModel]: if not selectable: raise RequestParameterInvalidException( "The object store index query currently needs to be called with selectable=true" ) selectable_ids = self.object_store.object_store_ids_allowing_selection() - return [self._dict_for(selectable_id) for selectable_id in selectable_ids] + return [self._model_for(selectable_id) for selectable_id in selectable_ids] @router.get( "/api/object_store/{object_store_id}", - summary="Return boolean to indicate if Galaxy's default object store allows selection.", - response_description="A list with details about the remote files available to the user.", + summary="Get information about a concrete object store configured with Galaxy.", ) def show_info( self, trans: ProvidesUserContext = DependsOnTrans, object_store_id: str = ConcreteObjectStoreIdPathParam, - ) -> Dict[str, Any]: - return self._dict_for(object_store_id) + ) -> ConcreteObjectStoreModel: + return self._model_for(object_store_id) - def _dict_for(self, object_store_id: str) -> Dict[str, Any]: + def _model_for(self, object_store_id: str) -> ConcreteObjectStoreModel: concrete_object_store = self.object_store.get_concrete_store_by_object_store_id(object_store_id) if concrete_object_store is None: raise ObjectNotFound() - as_dict = concrete_object_store.to_dict() - as_dict["object_store_id"] = object_store_id - return as_dict + return concrete_object_store.to_model(object_store_id) From 5462b7a3a214bacd0469de8dc10204363dce6457 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 27 Feb 2023 10:01:00 -0500 Subject: [PATCH 2/8] Convert SelectObjectStore to fetcher pattern. --- client/src/components/ObjectStore/SelectObjectStore.vue | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/src/components/ObjectStore/SelectObjectStore.vue b/client/src/components/ObjectStore/SelectObjectStore.vue index 2d3096f4b04c..1cd0aa5a2a04 100644 --- a/client/src/components/ObjectStore/SelectObjectStore.vue +++ b/client/src/components/ObjectStore/SelectObjectStore.vue @@ -57,8 +57,7 @@ + - - From c40672ac19eef9c47b39f930b34930b30f8c706e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 2 Mar 2023 15:32:42 -0500 Subject: [PATCH 7/8] Only show objectstore selection in the GUI if actual selection makes sense --- .../History/CurrentHistory/HistoryCounter.vue | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/client/src/components/History/CurrentHistory/HistoryCounter.vue b/client/src/components/History/CurrentHistory/HistoryCounter.vue index 30ef239cf236..0c41424f55f0 100644 --- a/client/src/components/History/CurrentHistory/HistoryCounter.vue +++ b/client/src/components/History/CurrentHistory/HistoryCounter.vue @@ -12,19 +12,25 @@ {{ historySize | niceFileSize }} - - - - - + + + + + + + + + Date: Thu, 2 Mar 2023 15:58:22 -0500 Subject: [PATCH 8/8] More consistent icon for object store selection. I had started by using fa-database everywhere but then I implemented the user selection piece and that icon was already used for custom DB keys and then David implemented the storage management functionality and so that icon couldn't be used in the history either. This I think switches everything else over to the now more consitent fa-hdd. --- client/src/components/Tool/ToolCard.vue | 2 +- .../components/Workflow/Run/WorkflowStorageConfiguration.vue | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/components/Tool/ToolCard.vue b/client/src/components/Tool/ToolCard.vue index f877b14af41c..72100d6c6e84 100644 --- a/client/src/components/Tool/ToolCard.vue +++ b/client/src/components/Tool/ToolCard.vue @@ -130,7 +130,7 @@ function onUpdatePreferredObjectStoreId(selectedToolPreferredObjectStoreId) { size="sm" class="float-right" @click="onShowObjectStoreSelect"> - + - + - +