From e259efa403b198826b1fe4d4b03daa893227f84a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 24 Apr 2024 12:21:26 -0400 Subject: [PATCH] WIP: file source templates --- client/src/api/schema/schema.ts | 252 ++++++++++- .../ConfigTemplates/ManageIndexHeader.vue | 35 ++ .../src/components/ConfigTemplates/fields.ts | 25 ++ .../src/components/ConfigTemplates/icons.ts | 5 + .../FileSources/Instances/ManageIndex.vue | 68 +++ .../components/FileSources/ManageIndex.vue | 0 .../ObjectStore/Instances/ManageIndex.vue | 65 +-- .../components/ObjectStore/Instances/types.ts | 4 +- .../components/ObjectStore/Instances/util.ts | 6 +- .../src/components/User/UserPreferences.vue | 28 +- client/src/entry/analysis/router.js | 12 +- client/src/stores/configTemplatesUtil.ts | 66 +++ client/src/stores/fileSourceInstancesStore.ts | 55 +++ client/src/stores/fileSourceTemplatesStore.ts | 70 +++ .../src/stores/objectStoreTemplatesStore.ts | 49 +- lib/galaxy/app.py | 39 +- lib/galaxy/app_unittest_utils/galaxy_mock.py | 22 + lib/galaxy/config/schemas/config_schema.yml | 15 + lib/galaxy/files/__init__.py | 136 +++++- lib/galaxy/files/plugins.py | 5 + lib/galaxy/files/sources/__init__.py | 11 +- lib/galaxy/files/templates/__init__.py | 17 + lib/galaxy/files/templates/manager.py | 86 ++++ lib/galaxy/files/templates/models.py | 125 ++++++ lib/galaxy/files/unittest_utils/__init__.py | 7 +- lib/galaxy/managers/file_source_instances.py | 420 ++++++++++++++++++ lib/galaxy/managers/object_store_instances.py | 36 +- lib/galaxy/model/__init__.py | 86 +++- ...a3c93d66b_add_user_defined_file_sources.py | 57 +++ lib/galaxy/objectstore/templates/manager.py | 92 +--- lib/galaxy/objectstore/templates/models.py | 59 +-- lib/galaxy/util/config_templates.py | 167 +++++++ lib/galaxy/webapps/galaxy/api/file_sources.py | 95 ++++ lib/galaxy/webapps/galaxy/buildapp.py | 5 + packages/util/setup.cfg | 2 + .../app/managers/test_user_file_sources.py | 109 +++++ test/unit/files/_util.py | 3 +- test/unit/files/test_posix.py | 29 +- test/unit/files/test_template_manager.py | 58 +++ test/unit/files/test_template_models.py | 91 ++++ 40 files changed, 2206 insertions(+), 306 deletions(-) create mode 100644 client/src/components/ConfigTemplates/ManageIndexHeader.vue create mode 100644 client/src/components/ConfigTemplates/fields.ts create mode 100644 client/src/components/ConfigTemplates/icons.ts create mode 100644 client/src/components/FileSources/Instances/ManageIndex.vue create mode 100644 client/src/components/FileSources/ManageIndex.vue create mode 100644 client/src/stores/configTemplatesUtil.ts create mode 100644 client/src/stores/fileSourceInstancesStore.ts create mode 100644 client/src/stores/fileSourceTemplatesStore.ts create mode 100644 lib/galaxy/files/templates/__init__.py create mode 100644 lib/galaxy/files/templates/manager.py create mode 100644 lib/galaxy/files/templates/models.py create mode 100644 lib/galaxy/managers/file_source_instances.py create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py create mode 100644 lib/galaxy/util/config_templates.py create mode 100644 lib/galaxy/webapps/galaxy/api/file_sources.py create mode 100644 test/unit/app/managers/test_user_file_sources.py create mode 100644 test/unit/files/test_template_manager.py create mode 100644 test/unit/files/test_template_models.py diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 67b97e8f6d7e..503f64a5e93f 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -318,6 +318,22 @@ export interface paths { /** Download */ get: operations["download_api_drs_download__object_id__get"]; }; + "/api/file_source_instances": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_index"]; + /** Create a user-bound object store. */ + post: operations["file_sources__create_instance"]; + }; + "/api/file_source_instances/{user_file_source_id}": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_get"]; + /** Update or upgrade user file source instance. */ + put: operations["file_sources__instances_update"]; + }; + "/api/file_source_templates": { + /** Get a list of file source templates available to build user defined file sources from */ + get: operations["file_sources__templates_index"]; + }; "/api/folders/{folder_id}/contents": { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. @@ -5230,6 +5246,36 @@ export interface components { */ update_time: string; }; + /** FileSourceTemplateSummaries */ + FileSourceTemplateSummaries: components["schemas"]["FileSourceTemplateSummary"][]; + /** FileSourceTemplateSummary */ + FileSourceTemplateSummary: { + /** Description */ + description: string | null; + /** + * Hidden + * @default false + */ + hidden?: boolean; + /** Id */ + id: string; + /** Name */ + name: string | null; + /** Secrets */ + secrets?: components["schemas"]["TemplateSecret"][] | null; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Variables */ + variables?: components["schemas"]["TemplateVariable"][] | null; + /** + * Version + * @default 0 + */ + version?: number; + }; /** FilesSourcePlugin */ FilesSourcePlugin: { /** @@ -9909,13 +9955,6 @@ export interface components { */ up_to_date: boolean; }; - /** ObjectStoreTemplateSecret */ - ObjectStoreTemplateSecret: { - /** Help */ - help: string | null; - /** Name */ - name: string; - }; /** ObjectStoreTemplateSummaries */ ObjectStoreTemplateSummaries: components["schemas"]["ObjectStoreTemplateSummary"][]; /** ObjectStoreTemplateSummary */ @@ -9934,32 +9973,20 @@ export interface components { /** Name */ name: string | null; /** Secrets */ - secrets?: components["schemas"]["ObjectStoreTemplateSecret"][] | null; + secrets?: components["schemas"]["TemplateSecret"][] | null; /** * Type * @enum {string} */ type: "s3" | "azure_blob" | "disk" | "generic_s3"; /** Variables */ - variables?: components["schemas"]["ObjectStoreTemplateVariable"][] | null; + variables?: components["schemas"]["TemplateVariable"][] | null; /** * Version * @default 0 */ version?: number; }; - /** ObjectStoreTemplateVariable */ - ObjectStoreTemplateVariable: { - /** Help */ - help: string | null; - /** Name */ - name: string; - /** - * Type - * @enum {string} - */ - type: "string" | "boolean" | "integer"; - }; /** OutputReferenceByLabel */ OutputReferenceByLabel: { /** @@ -11836,6 +11863,25 @@ export interface components { * @enum {string} */ TaskState: "PENDING" | "STARTED" | "RETRY" | "FAILURE" | "SUCCESS"; + /** TemplateSecret */ + TemplateSecret: { + /** Help */ + help: string | null; + /** Name */ + name: string; + }; + /** TemplateVariable */ + TemplateVariable: { + /** Help */ + help: string | null; + /** Name */ + name: string; + /** + * Type + * @enum {string} + */ + type: "string" | "boolean" | "integer"; + }; /** ToolDataDetails */ ToolDataDetails: { /** @@ -12566,6 +12612,34 @@ export interface components { */ id: string; }; + /** UserFileSourceModel */ + UserFileSourceModel: { + /** Description */ + description: string | null; + /** Id */ + id: string | number; + /** Name */ + name: string; + /** Secrets */ + secrets: string[]; + /** Template Id */ + template_id: string; + /** Template Version */ + template_version: number; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Uri Root */ + uri_root: string; + /** Uuid */ + uuid: string; + /** Variables */ + variables: { + [key: string]: (string | boolean | number) | undefined; + } | null; + }; /** * UserModel * @description User in a transaction context. @@ -14755,6 +14829,142 @@ export interface operations { }; }; }; + file_sources__instances_index: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__create_instance: { + /** Create a user-bound object store. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_get: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_update: { + /** Update or upgrade user file source instance. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + requestBody: { + content: { + "application/json": + | components["schemas"]["UpdateInstanceSecretPayload"] + | components["schemas"]["UpgradeInstancePayload"] + | components["schemas"]["UpdateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__templates_index: { + /** Get a list of file source templates available to build user defined file sources from */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description A list of the configured file source templates. */ + 200: { + content: { + "application/json": components["schemas"]["FileSourceTemplateSummaries"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_folders__folder_id__contents_get: { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. diff --git a/client/src/components/ConfigTemplates/ManageIndexHeader.vue b/client/src/components/ConfigTemplates/ManageIndexHeader.vue new file mode 100644 index 000000000000..2f03901f579a --- /dev/null +++ b/client/src/components/ConfigTemplates/ManageIndexHeader.vue @@ -0,0 +1,35 @@ + + + diff --git a/client/src/components/ConfigTemplates/fields.ts b/client/src/components/ConfigTemplates/fields.ts new file mode 100644 index 000000000000..33a00f7e69f4 --- /dev/null +++ b/client/src/components/ConfigTemplates/fields.ts @@ -0,0 +1,25 @@ +import _l from "@/utils/localization"; + +export const NAME_FIELD = { + key: "name", + label: _l("Name"), + sortable: true, +}; + +export const DESCRIPTION_FIELD = { + key: "description", + label: _l("Description"), + sortable: true, +}; + +export const TYPE_FIELD = { + key: "type", + label: _l("Type"), + sortable: true, +}; + +export const TEMPLATE_FIELD = { + key: "template", + label: _l("From Template"), + sortable: true, +}; diff --git a/client/src/components/ConfigTemplates/icons.ts b/client/src/components/ConfigTemplates/icons.ts new file mode 100644 index 000000000000..0c95a604c25c --- /dev/null +++ b/client/src/components/ConfigTemplates/icons.ts @@ -0,0 +1,5 @@ +import { library } from "@fortawesome/fontawesome-svg-core"; +import { faCaretDown, faPlus } from "@fortawesome/free-solid-svg-icons"; + +library.add(faCaretDown); +library.add(faPlus); diff --git a/client/src/components/FileSources/Instances/ManageIndex.vue b/client/src/components/FileSources/Instances/ManageIndex.vue new file mode 100644 index 000000000000..6331f5f8a236 --- /dev/null +++ b/client/src/components/FileSources/Instances/ManageIndex.vue @@ -0,0 +1,68 @@ + + + diff --git a/client/src/components/FileSources/ManageIndex.vue b/client/src/components/FileSources/ManageIndex.vue new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/client/src/components/ObjectStore/Instances/ManageIndex.vue b/client/src/components/ObjectStore/Instances/ManageIndex.vue index 9695f492dad2..1f87d85ebc09 100644 --- a/client/src/components/ObjectStore/Instances/ManageIndex.vue +++ b/client/src/components/ObjectStore/Instances/ManageIndex.vue @@ -1,20 +1,20 @@