From fc71a536aa701e7553c2612abf8d1105205c4e55 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 25 Apr 2024 02:55:28 -0400 Subject: [PATCH] WIP: file source templates --- client/src/api/configTemplates.ts | 21 + client/src/api/fileSources.ts | 6 + client/src/api/schema/schema.ts | 252 ++++++++++- .../ConfigTemplates/CreateInstance.vue | 22 + .../ConfigTemplates/EditSecretsForm.vue | 31 ++ .../ConfigTemplates/InstanceDropdown.vue | 50 +++ .../InstanceForm.vue | 9 +- .../ConfigTemplates/ManageIndexHeader.vue | 35 ++ .../ConfigTemplates/SelectTemplate.vue | 44 ++ .../VaultSecret.vue | 0 .../src/components/ConfigTemplates/fields.ts | 25 ++ .../components/ConfigTemplates/formUtil.ts | 158 +++++++ .../Instances => ConfigTemplates}/icons.ts | 0 .../FileSources/Instances/CreateForm.vue | 53 +++ .../FileSources/Instances/CreateInstance.vue | 34 ++ .../FileSources/Instances/EditInstance.vue | 77 ++++ .../FileSources/Instances/EditSecrets.vue | 28 ++ .../Instances/InstanceDropdown.vue | 30 ++ .../FileSources/Instances/ManageIndex.vue | 68 +++ .../FileSources/Instances/UpgradeForm.vue | 63 +++ .../FileSources/Instances/UpgradeInstance.vue | 27 ++ .../FileSources/Instances/instance.ts | 23 + .../FileSources/Instances/routing.ts | 16 + .../FileSources/Instances/services.ts | 4 + .../Templates/CreateUserFileSource.vue | 30 ++ .../FileSources/Templates/SelectTemplate.vue | 41 ++ .../ObjectStore/Instances/CreateForm.vue | 61 +-- .../ObjectStore/Instances/EditInstance.vue | 78 ++-- .../ObjectStore/Instances/EditSecrets.vue | 14 +- .../Instances/InstanceDropdown.vue | 42 +- .../ObjectStore/Instances/ManageIndex.vue | 65 +-- .../ObjectStore/Instances/UpgradeForm.vue | 52 +-- .../components/ObjectStore/Instances/types.ts | 5 - .../components/ObjectStore/Instances/util.ts | 41 -- .../Templates/CreateUserObjectStore.vue | 19 +- .../ObjectStore/Templates/SelectTemplate.vue | 28 +- .../src/components/User/UserPreferences.vue | 28 +- client/src/entry/analysis/router.js | 47 +- client/src/stores/configTemplatesUtil.ts | 56 +++ client/src/stores/fileSourceInstancesStore.ts | 51 +++ client/src/stores/fileSourceTemplatesStore.ts | 70 +++ .../src/stores/objectStoreTemplatesStore.ts | 49 +- lib/galaxy/app.py | 39 +- lib/galaxy/app_unittest_utils/galaxy_mock.py | 23 + lib/galaxy/config/schemas/config_schema.yml | 15 + lib/galaxy/files/__init__.py | 136 +++++- lib/galaxy/files/plugins.py | 5 + lib/galaxy/files/sources/__init__.py | 11 +- lib/galaxy/files/templates/__init__.py | 17 + lib/galaxy/files/templates/manager.py | 86 ++++ lib/galaxy/files/templates/models.py | 125 ++++++ lib/galaxy/files/unittest_utils/__init__.py | 7 +- lib/galaxy/managers/file_source_instances.py | 421 ++++++++++++++++++ lib/galaxy/managers/object_store_instances.py | 36 +- lib/galaxy/model/__init__.py | 86 +++- ...a3c93d66b_add_user_defined_file_sources.py | 57 +++ lib/galaxy/objectstore/templates/manager.py | 92 +--- lib/galaxy/objectstore/templates/models.py | 59 +-- lib/galaxy/util/config_templates.py | 167 +++++++ lib/galaxy/webapps/galaxy/api/file_sources.py | 95 ++++ lib/galaxy/webapps/galaxy/buildapp.py | 5 + packages/util/setup.cfg | 2 + .../app/managers/test_user_file_sources.py | 108 +++++ test/unit/files/_util.py | 3 +- test/unit/files/test_posix.py | 29 +- test/unit/files/test_template_manager.py | 58 +++ test/unit/files/test_template_models.py | 91 ++++ 67 files changed, 3062 insertions(+), 564 deletions(-) create mode 100644 client/src/api/configTemplates.ts create mode 100644 client/src/api/fileSources.ts create mode 100644 client/src/components/ConfigTemplates/CreateInstance.vue create mode 100644 client/src/components/ConfigTemplates/EditSecretsForm.vue create mode 100644 client/src/components/ConfigTemplates/InstanceDropdown.vue rename client/src/components/{ObjectStore/Instances => ConfigTemplates}/InstanceForm.vue (79%) create mode 100644 client/src/components/ConfigTemplates/ManageIndexHeader.vue create mode 100644 client/src/components/ConfigTemplates/SelectTemplate.vue rename client/src/components/{ObjectStore/Instances => ConfigTemplates}/VaultSecret.vue (100%) create mode 100644 client/src/components/ConfigTemplates/fields.ts create mode 100644 client/src/components/ConfigTemplates/formUtil.ts rename client/src/components/{ObjectStore/Instances => ConfigTemplates}/icons.ts (100%) create mode 100644 client/src/components/FileSources/Instances/CreateForm.vue create mode 100644 client/src/components/FileSources/Instances/CreateInstance.vue create mode 100644 client/src/components/FileSources/Instances/EditInstance.vue create mode 100644 client/src/components/FileSources/Instances/EditSecrets.vue create mode 100644 client/src/components/FileSources/Instances/InstanceDropdown.vue create mode 100644 client/src/components/FileSources/Instances/ManageIndex.vue create mode 100644 client/src/components/FileSources/Instances/UpgradeForm.vue create mode 100644 client/src/components/FileSources/Instances/UpgradeInstance.vue create mode 100644 client/src/components/FileSources/Instances/instance.ts create mode 100644 client/src/components/FileSources/Instances/routing.ts create mode 100644 client/src/components/FileSources/Instances/services.ts create mode 100644 client/src/components/FileSources/Templates/CreateUserFileSource.vue create mode 100644 client/src/components/FileSources/Templates/SelectTemplate.vue delete mode 100644 client/src/components/ObjectStore/Instances/util.ts create mode 100644 client/src/stores/configTemplatesUtil.ts create mode 100644 client/src/stores/fileSourceInstancesStore.ts create mode 100644 client/src/stores/fileSourceTemplatesStore.ts create mode 100644 lib/galaxy/files/templates/__init__.py create mode 100644 lib/galaxy/files/templates/manager.py create mode 100644 lib/galaxy/files/templates/models.py create mode 100644 lib/galaxy/managers/file_source_instances.py create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py create mode 100644 lib/galaxy/util/config_templates.py create mode 100644 lib/galaxy/webapps/galaxy/api/file_sources.py create mode 100644 test/unit/app/managers/test_user_file_sources.py create mode 100644 test/unit/files/test_template_manager.py create mode 100644 test/unit/files/test_template_models.py diff --git a/client/src/api/configTemplates.ts b/client/src/api/configTemplates.ts new file mode 100644 index 000000000000..c9679f5876fa --- /dev/null +++ b/client/src/api/configTemplates.ts @@ -0,0 +1,21 @@ +import type { components } from "@/api/schema/schema"; + +export type Instance = + | components["schemas"]["UserFileSourceModel"] + | components["schemas"]["UserConcreteObjectStoreModel"]; + +export type TemplateVariable = components["schemas"]["TemplateVariable"]; +export type TemplateSecret = components["schemas"]["TemplateSecret"]; +export type VariableValueType = (string | boolean | number) | undefined; +export type VariableData = { [key: string]: VariableValueType }; +export type SecretData = { [key: string]: string }; + +export interface TemplateSummary { + description: string | null; + hidden?: boolean; + id: string; + name: string | null; + secrets?: TemplateSecret[] | null; + variables?: TemplateVariable[] | null; + version?: number; +} diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts new file mode 100644 index 000000000000..68935db4573f --- /dev/null +++ b/client/src/api/fileSources.ts @@ -0,0 +1,6 @@ +import { type components } from "@/api/schema"; + +export type FileSourceTemplateSummary = components["schemas"]["FileSourceTemplateSummary"]; +export type FileSourceTemplateSummaries = FileSourceTemplateSummary[]; + +export type UserFileSourceModel = components["schemas"]["UserFileSourceModel"]; diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 67b97e8f6d7e..503f64a5e93f 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -318,6 +318,22 @@ export interface paths { /** Download */ get: operations["download_api_drs_download__object_id__get"]; }; + "/api/file_source_instances": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_index"]; + /** Create a user-bound object store. */ + post: operations["file_sources__create_instance"]; + }; + "/api/file_source_instances/{user_file_source_id}": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_get"]; + /** Update or upgrade user file source instance. */ + put: operations["file_sources__instances_update"]; + }; + "/api/file_source_templates": { + /** Get a list of file source templates available to build user defined file sources from */ + get: operations["file_sources__templates_index"]; + }; "/api/folders/{folder_id}/contents": { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. @@ -5230,6 +5246,36 @@ export interface components { */ update_time: string; }; + /** FileSourceTemplateSummaries */ + FileSourceTemplateSummaries: components["schemas"]["FileSourceTemplateSummary"][]; + /** FileSourceTemplateSummary */ + FileSourceTemplateSummary: { + /** Description */ + description: string | null; + /** + * Hidden + * @default false + */ + hidden?: boolean; + /** Id */ + id: string; + /** Name */ + name: string | null; + /** Secrets */ + secrets?: components["schemas"]["TemplateSecret"][] | null; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Variables */ + variables?: components["schemas"]["TemplateVariable"][] | null; + /** + * Version + * @default 0 + */ + version?: number; + }; /** FilesSourcePlugin */ FilesSourcePlugin: { /** @@ -9909,13 +9955,6 @@ export interface components { */ up_to_date: boolean; }; - /** ObjectStoreTemplateSecret */ - ObjectStoreTemplateSecret: { - /** Help */ - help: string | null; - /** Name */ - name: string; - }; /** ObjectStoreTemplateSummaries */ ObjectStoreTemplateSummaries: components["schemas"]["ObjectStoreTemplateSummary"][]; /** ObjectStoreTemplateSummary */ @@ -9934,32 +9973,20 @@ export interface components { /** Name */ name: string | null; /** Secrets */ - secrets?: components["schemas"]["ObjectStoreTemplateSecret"][] | null; + secrets?: components["schemas"]["TemplateSecret"][] | null; /** * Type * @enum {string} */ type: "s3" | "azure_blob" | "disk" | "generic_s3"; /** Variables */ - variables?: components["schemas"]["ObjectStoreTemplateVariable"][] | null; + variables?: components["schemas"]["TemplateVariable"][] | null; /** * Version * @default 0 */ version?: number; }; - /** ObjectStoreTemplateVariable */ - ObjectStoreTemplateVariable: { - /** Help */ - help: string | null; - /** Name */ - name: string; - /** - * Type - * @enum {string} - */ - type: "string" | "boolean" | "integer"; - }; /** OutputReferenceByLabel */ OutputReferenceByLabel: { /** @@ -11836,6 +11863,25 @@ export interface components { * @enum {string} */ TaskState: "PENDING" | "STARTED" | "RETRY" | "FAILURE" | "SUCCESS"; + /** TemplateSecret */ + TemplateSecret: { + /** Help */ + help: string | null; + /** Name */ + name: string; + }; + /** TemplateVariable */ + TemplateVariable: { + /** Help */ + help: string | null; + /** Name */ + name: string; + /** + * Type + * @enum {string} + */ + type: "string" | "boolean" | "integer"; + }; /** ToolDataDetails */ ToolDataDetails: { /** @@ -12566,6 +12612,34 @@ export interface components { */ id: string; }; + /** UserFileSourceModel */ + UserFileSourceModel: { + /** Description */ + description: string | null; + /** Id */ + id: string | number; + /** Name */ + name: string; + /** Secrets */ + secrets: string[]; + /** Template Id */ + template_id: string; + /** Template Version */ + template_version: number; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Uri Root */ + uri_root: string; + /** Uuid */ + uuid: string; + /** Variables */ + variables: { + [key: string]: (string | boolean | number) | undefined; + } | null; + }; /** * UserModel * @description User in a transaction context. @@ -14755,6 +14829,142 @@ export interface operations { }; }; }; + file_sources__instances_index: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__create_instance: { + /** Create a user-bound object store. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_get: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_update: { + /** Update or upgrade user file source instance. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + requestBody: { + content: { + "application/json": + | components["schemas"]["UpdateInstanceSecretPayload"] + | components["schemas"]["UpgradeInstancePayload"] + | components["schemas"]["UpdateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__templates_index: { + /** Get a list of file source templates available to build user defined file sources from */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description A list of the configured file source templates. */ + 200: { + content: { + "application/json": components["schemas"]["FileSourceTemplateSummaries"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_folders__folder_id__contents_get: { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. diff --git a/client/src/components/ConfigTemplates/CreateInstance.vue b/client/src/components/ConfigTemplates/CreateInstance.vue new file mode 100644 index 000000000000..061b29e250e3 --- /dev/null +++ b/client/src/components/ConfigTemplates/CreateInstance.vue @@ -0,0 +1,22 @@ + + + diff --git a/client/src/components/ConfigTemplates/EditSecretsForm.vue b/client/src/components/ConfigTemplates/EditSecretsForm.vue new file mode 100644 index 000000000000..764a6d81294f --- /dev/null +++ b/client/src/components/ConfigTemplates/EditSecretsForm.vue @@ -0,0 +1,31 @@ + + + diff --git a/client/src/components/ConfigTemplates/InstanceDropdown.vue b/client/src/components/ConfigTemplates/InstanceDropdown.vue new file mode 100644 index 000000000000..e825479bc3e3 --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceDropdown.vue @@ -0,0 +1,50 @@ + + + diff --git a/client/src/components/ObjectStore/Instances/InstanceForm.vue b/client/src/components/ConfigTemplates/InstanceForm.vue similarity index 79% rename from client/src/components/ObjectStore/Instances/InstanceForm.vue rename to client/src/components/ConfigTemplates/InstanceForm.vue index 97d4525aa4cb..7e9bba13e4aa 100644 --- a/client/src/components/ObjectStore/Instances/InstanceForm.vue +++ b/client/src/components/ConfigTemplates/InstanceForm.vue @@ -1,4 +1,6 @@